Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
cgroups: add host cgroup support
Browse files Browse the repository at this point in the history 
Fixes #344

Add host cgroup support for kata.

This commits only adds cpu.cfs_period and cpu.cfs_quota support.

It will create 3-level hierarchy, take "cpu" cgroup as an example:

```
/sys/fs/cgroup
|---cpu
   |---vc
      |---<sandbox-id>
         |--vcpu
      |---<sandbox-id>
```

* `vc` cgroup is common parent for all kata-container sandbox, it won't be removed
after sandbox removed. This cgroup has no limitation.
* `<sandbox-id>` cgroup is the layer for each sandbox, it contains all other qemu
threads except for vcpu threads. In future, we can consider putting all shim
processes and proxy process here. This cgroup has no limitation yet.
* `vcpu` cgroup contains vcpu threads from qemu. Currently cpu quota and period
constraint applies to this cgroup.

Signed-off-by: Wei Zhang <[email protected]>
Signed-off-by: Jingxiao Lu <[email protected]>
  • Loading branch information
@WeiZhang555
WeiZhang555 committed Oct 24, 2018
1 parent bfab0a1 commit 1abfb00
Show file tree
Hide file tree
Showing 13 changed files with 468 additions and 586 deletions.
113 changes: 0 additions & 113 deletions cli/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"

vc "github.com/kata-containers/runtime/virtcontainers"
Expand Down Expand Up @@ -177,26 +175,6 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
}
}

// config.json provides a cgroups path that has to be used to create "tasks"
// and "cgroups.procs" files. Those files have to be filled with a PID, which
// is shim's in our case. This is mandatory to make sure there is no one
// else (like Docker) trying to create those files on our behalf. We want to
// know those files location so that we can remove them when delete is called.
cgroupsPathList, err := processCgroupsPath(ctx, ociSpec, containerType.IsSandbox())
if err != nil {
return err
}

// cgroupsDirPath is CgroupsPath fetch from OCI spec
var cgroupsDirPath string
if ociSpec.Linux != nil {
cgroupsDirPath = ociSpec.Linux.CgroupsPath
}

if err := createCgroupsFiles(ctx, containerID, cgroupsDirPath, cgroupsPathList, process.Pid); err != nil {
return err
}

// Creation of PID file has to be the last thing done in the create
// because containerd considers the create complete after this file
// is created.
Expand Down Expand Up @@ -379,52 +357,6 @@ func createContainer(ctx context.Context, ociSpec oci.CompatOCISpec, containerID
return c.Process(), nil
}

func createCgroupsFiles(ctx context.Context, containerID string, cgroupsDirPath string, cgroupsPathList []string, pid int) error {
span, _ := trace(ctx, "createCgroupsFiles")
defer span.Finish()

if len(cgroupsPathList) == 0 {
kataLog.WithField("pid", pid).Info("Cgroups files not created because cgroupsPath was empty")
return nil
}

for _, cgroupsPath := range cgroupsPathList {
if err := os.MkdirAll(cgroupsPath, cgroupsDirMode); err != nil {
return err
}

if strings.Contains(cgroupsPath, "cpu") && cgroupsDirPath != "" {
parent := strings.TrimSuffix(cgroupsPath, cgroupsDirPath)
copyParentCPUSet(cgroupsPath, parent)
}

tasksFilePath := filepath.Join(cgroupsPath, cgroupsTasksFile)
procsFilePath := filepath.Join(cgroupsPath, cgroupsProcsFile)

pidStr := fmt.Sprintf("%d", pid)

for _, path := range []string{tasksFilePath, procsFilePath} {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, cgroupsFileMode)
if err != nil {
return err
}
defer f.Close()

n, err := f.WriteString(pidStr)
if err != nil {
return err
}

if n < len(pidStr) {
return fmt.Errorf("Could not write pid to %q: only %d bytes written out of %d",
path, n, len(pidStr))
}
}
}

return nil
}

func createPIDFile(ctx context.Context, pidFilePath string, pid int) error {
span, _ := trace(ctx, "createPIDFile")
defer span.Finish()
Expand Down Expand Up @@ -457,48 +389,3 @@ func createPIDFile(ctx context.Context, pidFilePath string, pid int) error {

return nil
}

// copyParentCPUSet copies the cpuset.cpus and cpuset.mems from the parent
// directory to the current directory if the file's contents are 0
func copyParentCPUSet(current, parent string) error {
currentCpus, currentMems, err := getCPUSet(current)
if err != nil {
return err
}

parentCpus, parentMems, err := getCPUSet(parent)
if err != nil {
return err
}

if len(parentCpus) < 1 || len(parentMems) < 1 {
return nil
}

var cgroupsFileMode = os.FileMode(0600)
if isEmptyString(currentCpus) {
if err := writeFile(filepath.Join(current, "cpuset.cpus"), string(parentCpus), cgroupsFileMode); err != nil {
return err
}
}

if isEmptyString(currentMems) {
if err := writeFile(filepath.Join(current, "cpuset.mems"), string(parentMems), cgroupsFileMode); err != nil {
return err
}
}

return nil
}

func getCPUSet(parent string) (cpus []byte, mems []byte, err error) {
if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil {
return
}

if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems")); err != nil {
return
}

return cpus, mems, nil
}
122 changes: 0 additions & 122 deletions cli/create_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,22 +36,6 @@ const (

var testStrPID = fmt.Sprintf("%d", testPID)

func mockCPUSetContent(contents map[string]string) error {
for filePath, data := range contents {
if err := writeFile(filePath, data, testFileMode); err != nil {
return err
}
}

return nil
}

func testCreateCgroupsFilesSuccessful(t *testing.T, cgroupsDirPath string, cgroupsPathList []string, pid int) {
if err := createCgroupsFiles(context.Background(), "foo", cgroupsDirPath, cgroupsPathList, pid); err != nil {
t.Fatalf("This test should succeed (cgroupsPath %q, pid %d): %s", cgroupsPathList, pid, err)
}
}

// return the value of the *last* param with the specified key
func findLastParam(key string, params []vc.Param) (string, error) {
if key == "" {
Expand All @@ -74,62 +58,6 @@ func findLastParam(key string, params []vc.Param) (string, error) {
return "", fmt.Errorf("no param called %q found", name)
}

func TestCgroupsFilesEmptyCgroupsPathSuccessful(t *testing.T) {
testCreateCgroupsFilesSuccessful(t, "", []string{}, testPID)
}

func TestCreateCgroupsFilesFailToWriteFile(t *testing.T) {
if os.Geteuid() == 0 {
// The os.FileMode(0000) trick doesn't work for root.
t.Skip(testDisabledNeedNonRoot)
}

assert := assert.New(t)

tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)

// create the file as a directory to force an error
file := filepath.Join(tmpdir, "cgroups-file")
err = os.MkdirAll(file, os.FileMode(0000))
assert.NoError(err)

files := []string{file}

err = createCgroupsFiles(context.Background(), "foo", "cgroups-file", files, testPID)
assert.Error(err)
}

func TestCgroupsFilesNonEmptyCgroupsPathSuccessful(t *testing.T) {
cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
if err != nil {
t.Fatalf("Could not create temporary cgroups directory: %s", err)
}

testCreateCgroupsFilesSuccessful(t, "cgroups-path-", []string{cgroupsPath}, testPID)

defer os.RemoveAll(cgroupsPath)

tasksPath := filepath.Join(cgroupsPath, cgroupsTasksFile)
procsPath := filepath.Join(cgroupsPath, cgroupsProcsFile)

for _, path := range []string{tasksPath, procsPath} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("Path %q should have been created: %s", path, err)
}

fileBytes, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Could not read %q previously created: %s", path, err)
}

if string(fileBytes) != testStrPID {
t.Fatalf("PID %s read from %q different from expected PID %s", string(fileBytes), path, testStrPID)
}
}
}

func TestCreatePIDFileSuccessful(t *testing.T) {
pidDirPath, err := ioutil.TempDir(testDir, "pid-path-")
if err != nil {
Expand Down Expand Up @@ -1087,56 +1015,6 @@ func TestCreateCreateContainer(t *testing.T) {
}
}

func TestCopyParentCPUSetFail(t *testing.T) {
assert := assert.New(t)

cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
assert.NoError(err)
defer os.RemoveAll(cgroupsPath)

err = copyParentCPUSet(cgroupsPath, testDir)
assert.Error(err)
}

func TestCopyParentCPUSetSuccessful(t *testing.T) {
assert := assert.New(t)

cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
assert.NoError(err)
defer os.RemoveAll(cgroupsPath)

cgroupsSrcPath := filepath.Join(cgroupsPath, "src")
err = os.Mkdir(cgroupsSrcPath, testDirMode)
assert.NoError(err)

err = mockCPUSetContent(map[string]string{
filepath.Join(cgroupsSrcPath, "cpuset.cpus"): "0-1",
filepath.Join(cgroupsSrcPath, "cpuset.mems"): "0-1",
})
assert.NoError(err)

cgroupsDstPath := filepath.Join(cgroupsPath, "dst")
err = os.Mkdir(cgroupsDstPath, testDirMode)
assert.NoError(err)

fd, err := os.Create(filepath.Join(cgroupsDstPath, "cpuset.cpus"))
assert.NoError(err)
fd.Close()

fd, err = os.Create(filepath.Join(cgroupsDstPath, "cpuset.mems"))
assert.NoError(err)
fd.Close()

err = copyParentCPUSet(cgroupsDstPath, cgroupsSrcPath)
assert.NoError(err)

currentCpus, currentMems, err := getCPUSet(cgroupsDstPath)
assert.NoError(err)

assert.False(isEmptyString(currentCpus))
assert.False(isEmptyString(currentMems))
}

func TestSetKernelParams(t *testing.T) {
assert := assert.New(t)

Expand Down
14 changes: 1 addition & 13 deletions cli/delete.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,19 +123,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
return err
}

// In order to prevent any file descriptor leak related to cgroups files
// that have been previously created, we have to remove them before this
// function returns.
cgroupsPathList, err := processCgroupsPath(ctx, ociSpec, containerType.IsSandbox())
if err != nil {
return err
}

if err := delContainerIDMapping(ctx, containerID); err != nil {
return err
}

return removeCgroupsPath(ctx, containerID, cgroupsPathList)
return delContainerIDMapping(ctx, containerID)
}

func deleteSandbox(ctx context.Context, sandboxID string) error {
Expand Down
Loading

0 comments on commit 1abfb00

Please sign in to comment.