From 8a9f5c26c8958a7b9de59f6929b9127c54e13f6e Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 2 Feb 2016 11:58:30 -0800 Subject: [PATCH 01/20] Create the task and alloc dirs before proceeding --- client/driver/rkt.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 2b0f6218ab2..17f82e6c8ad 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -147,8 +147,9 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e cmdArgs = append(cmdArgs, "--insecure-options=all") } - d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)). - SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build() + d.taskEnv.Build() + d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) + d.taskEnv.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) for k, v := range d.taskEnv.EnvMap() { cmdArgs = append(cmdArgs, fmt.Sprintf("--set-env=%v=%v", k, v)) From 848404a48d0c17f6b048fc46503a1c753a464bb7 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 2 Feb 2016 12:00:07 -0800 Subject: [PATCH 02/20] Use a host volume to mount alloc and task dirs host volumes are the preferred way to share data. --- client/driver/rkt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 17f82e6c8ad..c959301dd41 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -163,7 +163,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e if !ok { return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) } - cmdArgs = append(cmdArgs, fmt.Sprintf("--volume %s,kind=empty,readOnly=false,source=%s --mount volume=data,target=%s", task.Name, local, ctx.AllocDir.SharedDir)) + cmdArgs = append(cmdArgs, fmt.Sprintf("--volume %s,kind=host,readOnly=false,source=%s --mount volume=%s,target=%s", task.Name, local, task.Name, ctx.AllocDir.SharedDir)) // Check if the user has overriden the exec command. if execCmd, ok := task.Config["command"]; ok { From adf0c0677f92e05c16370d24517a17ae3a968a28 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Fri, 5 Feb 2016 13:28:41 -0800 Subject: [PATCH 03/20] A number of rkt fixes - Reorder rkt arguments to make it work - Change a few things in test --- client/driver/rkt.go | 28 +++++++++++++++++----------- client/driver/rkt_test.go | 6 ------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index c959301dd41..44d88ab30df 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -132,9 +132,10 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e // Add the given trust prefix trustPrefix, trustCmd := task.Config["trust_prefix"] + insecure := false if trustCmd { var outBuf, errBuf bytes.Buffer - cmd := exec.Command("rkt", "trust", fmt.Sprintf("--prefix=%s", trustPrefix)) + cmd := exec.Command("rkt", "trust", "--skip-fingerprint-review=true", fmt.Sprintf("--prefix=%s", trustPrefix)) cmd.Stdout = &outBuf cmd.Stderr = &errBuf if err := cmd.Run(); err != nil { @@ -144,27 +145,32 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e d.logger.Printf("[DEBUG] driver.rkt: added trust prefix: %q", trustPrefix) } else { // Disble signature verification if the trust command was not run. + insecure = true + } + + local, ok := ctx.AllocDir.TaskDirs[task.Name] + if !ok { + return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) + } + + cmdArgs = append(cmdArgs, "run") + cmdArgs = append(cmdArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", task.Name, local)) + cmdArgs = append(cmdArgs, fmt.Sprintf("--mount=volume=%s,target=%s", task.Name, ctx.AllocDir.SharedDir)) + cmdArgs = append(cmdArgs, img) + if insecure == true { cmdArgs = append(cmdArgs, "--insecure-options=all") } + // Build task and alloc dirs d.taskEnv.Build() d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) d.taskEnv.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) + // Inject enviornment variables for k, v := range d.taskEnv.EnvMap() { cmdArgs = append(cmdArgs, fmt.Sprintf("--set-env=%v=%v", k, v)) } - // Append the run command. - cmdArgs = append(cmdArgs, "run", "--mds-register=false", img) - - // Mount allc and task dirs - local, ok := ctx.AllocDir.TaskDirs[task.Name] - if !ok { - return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) - } - cmdArgs = append(cmdArgs, fmt.Sprintf("--volume %s,kind=host,readOnly=false,source=%s --mount volume=%s,target=%s", task.Name, local, task.Name, ctx.AllocDir.SharedDir)) - // Check if the user has overriden the exec command. if execCmd, ok := task.Config["command"]; ok { cmdArgs = append(cmdArgs, fmt.Sprintf("--exec=%v", execCmd)) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index f3ee857fe4a..a4bc01201c6 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -91,7 +91,6 @@ func TestRktDriver_Start(t *testing.T) { } driverCtx, execCtx := testDriverContexts(task) - defer execCtx.AllocDir.Destroy() d := NewRktDriver(driverCtx) handle, err := d.Start(execCtx, task) @@ -110,11 +109,6 @@ func TestRktDriver_Start(t *testing.T) { if handle2 == nil { t.Fatalf("missing handle") } - - // Clean up - if err := handle.Kill(); err != nil { - fmt.Printf("\nError killing Rkt test: %s", err) - } } func TestRktDriver_Start_Wait(t *testing.T) { From 0905dccb1cac3f9eb30995b5ea6c78eaa6733c51 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Fri, 5 Feb 2016 13:48:27 -0800 Subject: [PATCH 04/20] Enable running rkt tests in travis - Introduces a travis target in Makefile to update docker and install rkt --- .travis.yml | 2 +- GNUmakefile | 11 +++-------- scripts/install_rkt.sh | 11 +++++++++++ scripts/update_docker.sh | 7 +++++++ 4 files changed, 22 insertions(+), 9 deletions(-) create mode 100755 scripts/install_rkt.sh create mode 100755 scripts/update_docker.sh diff --git a/.travis.yml b/.travis.yml index 59038bceb8f..281d3ad5281 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ branches: - master install: - - make prepare_docker + - make travis - make bootstrap script: diff --git a/GNUmakefile b/GNUmakefile index 769a74c02fb..26fadfc834f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -72,13 +72,8 @@ bootstrap: go get $$tool; \ done -prepare_docker: - sudo stop docker - sudo rm -rf /var/lib/docker - sudo rm -f `which docker` - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | sudo tee /etc/apt/sources.list.d/docker.list - sudo apt-get update - sudo apt-get install docker-engine=$(DOCKER_VERSION)-0~$(shell lsb_release -cs) -y --force-yes +travis: + @sh -c "'$(PWD)/scripts/update_docker.sh'" + @sh -c "'$(PWD)/scripts/install_rkt.sh'" .PHONY: all bin cov integ test vet web web-push test-nodep diff --git a/scripts/install_rkt.sh b/scripts/install_rkt.sh new file mode 100755 index 00000000000..13209b12f42 --- /dev/null +++ b/scripts/install_rkt.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +RKT_VERSION="v1.0.0" +DEST_DIR="/usr/local/bin" + +wget https://github.com/coreos/rkt/releases/download/$RKT_VERSION/rkt-$RKT_VERSION.tar.gz +tar xzvf rkt-$RKT_VERSION.tar.gz +sudo cp rkt-$RKT_VERSION/rkt $DEST_DIR +sudo cp rkt-$RKT_VERSION/*.aci $DEST_DIR diff --git a/scripts/update_docker.sh b/scripts/update_docker.sh new file mode 100755 index 00000000000..29fec35ea54 --- /dev/null +++ b/scripts/update_docker.sh @@ -0,0 +1,7 @@ +sudo stop docker +sudo rm -rf /var/lib/docker +sudo rm -f `which docker` +sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D +echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | sudo tee /etc/apt/sources.list.d/docker.list +sudo apt-get update +sudo apt-get install docker-engine=$(DOCKER_VERSION)-0~$(shell lsb_release -cs) -y --force-yes From e7413a57f5ec66c00acfbac9213ca570953cdf6b Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Fri, 5 Feb 2016 13:59:59 -0800 Subject: [PATCH 05/20] Add a shebang to docker update script --- scripts/update_docker.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/update_docker.sh b/scripts/update_docker.sh index 29fec35ea54..01946ec96e9 100755 --- a/scripts/update_docker.sh +++ b/scripts/update_docker.sh @@ -1,3 +1,7 @@ +#!/bin/bash + +set -ex + sudo stop docker sudo rm -rf /var/lib/docker sudo rm -f `which docker` From db549c68fbfcbfdee457fa63243ea5cd4e388d44 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Fri, 5 Feb 2016 14:03:38 -0800 Subject: [PATCH 06/20] Put docker version in the script --- .travis.yml | 3 --- scripts/update_docker.sh | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 281d3ad5281..7829769448f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,6 @@ language: go go: - 1.6 -env: - - DOCKER_VERSION=1.9.1 - matrix: allow_failures: - go: tip diff --git a/scripts/update_docker.sh b/scripts/update_docker.sh index 01946ec96e9..f609ec54438 100755 --- a/scripts/update_docker.sh +++ b/scripts/update_docker.sh @@ -2,10 +2,12 @@ set -ex +DOCKER_VERSION="1.9.1" + sudo stop docker sudo rm -rf /var/lib/docker sudo rm -f `which docker` sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | sudo tee /etc/apt/sources.list.d/docker.list sudo apt-get update -sudo apt-get install docker-engine=$(DOCKER_VERSION)-0~$(shell lsb_release -cs) -y --force-yes +sudo apt-get install docker-engine=$DOCKER_VERSION-0~`lsb_release -cs` -y --force-yes From f2a12188818b3f70195c46db1e468aa8f70bcc8f Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Fri, 5 Feb 2016 15:08:33 -0800 Subject: [PATCH 07/20] Print rkt and docker versions after installing --- scripts/install_rkt.sh | 2 ++ scripts/update_docker.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/install_rkt.sh b/scripts/install_rkt.sh index 13209b12f42..ea00a4e83c6 100755 --- a/scripts/install_rkt.sh +++ b/scripts/install_rkt.sh @@ -9,3 +9,5 @@ wget https://github.com/coreos/rkt/releases/download/$RKT_VERSION/rkt-$RKT_VERSI tar xzvf rkt-$RKT_VERSION.tar.gz sudo cp rkt-$RKT_VERSION/rkt $DEST_DIR sudo cp rkt-$RKT_VERSION/*.aci $DEST_DIR + +rkt version diff --git a/scripts/update_docker.sh b/scripts/update_docker.sh index f609ec54438..e1cc6617620 100755 --- a/scripts/update_docker.sh +++ b/scripts/update_docker.sh @@ -11,3 +11,5 @@ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58 echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | sudo tee /etc/apt/sources.list.d/docker.list sudo apt-get update sudo apt-get install docker-engine=$DOCKER_VERSION-0~`lsb_release -cs` -y --force-yes + +docker version From 1a9438cdd12f9e6a68eba943d78343184460f734 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Sat, 6 Feb 2016 00:23:56 -0800 Subject: [PATCH 08/20] Use less resources for containers --- client/driver/rkt_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index a4bc01201c6..3dca2d23f3a 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -85,8 +85,8 @@ func TestRktDriver_Start(t *testing.T) { "command": "/etcd", }, Resources: &structs.Resources{ - MemoryMB: 256, - CPU: 512, + MemoryMB: 128, + CPU: 100, }, } @@ -122,8 +122,8 @@ func TestRktDriver_Start_Wait(t *testing.T) { "args": []string{"--version"}, }, Resources: &structs.Resources{ - MemoryMB: 256, - CPU: 512, + MemoryMB: 128, + CPU: 100, }, } @@ -166,8 +166,8 @@ func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) { "args": []string{"--version"}, }, Resources: &structs.Resources{ - MemoryMB: 256, - CPU: 512, + MemoryMB: 128, + CPU: 100, }, } @@ -211,8 +211,8 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) { "args": []string{"--version"}, }, Resources: &structs.Resources{ - MemoryMB: 256, - CPU: 512, + MemoryMB: 128, + CPU: 100, }, } From 07d4152017d7ad687586e05f5a97daaf74898c47 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Wed, 10 Feb 2016 12:24:18 -0800 Subject: [PATCH 09/20] Change how a result is returned form wait --- client/driver/rkt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 44d88ab30df..5ea239ff86c 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -312,6 +312,6 @@ func (h *rktHandle) run() { // TODO: Better exit code parsing. code = 1 } - h.waitCh <- cstructs.NewWaitResult(code, 0, err) + h.waitCh <- &cstructs.WaitResult{ExitCode: code, Signal: 0, Err: err} close(h.waitCh) } From 877dd7de61bf8a22287178af22afe569531bddcb Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Wed, 10 Feb 2016 15:55:24 -0800 Subject: [PATCH 10/20] Defer cleanup --- client/driver/rkt_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index 3dca2d23f3a..bf883833553 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -91,6 +91,8 @@ func TestRktDriver_Start(t *testing.T) { } driverCtx, execCtx := testDriverContexts(task) + defer execCtx.AllocDir.Destroy() + d := NewRktDriver(driverCtx) handle, err := d.Start(execCtx, task) @@ -100,6 +102,7 @@ func TestRktDriver_Start(t *testing.T) { if handle == nil { t.Fatalf("missing handle") } + defer handle.Kill() // Attempt to open handle2, err := d.Open(execCtx, handle.ID()) @@ -249,6 +252,6 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) { } if len(data) == 0 { - t.Fatal("Task's stdout is empty") + t.Fatal("Task's stdout is empty: %q", stdout) } } From fb2607bcec55f9cd0e2b9a303caee6c54be6339b Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 16 Feb 2016 16:10:45 -0800 Subject: [PATCH 11/20] Use the plugin mechanism in the rkt driver --- client/driver/rkt.go | 148 ++++++++++++++++++++++---------------- client/driver/rkt_test.go | 34 +++++---- 2 files changed, 101 insertions(+), 81 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 5ea239ff86c..f30608b53d5 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log" - "os" "os/exec" "path/filepath" "regexp" @@ -14,11 +13,14 @@ import ( "syscall" "time" + "github.com/hashicorp/go-plugin" "github.com/hashicorp/go-version" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" + "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" ) @@ -53,20 +55,27 @@ type RktDriverConfig struct { // rktHandle is returned from Start/Open as a handle to the PID type rktHandle struct { - proc *os.Process - image string - logger *log.Logger - killTimeout time.Duration - waitCh chan *cstructs.WaitResult - doneCh chan struct{} + pluginClient *plugin.Client + userPid int + executor executor.Executor + taskDir string + allocDir *allocdir.AllocDir + image string + logger *log.Logger + killTimeout time.Duration + waitCh chan *cstructs.WaitResult + doneCh chan struct{} } // rktPID is a struct to map the pid running the process to the vm image on // disk type rktPID struct { - Pid int - Image string - KillTimeout time.Duration + PluginConfig *PluginReattachConfig + TaskDir string + AllocDir *allocdir.AllocDir + UserPid int + Image string + KillTimeout time.Duration } // NewRktDriver is used to create a new exec driver @@ -125,7 +134,6 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e if !ok { return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) // Build the command. var cmdArgs []string @@ -148,10 +156,10 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e insecure = true } - local, ok := ctx.AllocDir.TaskDirs[task.Name] - if !ok { - return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) - } + local, ok := ctx.AllocDir.TaskDirs[task.Name] + if !ok { + return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) + } cmdArgs = append(cmdArgs, "run") cmdArgs = append(cmdArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", task.Name, local)) @@ -161,11 +169,6 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e cmdArgs = append(cmdArgs, "--insecure-options=all") } - // Build task and alloc dirs - d.taskEnv.Build() - d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) - d.taskEnv.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) - // Inject enviornment variables for k, v := range d.taskEnv.EnvMap() { cmdArgs = append(cmdArgs, fmt.Sprintf("--set-env=%v=%v", k, v)) @@ -203,36 +206,45 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e } } - // Create files to capture stdin and out. - stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName)) - stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName)) - - stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + bin, err := discover.NomadExecutable() if err != nil { - return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err) + return nil, fmt.Errorf("unable to find the nomad binary: %v", err) } - stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) - if err != nil { - return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err) + pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name)) + pluginConfig := &plugin.ClientConfig{ + Cmd: exec.Command(bin, "executor", pluginLogFile), } - cmd := exec.Command("rkt", cmdArgs...) - cmd.Stdout = stdo - cmd.Stderr = stde + exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + if err != nil { + return nil, err + } + executorCtx := &executor.ExecutorContext{ + TaskEnv: d.taskEnv, + AllocDir: ctx.AllocDir, + TaskName: task.Name, + TaskResources: task.Resources, + UnprivilegedUser: false, + LogConfig: task.LogConfig, + } - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("Error running rkt: %v", err) + ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: "/usr/bin/rkt", Args: cmdArgs}, executorCtx) + if err != nil { + pluginClient.Kill() + return nil, fmt.Errorf("error starting process via the plugin: %v", err) } - d.logger.Printf("[DEBUG] driver.rkt: started ACI %q with: %v", img, cmd.Args) + d.logger.Printf("[DEBUG] driver.rkt: started ACI %q with: %v", img, cmdArgs) h := &rktHandle{ - proc: cmd.Process, - image: img, - logger: d.logger, - killTimeout: d.DriverContext.KillTimeout(task), - doneCh: make(chan struct{}), - waitCh: make(chan *cstructs.WaitResult, 1), + pluginClient: pluginClient, + executor: exec, + image: img, + userPid: ps.Pid, + logger: d.logger, + killTimeout: d.DriverContext.KillTimeout(task), + doneCh: make(chan struct{}), + waitCh: make(chan *cstructs.WaitResult, 1), } go h.run() return h, nil @@ -246,20 +258,30 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error return nil, fmt.Errorf("failed to parse Rkt handle '%s': %v", handleID, err) } - // Find the process - proc, err := os.FindProcess(qpid.Pid) - if proc == nil || err != nil { - return nil, fmt.Errorf("failed to find Rkt PID %d: %v", qpid.Pid, err) + pluginConfig := &plugin.ClientConfig{ + Reattach: qpid.PluginConfig.PluginConfig(), + } + executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + if err != nil { + d.logger.Println("[ERROR] driver.rkt: error connecting to plugin so destroying plugin pid and user pid") + if e := destroyPlugin(qpid.PluginConfig.Pid, qpid.UserPid); e != nil { + d.logger.Printf("[ERROR] driver.rkt: error destroying plugin and userpid: %v", e) + } + return nil, fmt.Errorf("error connecting to plugin: %v", err) } // Return a driver handle h := &rktHandle{ - proc: proc, - image: qpid.Image, - logger: d.logger, - killTimeout: qpid.KillTimeout, - doneCh: make(chan struct{}), - waitCh: make(chan *cstructs.WaitResult, 1), + pluginClient: pluginClient, + userPid: qpid.UserPid, + taskDir: qpid.TaskDir, + allocDir: qpid.AllocDir, + executor: executor, + image: qpid.Image, + logger: d.logger, + killTimeout: qpid.KillTimeout, + doneCh: make(chan struct{}), + waitCh: make(chan *cstructs.WaitResult, 1), } go h.run() @@ -269,9 +291,12 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error func (h *rktHandle) ID() string { // Return a handle to the PID pid := &rktPID{ - Pid: h.proc.Pid, - Image: h.image, - KillTimeout: h.killTimeout, + PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), + Image: h.image, + KillTimeout: h.killTimeout, + UserPid: h.userPid, + TaskDir: h.taskDir, + AllocDir: h.allocDir, } data, err := json.Marshal(pid) if err != nil { @@ -287,6 +312,7 @@ func (h *rktHandle) WaitCh() chan *cstructs.WaitResult { func (h *rktHandle) Update(task *structs.Task) error { // Store the updated kill timeout. h.killTimeout = task.KillTimeout + h.executor.UpdateLogConfig(task.LogConfig) // Update is not possible return nil @@ -295,23 +321,19 @@ func (h *rktHandle) Update(task *structs.Task) error { // Kill is used to terminate the task. We send an Interrupt // and then provide a 5 second grace period before doing a Kill. func (h *rktHandle) Kill() error { - h.proc.Signal(os.Interrupt) + h.executor.ShutDown() select { case <-h.doneCh: return nil case <-time.After(h.killTimeout): - return h.proc.Kill() + return h.executor.Exit() } } func (h *rktHandle) run() { - ps, err := h.proc.Wait() + ps, err := h.executor.Wait() close(h.doneCh) - code := 0 - if !ps.Success() { - // TODO: Better exit code parsing. - code = 1 - } - h.waitCh <- &cstructs.WaitResult{ExitCode: code, Signal: 0, Err: err} + h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err} close(h.waitCh) + h.pluginClient.Kill() } diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index bf883833553..52f48f56033 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -3,7 +3,6 @@ package driver import ( "fmt" "io/ioutil" - "os" "path/filepath" "testing" "time" @@ -13,7 +12,6 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" - cstructs "github.com/hashicorp/nomad/client/driver/structs" ctestutils "github.com/hashicorp/nomad/client/testutil" ) @@ -32,22 +30,6 @@ func TestRktVersionRegex(t *testing.T) { } } -func TestRktDriver_Handle(t *testing.T) { - h := &rktHandle{ - proc: &os.Process{Pid: 123}, - image: "foo", - killTimeout: 5 * time.Nanosecond, - doneCh: make(chan struct{}), - waitCh: make(chan *cstructs.WaitResult, 1), - } - - actual := h.ID() - expected := `Rkt:{"Pid":123,"Image":"foo","KillTimeout":5}` - if actual != expected { - t.Errorf("Expected `%s`, found `%s`", expected, actual) - } -} - // The fingerprinter test should always pass, even if rkt is not installed. func TestRktDriver_Fingerprint(t *testing.T) { ctestutils.RktCompatible(t) @@ -84,6 +66,10 @@ func TestRktDriver_Start(t *testing.T) { "image": "coreos.com/etcd:v2.0.4", "command": "/etcd", }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, @@ -124,6 +110,10 @@ func TestRktDriver_Start_Wait(t *testing.T) { "command": "/etcd", "args": []string{"--version"}, }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, @@ -168,6 +158,10 @@ func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) { "command": "/etcd", "args": []string{"--version"}, }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, @@ -213,6 +207,10 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) { "command": "/etcd", "args": []string{"--version"}, }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, From 65524d90e5438dfaa489ff6c10f59349b7ec4880 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 16 Feb 2016 16:37:50 -0800 Subject: [PATCH 12/20] Do not hard-code binary location --- client/driver/rkt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index f30608b53d5..b54c77e21ed 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -229,7 +229,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e LogConfig: task.LogConfig, } - ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: "/usr/bin/rkt", Args: cmdArgs}, executorCtx) + ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: "rkt", Args: cmdArgs}, executorCtx) if err != nil { pluginClient.Kill() return nil, fmt.Errorf("error starting process via the plugin: %v", err) From 0f4472d94fdef4f6900ea14c4b79bb4525831da3 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 16 Feb 2016 18:47:53 -0800 Subject: [PATCH 13/20] Change name of the stdout file --- client/driver/rkt_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index 52f48f56033..2f2b55c5a2e 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -243,7 +243,7 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) { if !ok { t.Fatalf("Could not find task directory for task: %v", task) } - stdout := filepath.Join(taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", task.Name)) + stdout := filepath.Join(taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout.0", task.Name)) data, err := ioutil.ReadFile(stdout) if err != nil { t.Fatalf("Failed to read tasks stdout: %v", err) From 45b867082965e2ddae3a553ee672cebc6a26e222 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 16 Feb 2016 18:57:12 -0800 Subject: [PATCH 14/20] Disable ipmasq in the default ptp network --- scripts/install_rkt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/install_rkt.sh b/scripts/install_rkt.sh index ea00a4e83c6..d3987e480c7 100755 --- a/scripts/install_rkt.sh +++ b/scripts/install_rkt.sh @@ -5,6 +5,9 @@ set -ex RKT_VERSION="v1.0.0" DEST_DIR="/usr/local/bin" +sudo mkdir -p /etc/rkt/net.d +echo '{"name": "default", "type": "ptp", "ipMasq": false, "ipam": { "type": "host-local", "subnet": "172.16.28.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } }' | sudo tee -a /etc/rkt/net.d/99-network.conf + wget https://github.com/coreos/rkt/releases/download/$RKT_VERSION/rkt-$RKT_VERSION.tar.gz tar xzvf rkt-$RKT_VERSION.tar.gz sudo cp rkt-$RKT_VERSION/rkt $DEST_DIR From aa288c91d0768298aa6cf89f5436e4e0fb77d0df Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Fri, 19 Feb 2016 00:46:54 +0000 Subject: [PATCH 15/20] A number of cleanups - Removed some unused variables. --- client/driver/rkt.go | 23 +++++++---------------- client/driver/rkt_test.go | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index b54c77e21ed..918dd1db6f8 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -56,11 +56,9 @@ type RktDriverConfig struct { // rktHandle is returned from Start/Open as a handle to the PID type rktHandle struct { pluginClient *plugin.Client - userPid int + executorPid int executor executor.Executor - taskDir string allocDir *allocdir.AllocDir - image string logger *log.Logger killTimeout time.Duration waitCh chan *cstructs.WaitResult @@ -71,10 +69,8 @@ type rktHandle struct { // disk type rktPID struct { PluginConfig *PluginReattachConfig - TaskDir string AllocDir *allocdir.AllocDir - UserPid int - Image string + ExecutorPid int KillTimeout time.Duration } @@ -239,8 +235,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e h := &rktHandle{ pluginClient: pluginClient, executor: exec, - image: img, - userPid: ps.Pid, + executorPid: ps.Pid, logger: d.logger, killTimeout: d.DriverContext.KillTimeout(task), doneCh: make(chan struct{}), @@ -264,8 +259,8 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) if err != nil { d.logger.Println("[ERROR] driver.rkt: error connecting to plugin so destroying plugin pid and user pid") - if e := destroyPlugin(qpid.PluginConfig.Pid, qpid.UserPid); e != nil { - d.logger.Printf("[ERROR] driver.rkt: error destroying plugin and userpid: %v", e) + if e := destroyPlugin(qpid.PluginConfig.Pid, qpid.ExecutorPid); e != nil { + d.logger.Printf("[ERROR] driver.rkt: error destroying plugin and executor pid: %v", e) } return nil, fmt.Errorf("error connecting to plugin: %v", err) } @@ -273,11 +268,9 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error // Return a driver handle h := &rktHandle{ pluginClient: pluginClient, - userPid: qpid.UserPid, - taskDir: qpid.TaskDir, + executorPid: qpid.ExecutorPid, allocDir: qpid.AllocDir, executor: executor, - image: qpid.Image, logger: d.logger, killTimeout: qpid.KillTimeout, doneCh: make(chan struct{}), @@ -292,10 +285,8 @@ func (h *rktHandle) ID() string { // Return a handle to the PID pid := &rktPID{ PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), - Image: h.image, KillTimeout: h.killTimeout, - UserPid: h.userPid, - TaskDir: h.taskDir, + ExecutorPid: h.executorPid, AllocDir: h.allocDir, } data, err := json.Marshal(pid) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index 2f2b55c5a2e..39c36af3a98 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -66,10 +66,10 @@ func TestRktDriver_Start(t *testing.T) { "image": "coreos.com/etcd:v2.0.4", "command": "/etcd", }, - LogConfig: &structs.LogConfig{ - MaxFiles: 10, - MaxFileSizeMB: 10, - }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, @@ -110,10 +110,10 @@ func TestRktDriver_Start_Wait(t *testing.T) { "command": "/etcd", "args": []string{"--version"}, }, - LogConfig: &structs.LogConfig{ - MaxFiles: 10, - MaxFileSizeMB: 10, - }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, @@ -158,10 +158,10 @@ func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) { "command": "/etcd", "args": []string{"--version"}, }, - LogConfig: &structs.LogConfig{ - MaxFiles: 10, - MaxFileSizeMB: 10, - }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, @@ -207,10 +207,10 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) { "command": "/etcd", "args": []string{"--version"}, }, - LogConfig: &structs.LogConfig{ - MaxFiles: 10, - MaxFileSizeMB: 10, - }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, Resources: &structs.Resources{ MemoryMB: 128, CPU: 100, From 5419603bb4014340188bb41edebecec3aac8dbbe Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Thu, 18 Feb 2016 18:57:13 -0800 Subject: [PATCH 16/20] Cleanup if the plugin executor crashes. --- client/driver/rkt.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 918dd1db6f8..36f3743e859 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -324,6 +324,14 @@ func (h *rktHandle) Kill() error { func (h *rktHandle) run() { ps, err := h.executor.Wait() close(h.doneCh) + if ps.ExitCode == 0 && err != nil { + if e := killProcess(h.executorPid); e != nil { + h.logger.Printf("[ERROR] driver.rkt: error killing user process: %v", e) + } + if e := h.allocDir.UnmountAll(); e != nil { + h.logger.Printf("[ERROR] driver.rkt: unmounting dev,proc and alloc dirs failed: %v", e) + } + } h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err} close(h.waitCh) h.pluginClient.Kill() From b92546a9c5e4c877092169d9960bd6700a2c6d06 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Sat, 20 Feb 2016 18:04:34 -0800 Subject: [PATCH 17/20] Run rkt interactively --- client/driver/rkt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 36f3743e859..fbf14c865e5 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -157,7 +157,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) } - cmdArgs = append(cmdArgs, "run") + cmdArgs = append(cmdArgs, "run", "--interactive") cmdArgs = append(cmdArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", task.Name, local)) cmdArgs = append(cmdArgs, fmt.Sprintf("--mount=volume=%s,target=%s", task.Name, ctx.AllocDir.SharedDir)) cmdArgs = append(cmdArgs, img) From 54b97273175ad50020138b463ab4197bfd1c0fc4 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Sun, 21 Feb 2016 19:16:57 -0800 Subject: [PATCH 18/20] Set allocDir in handle --- client/driver/rkt.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index fbf14c865e5..fe4444c0e40 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -236,6 +236,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e pluginClient: pluginClient, executor: exec, executorPid: ps.Pid, + allocDir: ctx.AllocDir, logger: d.logger, killTimeout: d.DriverContext.KillTimeout(task), doneCh: make(chan struct{}), From 253f32e541848928d4b6b72b7a30b2a595f254fe Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Tue, 1 Mar 2016 09:30:53 -0800 Subject: [PATCH 19/20] Use NewWaitResult in the wait channel --- client/driver/rkt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index fe4444c0e40..08c154c9721 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -333,7 +333,7 @@ func (h *rktHandle) run() { h.logger.Printf("[ERROR] driver.rkt: unmounting dev,proc and alloc dirs failed: %v", e) } } - h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err} + h.waitCh <- cstructs.NewWaitResult(ps.ExitCode, 0, err) close(h.waitCh) h.pluginClient.Kill() } From d8d62aaddf7c3e94e0c72e3a86981a6e532aa9e9 Mon Sep 17 00:00:00 2001 From: Abhishek Chanda Date: Wed, 2 Mar 2016 11:28:50 -0800 Subject: [PATCH 20/20] Fix location of volume mount Also add a test for mounting alloc dirs --- client/driver/rkt.go | 7 +------ client/driver/rkt_test.go | 42 +++++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 08c154c9721..574fc74f202 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -152,13 +152,8 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e insecure = true } - local, ok := ctx.AllocDir.TaskDirs[task.Name] - if !ok { - return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) - } - cmdArgs = append(cmdArgs, "run", "--interactive") - cmdArgs = append(cmdArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", task.Name, local)) + cmdArgs = append(cmdArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", task.Name, ctx.AllocDir.SharedDir)) cmdArgs = append(cmdArgs, fmt.Sprintf("--mount=volume=%s,target=%s", task.Name, ctx.AllocDir.SharedDir)) cmdArgs = append(cmdArgs, img) if insecure == true { diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index 39c36af3a98..c2bbd600cd8 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -4,11 +4,12 @@ import ( "fmt" "io/ioutil" "path/filepath" + "reflect" "testing" "time" - "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/env" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" @@ -197,15 +198,21 @@ func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) { } } -func TestRktDriver_Start_Wait_Logs(t *testing.T) { +func TestRktDriver_Start_Wait_AllocDir(t *testing.T) { ctestutils.RktCompatible(t) + + exp := []byte{'w', 'i', 'n'} + file := "output.txt" + task := &structs.Task{ - Name: "etcd", + Name: "alpine", Config: map[string]interface{}{ - "trust_prefix": "coreos.com/etcd", - "image": "coreos.com/etcd:v2.0.4", - "command": "/etcd", - "args": []string{"--version"}, + "image": "docker://alpine", + "command": "/bin/sh", + "args": []string{ + "-c", + fmt.Sprintf(`echo -n %s > ${%s}/%s`, string(exp), env.AllocDir, file), + }, }, LogConfig: &structs.LogConfig{ MaxFiles: 10, @@ -239,17 +246,14 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) { t.Fatalf("timeout") } - taskDir, ok := execCtx.AllocDir.TaskDirs[task.Name] - if !ok { - t.Fatalf("Could not find task directory for task: %v", task) - } - stdout := filepath.Join(taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout.0", task.Name)) - data, err := ioutil.ReadFile(stdout) - if err != nil { - t.Fatalf("Failed to read tasks stdout: %v", err) - } + // Check that data was written to the shared alloc directory. + outputFile := filepath.Join(execCtx.AllocDir.SharedDir, file) + act, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatalf("Couldn't read expected output: %v", err) + } - if len(data) == 0 { - t.Fatal("Task's stdout is empty: %q", stdout) - } + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Command output is %v; expected %v", act, exp) + } }