Skip to content

Commit

Permalink
Added version flag in sandbox (#137)
Browse files Browse the repository at this point in the history 
* Added version flag in the sandbox

Signed-off-by: Yuvraj <[email protected]>
  • Loading branch information
@yindia
yindia authored Jul 12, 2021
1 parent 41e0a40 commit 03be16b
Show file tree
Hide file tree
Showing 9 changed files with 261 additions and 75 deletions.
1 change: 1 addition & 0 deletions flytectl/cmd/config/subcommand/sandbox/config_flags.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions flytectl/cmd/config/subcommand/sandbox/config_flags_test.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion flytectl/cmd/config/subcommand/sandbox/sandbox_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,6 @@ var (

//Config
type Config struct {
Source string `json:"source" pflag:", Path of your source code"`
Source string `json:"source" pflag:",Path of your source code"`
Version string `json:"version" pflag:",Version of flyte"`
}
76 changes: 63 additions & 13 deletions flytectl/cmd/sandbox/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import (
"os"
"path/filepath"

f "github.com/flyteorg/flytectl/pkg/filesystemutils"
"github.com/flyteorg/flytectl/pkg/util"

"github.com/flyteorg/flytectl/pkg/docker"

"github.com/docker/docker/api/types/mount"
Expand All @@ -30,9 +33,22 @@ Mount your source code repository inside sandbox
::
bin/flytectl sandbox start --source=$HOME/flyteorg/flytesnacks
Run specific version of flyte, Only available after v0.14.0+
::
bin/flytectl sandbox start --version=v0.14.0
Usage
`
GeneratedManifest = "/flyteorg/share/flyte_generated.yaml"
FlyteReleaseURL = "/flyteorg/flyte/releases/download/%v/flyte_sandbox_manifest.yaml"
FlyteMinimumVersionSupported = "v0.14.0"
GithubURL = "https://github.com"
)

var (
FlyteManifest = f.FilePathJoin(f.UserHomeDir(), ".flyte", "flyte_generated.yaml")
)

type ExecResult struct {
Expand All @@ -57,6 +73,10 @@ func startSandboxCluster(ctx context.Context, args []string, cmdCtx cmdCore.Comm

func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bufio.Scanner, error) {
fmt.Printf("%v Bootstrapping a brand new flyte cluster... %v %v\n", emoji.FactoryWorker, emoji.Hammer, emoji.Wrench)
if err := docker.RemoveSandbox(ctx, cli, reader); err != nil {
return nil, err
}

if err := docker.SetupFlyteDir(); err != nil {
return nil, err
}
Expand All @@ -65,25 +85,15 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu
return nil, err
}

if err := docker.RemoveSandbox(ctx, cli, reader); err != nil {
if err := mountSourceCode(); err != nil {
return nil, err
}

if len(sandboxConfig.DefaultConfig.Source) > 0 {
source, err := filepath.Abs(sandboxConfig.DefaultConfig.Source)
if err != nil {
return nil, err
}
docker.Volumes = append(docker.Volumes, mount.Mount{
Type: mount.TypeBind,
Source: source,
Target: docker.Source,
})
if err := mountFlyteManifest(sandboxConfig.DefaultConfig.Version); err != nil {
return nil, err
}

fmt.Printf("%v pulling docker image %s\n", emoji.Whale, docker.ImageName)
os.Setenv("KUBECONFIG", docker.Kubeconfig)
os.Setenv("FLYTECTL_CONFIG", docker.FlytectlConfig)
if err := docker.PullDockerImage(ctx, cli, docker.ImageName); err != nil {
return nil, err
}
Expand Down Expand Up @@ -111,3 +121,43 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu

return logReader, nil
}

func mountSourceCode() error {
if len(sandboxConfig.DefaultConfig.Source) > 0 {
source, err := filepath.Abs(sandboxConfig.DefaultConfig.Source)
if err != nil {
return err
}
docker.Volumes = append(docker.Volumes, mount.Mount{
Type: mount.TypeBind,
Source: source,
Target: docker.Source,
})
}
return nil
}

func mountFlyteManifest(version string) error {
if len(version) > 0 {
isGreater, err := util.IsVersionGreaterThan(version, FlyteMinimumVersionSupported)
if err != nil {
return err
}
if !isGreater {
return fmt.Errorf("version flag only support %s+ flyte version", FlyteMinimumVersionSupported)
}
response, err := util.GetRequest(GithubURL, fmt.Sprintf(FlyteReleaseURL, version))
if err != nil {
return err
}
if err := util.WriteIntoFile(response, FlyteManifest); err != nil {
return err
}
docker.Volumes = append(docker.Volumes, mount.Mount{
Type: mount.TypeBind,
Source: FlyteManifest,
Target: GeneratedManifest,
})
}
return nil
}
108 changes: 95 additions & 13 deletions flytectl/cmd/sandbox/start_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ func TestStartSandboxFunc(t *testing.T) {
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Source = f.UserHomeDir()
volumes := append(docker.Volumes, mount.Mount{
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: sandboxConfig.DefaultConfig.Source,
Target: docker.Source,
Expand Down Expand Up @@ -101,7 +102,8 @@ func TestStartSandboxFunc(t *testing.T) {
sandboxConfig.DefaultConfig.Source = "../"
absPath, err := filepath.Abs(sandboxConfig.DefaultConfig.Source)
assert.Nil(t, err)
volumes := append(docker.Volumes, mount.Mount{
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: absPath,
Target: docker.Source,
Expand Down Expand Up @@ -131,13 +133,90 @@ func TestStartSandboxFunc(t *testing.T) {
_, err = startSandbox(ctx, mockDocker, os.Stdin)
assert.Nil(t, err)
})
t.Run("Successfully run sandbox cluster with specific version", func(t *testing.T) {
ctx := context.Background()
errCh := make(chan error)
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Version = "v0.15.0"
sandboxConfig.DefaultConfig.Source = ""
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: FlyteManifest,
Target: GeneratedManifest,
})
mockDocker.OnContainerCreate(ctx, &container.Config{
Env: docker.Environment,
Image: docker.ImageName,
Tty: false,
ExposedPorts: p1,
}, &container.HostConfig{
Mounts: volumes,
PortBindings: p2,
Privileged: true,
}, nil, nil, mock.Anything).Return(container.ContainerCreateCreatedBody{
ID: "Hello",
}, nil)
mockDocker.OnContainerStart(ctx, "Hello", types.ContainerStartOptions{}).Return(nil)
mockDocker.OnContainerList(ctx, types.ContainerListOptions{All: true}).Return([]types.Container{}, nil)
mockDocker.OnImagePullMatch(ctx, mock.Anything, types.ImagePullOptions{}).Return(os.Stdin, nil)
mockDocker.OnContainerLogsMatch(ctx, mock.Anything, types.ContainerLogsOptions{
ShowStderr: true,
ShowStdout: true,
Timestamps: true,
Follow: true,
}).Return(nil, nil)
mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh)
_, err := startSandbox(ctx, mockDocker, os.Stdin)
assert.Nil(t, err)
})
t.Run("Failed run sandbox cluster with wrong version", func(t *testing.T) {
ctx := context.Background()
errCh := make(chan error)
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Version = "v0.13.0"
sandboxConfig.DefaultConfig.Source = ""
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: FlyteManifest,
Target: GeneratedManifest,
})
mockDocker.OnContainerCreate(ctx, &container.Config{
Env: docker.Environment,
Image: docker.ImageName,
Tty: false,
ExposedPorts: p1,
}, &container.HostConfig{
Mounts: volumes,
PortBindings: p2,
Privileged: true,
}, nil, nil, mock.Anything).Return(container.ContainerCreateCreatedBody{
ID: "Hello",
}, nil)
mockDocker.OnContainerStart(ctx, "Hello", types.ContainerStartOptions{}).Return(nil)
mockDocker.OnContainerList(ctx, types.ContainerListOptions{All: true}).Return([]types.Container{}, nil)
mockDocker.OnImagePullMatch(ctx, mock.Anything, types.ImagePullOptions{}).Return(os.Stdin, nil)
mockDocker.OnContainerLogsMatch(ctx, mock.Anything, types.ContainerLogsOptions{
ShowStderr: true,
ShowStdout: true,
Timestamps: true,
Follow: true,
}).Return(nil, nil)
mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh)
_, err := startSandbox(ctx, mockDocker, os.Stdin)
assert.NotNil(t, err)
})
t.Run("Error in pulling image", func(t *testing.T) {
ctx := context.Background()
errCh := make(chan error)
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Source = f.UserHomeDir()
volumes := append(docker.Volumes, mount.Mount{
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: sandboxConfig.DefaultConfig.Source,
Target: docker.Source,
Expand Down Expand Up @@ -173,7 +252,8 @@ func TestStartSandboxFunc(t *testing.T) {
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Source = f.UserHomeDir()
volumes := append(docker.Volumes, mount.Mount{
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: sandboxConfig.DefaultConfig.Source,
Target: docker.Source,
Expand Down Expand Up @@ -216,19 +296,15 @@ func TestStartSandboxFunc(t *testing.T) {
errCh := make(chan error)
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Source = f.UserHomeDir()
volumes := append(docker.Volumes, mount.Mount{
Type: mount.TypeBind,
Source: sandboxConfig.DefaultConfig.Source,
Target: docker.Source,
})
sandboxConfig.DefaultConfig.Source = ""
sandboxConfig.DefaultConfig.Version = ""
mockDocker.OnContainerCreate(ctx, &container.Config{
Env: docker.Environment,
Image: docker.ImageName,
Tty: false,
ExposedPorts: p1,
}, &container.HostConfig{
Mounts: volumes,
Mounts: docker.Volumes,
PortBindings: p2,
Privileged: true,
}, nil, nil, mock.Anything).Return(container.ContainerCreateCreatedBody{
Expand All @@ -247,13 +323,18 @@ func TestStartSandboxFunc(t *testing.T) {
_, err := startSandbox(ctx, mockDocker, os.Stdin)
assert.NotNil(t, err)
})
t.Run("Failed manifest", func(t *testing.T) {
err := mountFlyteManifest("v100.9.9")
assert.NotNil(t, err)
})
t.Run("Error in reading logs", func(t *testing.T) {
ctx := context.Background()
errCh := make(chan error)
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Source = f.UserHomeDir()
volumes := append(docker.Volumes, mount.Mount{
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: sandboxConfig.DefaultConfig.Source,
Target: docker.Source,
Expand Down Expand Up @@ -289,7 +370,8 @@ func TestStartSandboxFunc(t *testing.T) {
bodyStatus := make(chan container.ContainerWaitOKBody)
mockDocker := &mocks.Docker{}
sandboxConfig.DefaultConfig.Source = f.UserHomeDir()
volumes := append(docker.Volumes, mount.Mount{
volumes := docker.Volumes
volumes = append(volumes, mount.Mount{
Type: mount.TypeBind,
Source: sandboxConfig.DefaultConfig.Source,
Target: docker.Source,
Expand Down
Loading

0 comments on commit 03be16b

Please sign in to comment.