SageMaker on Flyte: installing SageMaker plugins [Alpha] (flyteorg#163)

* install the sagemaker plugin

* upgrade flyteplugin

* upgrade dependency on flyteplugins

* upgrade flyteplugins

* upgrade flyteplugins

* go mod

* go mod

* go mod

* add to sandbox config and pull latest flyteidl and flyteplugins

* flyteplugins v0.4.0

flytepropeller/cmd/controller/main.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/lyft/flyteplugins/go/tasks/plugins/hive"
 	_ "github.com/lyft/flyteplugins/go/tasks/plugins/k8s/container"
 	_ "github.com/lyft/flyteplugins/go/tasks/plugins/k8s/kfoperators/pytorch"
+	_ "github.com/lyft/flyteplugins/go/tasks/plugins/k8s/sagemaker"
 	_ "github.com/lyft/flyteplugins/go/tasks/plugins/k8s/sidecar"
 	_ "github.com/lyft/flyteplugins/go/tasks/plugins/k8s/spark"
 	"github.com/lyft/flytestdlib/contextutils"

flytepropeller/config.yaml

@@ -32,6 +32,8 @@ tasks:
       - container
       - K8S-ARRAY
       - qubole-hive-executor
+      - sagemaker_training
+      - sagemaker_hyperparameter_tuning
 # Sample plugins config
 plugins:
   # All k8s plugins default configuration
@@ -55,6 +57,18 @@ plugins:
       name: "flyte-copilot-"
       image: "flyteplugins:24c62d97452ce83ad6b4fd24e0eea2b4c44ff0c6"
       start-timeout: "5s"
+  sagemaker:
+    roleArn: "arn:aws:iam::123456789012:role/test-development"
+    region: "us-east-1"
+    prebuiltAlgorithms:
+      - name: xgboost
+        regionalConfigs:
+          - region: "us-east-1"
+            versionConfigs:
+              - version: 0.72
+                image: "811284229777.dkr.ecr.us-east-1.amazonaws.com/xgboost:latest"
+              - version: 0.90
+                image: "683313688378.dkr.ecr.us-east-1.amazonaws.com/xgboost:latest"
   # Logging configuration
   logs:
     kubernetes-enabled: true

flytepropeller/go.mod

@@ -23,8 +23,8 @@ require (
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/jmespath/go-jmespath v0.3.0 // indirect
 	github.com/lyft/datacatalog v0.2.1
-	github.com/lyft/flyteidl v0.17.38
-	github.com/lyft/flyteplugins v0.3.38
+	github.com/lyft/flyteidl v0.18.0
+	github.com/lyft/flyteplugins v0.4.0
 	github.com/lyft/flytestdlib v0.3.9
 	github.com/magiconair/properties v1.8.1
 	github.com/mattn/go-colorable v0.1.6 // indirect
@@ -36,15 +36,14 @@ require (
 	github.com/spf13/cobra v0.0.6
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
-	golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect
 	google.golang.org/genproto v0.0.0-20200312145019-da6875a35672 // indirect
 	google.golang.org/grpc v1.28.0
 	gopkg.in/ini.v1 v1.54.0 // indirect
 	k8s.io/api v0.17.3
 	k8s.io/apimachinery v0.17.3
-	k8s.io/client-go v11.0.0+incompatible
+	k8s.io/client-go v11.0.1-0.20190918222721-c0e3722d5cf0+incompatible
 	k8s.io/klog v1.0.0
 	k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe // indirect
 	k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect