Supporting generic types through flytectl (flyteorg#78)

* Supporting generic types through flytectl Signed-off-by: Prafulla Mahindrakar <[email protected]>
eapolinario · Jun 7, 2021 · 8287cbf · 8287cbf
1 parent e92ad2e
commit 8287cbf
Show file tree

Hide file tree

Showing 11 changed files with 132 additions and 22 deletions.
diff --git a/cmd/create/execution.go b/cmd/create/execution.go
@@ -78,6 +78,51 @@ Also an execution can be relaunched by passing in current execution id.
 
  flytectl create execution --relaunch ffb31066a0f8b4d52b77 -p flytectldemo -d development
 
+Generic data types are also supported for execution in similar way.Following is sample of how the inputs need to be specified while creating the execution.
+As usual the spec file should be generated first and then run the execution using the spec file.
+
+::
+
+ flytectl get task -d development -p flytectldemo  core.type_system.custom_objects.add --execFile adddatanum.yaml
+
+The generated file would look similar to this. Here you can see empty values dumped for generic data type x and y. 
+
+::
+
+    iamRoleARN: ""
+    inputs:
+      "x": {}
+      "y": {}
+    kubeServiceAcct: ""
+    targetDomain: ""
+    targetProject: ""
+    task: core.type_system.custom_objects.add
+    version: v3
+
+Modified file with struct data populated for x and y parameters for the task core.type_system.custom_objects.add
+
+::
+
+  iamRoleARN: "arn:aws:iam::123456789:role/dummy"
+  inputs:
+    "x":
+      "x": 2
+      "y": ydatafory
+      "z":
+        1 : "foo"
+        2 : "bar"
+    "y":
+      "x": 3
+      "y": ydataforx
+      "z":
+        3 : "buzz"
+        4 : "lightyear"
+  kubeServiceAcct: ""
+  targetDomain: ""
+  targetProject: ""
+  task: core.type_system.custom_objects.add
+  version: v3
+
 Usage
 `
 )

diff --git a/cmd/create/execution_test.go b/cmd/create/execution_test.go
@@ -144,7 +144,7 @@ func TestCreateTaskExecutionFunc(t *testing.T) {
 		},
 	}
 	mockClient.OnCreateExecutionMatch(ctx, mock.Anything).Return(executionCreateResponseTask, nil)
-	executionConfig.ExecFile = testDataFolder + "task_execution_spec.yaml"
+	executionConfig.ExecFile = testDataFolder + "task_execution_spec_with_iamrole.yaml"
 	err = createExecutionCommand(ctx, args, cmdCtx)
 	assert.Nil(t, err)
 	mockClient.AssertCalled(t, "CreateExecution", ctx, mock.Anything)

diff --git a/cmd/create/execution_util.go b/cmd/create/execution_util.go
@@ -57,11 +57,10 @@ func createExecutionRequestForTask(ctx context.Context, taskName string, project
 		Literals: variableLiterals,
 	}
 	var authRole *admin.AuthRole
-	if executionConfig.KubeServiceAcct != "" || executionConfig.IamRoleARN != "" {
-		authRole = &admin.AuthRole{
-			AssumableIamRole:         executionConfig.IamRoleARN,
-			KubernetesServiceAccount: executionConfig.KubeServiceAcct,
-		}
+	if executionConfig.KubeServiceAcct != "" {
+		authRole = &admin.AuthRole{KubernetesServiceAccount: executionConfig.KubeServiceAcct}
+	} else {
+		authRole = &admin.AuthRole{AssumableIamRole: executionConfig.IamRoleARN}
 	}
 	ID := &core.Identifier{
 		ResourceType: core.ResourceType_TASK,

diff --git a/cmd/testdata/task_execution_spec_with_iamrole.yaml b/cmd/testdata/task_execution_spec_with_iamrole.yaml
@@ -0,0 +1,15 @@
+iamRoleARN: "iamRoleARN"
+inputs:
+  sorted_list1:
+  - 0
+  - 2
+  - 4
+  sorted_list2:
+  - 1
+  - 3
+  - 5
+kubeServiceAcct: ""
+targetDomain: "development"
+targetProject: "flytesnacks"
+task: core.advanced.run_merge_sort.merge
+version: v2
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -99,17 +99,17 @@
         "color-brand-primary": "#9D68E4",
         "color-brand-content": "#9D68E4",
     },
+}
+
+html_context = {
+    "home_page": "https://docs.flyte.org",
     # custom flyteorg furo theme options
     "github_repo": "flytectl",
     "github_username": "flyteorg",
     "github_commit": "master",
     "docs_path": "docs/source",  # path to documentation source
 }
 
-html_context = {
-    "home_page": "https://docs.flyte.org",
-}
-
 # The default sidebars (for documents that don't match any pattern) are
 # defined by theme itself.  Builtin themes are using these templates by
 # default: ``['localtoc.html', 'relations.html', 'sourcelink.html',

diff --git a/docs/source/gen/flytectl_create_execution.rst b/docs/source/gen/flytectl_create_execution.rst
@@ -76,6 +76,51 @@ Also an execution can be relaunched by passing in current execution id.
 
  flytectl create execution --relaunch ffb31066a0f8b4d52b77 -p flytectldemo -d development
 
+Generic data types are also supported for execution in similar way.Following is sample of how the inputs need to be specified while creating the execution.
+As usual the spec file should be generated first and then run the execution using the spec file.
+
+::
+
+ flytectl get task -d development -p flytectldemo  core.type_system.custom_objects.add --execFile adddatanum.yaml
+
+The generated file would look similar to this. Here you can see empty values dumped for generic data type x and y. 
+
+::
+
+    iamRoleARN: ""
+    inputs:
+      "x": {}
+      "y": {}
+    kubeServiceAcct: ""
+    targetDomain: ""
+    targetProject: ""
+    task: core.type_system.custom_objects.add
+    version: v3
+
+Modified file with struct data populated for x and y parameters for the task core.type_system.custom_objects.add
+
+::
+
+  iamRoleARN: "arn:aws:iam::123456789:role/dummy"
+  inputs:
+    "x":
+      "x": 2
+      "y": ydatafory
+      "z":
+        1 : "foo"
+        2 : "bar"
+    "y":
+      "x": 3
+      "y": ydataforx
+      "z":
+        3 : "buzz"
+        4 : "lightyear"
+  kubeServiceAcct: ""
+  targetDomain: ""
+  targetProject: ""
+  task: core.type_system.custom_objects.add
+  version: v3
+
 Usage
 
 

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -18,13 +18,7 @@ golang. To install simply copy paste the following into the command-line:
 Configure
 =========
 Flytectl allows configuring using a YAML file or pass every configuration value
-on command-line. The follow configuration is useful to setup.
-Place this in $HOME/.flyte directory with name config.yaml.
-This file is searched in
-* $HOME/.flyte
-* currDir from where you run flytectl
-* /etc/flyte/config
-* You can pass it commandline using --config <config-file-path>
+on command-line. The following configuration is useful to setup.
 
 Basic Configuration
 --------------------
@@ -34,6 +28,7 @@ Basic Configuration
   admin:
     # For GRPC endpoints you might want to use dns:///flyte.myexample.com
     endpoint: dns:///flyte.lyft.net
+    authType: Pkce
     # Change insecure flag to ensure that you use the right setting for your environment
     insecure: true
   logger:
@@ -42,6 +37,15 @@ Basic Configuration
     level: 1
 
 
+Place this in $HOME/.flyte directory with name config.yaml.
+This file is searched in
+
+- $HOME/.flyte
+- currDir from where you run flytectl
+- /etc/flyte/config
+- You can pass it commandline using --config <config-file-path>
+
+
 .. toctree::
    :maxdepth: 1
    :hidden:

diff --git a/docs/source/nouns.rst b/docs/source/nouns.rst
@@ -1,10 +1,10 @@
 Nouns
 ------
+Flytectl noun specify the resource on which the action needs to be performed eg: resources like project/workflow/task/execution
 
 .. toctree::
     :maxdepth: 1
     :caption: Nouns
-    :hidden:
 
     gen/flytectl_create_project
     gen/flytectl_create_execution

diff --git a/docs/source/verbs.rst b/docs/source/verbs.rst
@@ -1,14 +1,14 @@
 Verbs
 ------
+Flytectl verbs specify the actions to be performed on the resources like create/get/update/delete
 
 .. toctree::
     :maxdepth: 1
     :caption: Verbs
-    :hidden:
 
     gen/flytectl_create
     gen/flytectl_get
     gen/flytectl_update
     gen/flytectl_delete
     gen/flytectl_register
-    gen/flytectl_config
+    gen/flytectl_config
diff --git a/go.mod b/go.mod
@@ -28,3 +28,5 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 	sigs.k8s.io/yaml v1.2.0
 )
+
+replace github.com/flyteorg/flyteidl => github.com/flyteorg/flyteidl v0.18.51-0.20210602050605-9ebebd25056e
diff --git a/go.sum b/go.sum
@@ -172,8 +172,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
-github.com/flyteorg/flyteidl v0.18.51 h1:fp4Qq9WtO3QeDbFpKk0uQqB/cF78+Ul7D3DhvQPlV5Q=
-github.com/flyteorg/flyteidl v0.18.51/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
+github.com/flyteorg/flyteidl v0.18.51-0.20210602050605-9ebebd25056e h1:J3Uaju9mBrJhrU4kiv3ozpZNkDdjDepkn0X6ueB8iag=
+github.com/flyteorg/flyteidl v0.18.51-0.20210602050605-9ebebd25056e/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
 github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220=
 github.com/flyteorg/flytestdlib v0.3.21 h1:AF+y6wI64DNfoi4WtZU/v18Cfwksg32fijy7PZJ8d+I=
 github.com/flyteorg/flytestdlib v0.3.21/go.mod h1:1XG0DwYTUm34Yrffm1Qy9Tdr/pWQypEqTq5dUxw3/cM=