From 7c31c1e8f96024bdb9bdf10dcda862eb09d4b579 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Thu, 2 Sep 2021 11:22:26 -0700 Subject: [PATCH] Adopt flyteidl's ordered variable map change (#158) * Adopt flyteidl's ordered variable map change Signed-off-by: Sean Lin * wip Adopt flyteidl's ordered variable map change Signed-off-by: Sean Lin * wip Adopt flyteidl's ordered variable map change Signed-off-by: Sean Lin * Reformat json Signed-off-by: Sean Lin * Reformat json Signed-off-by: Sean Lin * Parameter field name change Signed-off-by: Sean Lin * Make execfile backward compatible Signed-off-by: Sean Lin * Remove legacy proto Signed-off-by: Sean Lin * Format test data Signed-off-by: Sean Lin * Update flyteidl Signed-off-by: Haytham Abuelfutuh * goimports Signed-off-by: Haytham Abuelfutuh Co-authored-by: Haytham Abuelfutuh --- flytectl/cmd/create/execution_test.go | 52 +- flytectl/cmd/create/serialization_utils.go | 22 +- .../cmd/create/serialization_utils_test.go | 52 +- flytectl/cmd/get/execution_util.go | 34 +- flytectl/cmd/get/execution_util_test.go | 13 +- flytectl/cmd/get/launch_plan.go | 4 +- flytectl/cmd/get/launch_plan_test.go | 864 ++++++++++++------ flytectl/cmd/get/task.go | 14 +- flytectl/cmd/get/task_test.go | 328 +++++-- flytectl/cmd/get/workflow.go | 4 +- flytectl/cmd/get/workflow_test.go | 38 +- flytectl/go.mod | 2 +- flytectl/go.sum | 4 +- flytectl/pkg/ext/launch_plan_fetcher_test.go | 41 +- flytectl/pkg/ext/task_fetcher_test.go | 11 +- flytectl/pkg/ext/workflow_fetcher_test.go | 11 +- flytectl/pkg/printer/printer.go | 46 +- flytectl/pkg/printer/printer_test.go | 65 +- .../compiled_closure_branch_nested.json | 104 ++- .../testdata/compiled_subworkflows.json | 157 ++-- 20 files changed, 1221 insertions(+), 645 deletions(-) diff --git a/flytectl/cmd/create/execution_test.go b/flytectl/cmd/create/execution_test.go index 65df850ae5..36166fdf21 100644 --- a/flytectl/cmd/create/execution_test.go +++ b/flytectl/cmd/create/execution_test.go @@ -32,9 +32,14 @@ func createExecutionSetup() { }, }, } - variableMap := map[string]*core.Variable{ - "sorted_list1": &sortedListLiteralType, - "sorted_list2": &sortedListLiteralType, + variableMap := []*core.VariableMapEntry{ + { + Name: "sorted_list1", + Var: &sortedListLiteralType, + }, { + Name: "sorted_list2", + Var: &sortedListLiteralType, + }, } task1 := &admin.Task{ @@ -56,9 +61,10 @@ func createExecutionSetup() { }, } mockClient.OnGetTaskMatch(ctx, mock.Anything).Return(task1, nil) - parameterMap := map[string]*core.Parameter{ - "numbers": { - Var: &core.Variable{ + parameterMap := []*core.ParameterMapEntry{ + { + Name: "numbers", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_CollectionType{ CollectionType: &core.LiteralType{ @@ -68,40 +74,42 @@ func createExecutionSetup() { }, }, }, - }, + }}, }, - "numbers_count": { - Var: &core.Variable{ + { + Name: "numbers_count", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_INTEGER, }, }, - }, + }}, }, - "run_local_at_count": { - Var: &core.Variable{ + { + Name: "run_local_at_count", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_INTEGER, }, }, }, - Behavior: &core.Parameter_Default{ - Default: &core.Literal{ - Value: &core.Literal_Scalar{ - Scalar: &core.Scalar{ - Value: &core.Scalar_Primitive{ - Primitive: &core.Primitive{ - Value: &core.Primitive_Integer{ - Integer: 10, + Behavior: &core.Parameter_Default{ + Default: &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 10, + }, }, }, }, }, }, - }, - }, + }}, }, } launchPlan1 := &admin.LaunchPlan{ diff --git a/flytectl/cmd/create/serialization_utils.go b/flytectl/cmd/create/serialization_utils.go index 53045d2c9e..568ed6dc27 100644 --- a/flytectl/cmd/create/serialization_utils.go +++ b/flytectl/cmd/create/serialization_utils.go @@ -11,15 +11,15 @@ import ( // MakeLiteralForVariables builds a map of literals for the provided serialized values. If a provided value does not have // a corresponding variable or if that variable is invalid (e.g. doesn't have Type property populated), it returns an // error. -func MakeLiteralForVariables(serialize map[string]interface{}, variables map[string]*core.Variable) (map[string]*core.Literal, error) { +func MakeLiteralForVariables(serialize map[string]interface{}, variables []*core.VariableMapEntry) (map[string]*core.Literal, error) { types := make(map[string]*core.LiteralType) - for k, v := range variables { - t := v.GetType() + for _, e := range variables { + t := e.GetVar().GetType() if t == nil { - return nil, fmt.Errorf("variable [%v] has nil type", k) + return nil, fmt.Errorf("variable [%v] has nil type", e.GetName()) } - types[k] = t + types[e.GetName()] = t } return MakeLiteralForTypes(serialize, types) @@ -28,15 +28,15 @@ func MakeLiteralForVariables(serialize map[string]interface{}, variables map[str // MakeLiteralForParams builds a map of literals for the provided serialized values. If a provided value does not have // a corresponding parameter or if that parameter is invalid (e.g. doesn't have Type property populated), it returns an // error. -func MakeLiteralForParams(serialize map[string]interface{}, parameters map[string]*core.Parameter) (map[string]*core.Literal, error) { +func MakeLiteralForParams(serialize map[string]interface{}, parameters []*core.ParameterMapEntry) (map[string]*core.Literal, error) { types := make(map[string]*core.LiteralType) - for k, v := range parameters { - if variable := v.GetVar(); variable == nil { - return nil, fmt.Errorf("parameter [%v] has nil Variable", k) + for _, e := range parameters { + if variable := e.GetParameter().GetVar(); variable == nil { + return nil, fmt.Errorf("parameter [%v] has nil Variable", e.GetName()) } else if t := variable.GetType(); t == nil { - return nil, fmt.Errorf("parameter [%v] has nil variable type", k) + return nil, fmt.Errorf("parameter [%v] has nil variable type", e.GetName()) } else { - types[k] = t + types[e.GetName()] = t } } diff --git a/flytectl/cmd/create/serialization_utils_test.go b/flytectl/cmd/create/serialization_utils_test.go index 5c7326493d..ebe4484472 100644 --- a/flytectl/cmd/create/serialization_utils_test.go +++ b/flytectl/cmd/create/serialization_utils_test.go @@ -67,15 +67,16 @@ func TestMakeLiteralForParams(t *testing.T) { } t.Run("Happy path", func(t *testing.T) { - inputParams := map[string]*core.Parameter{ - "a": { - Var: &core.Variable{ + inputParams := []*core.ParameterMapEntry{ + { + Name: "a", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_STRING, }, }, - }, + }}, }, } @@ -85,8 +86,11 @@ func TestMakeLiteralForParams(t *testing.T) { }) t.Run("Invalid Param", func(t *testing.T) { - inputParams := map[string]*core.Parameter{ - "a": nil, + inputParams := []*core.ParameterMapEntry{ + { + Name: "a", + Parameter: nil, + }, } _, err := MakeLiteralForParams(inputValues, inputParams) @@ -94,9 +98,10 @@ func TestMakeLiteralForParams(t *testing.T) { }) t.Run("Invalid Type", func(t *testing.T) { - inputParams := map[string]*core.Parameter{ - "a": { - Var: &core.Variable{}, + inputParams := []*core.ParameterMapEntry{ + { + Name: "a", + Parameter: &core.Parameter{Var: &core.Variable{}}, }, } @@ -111,11 +116,14 @@ func TestMakeLiteralForVariables(t *testing.T) { } t.Run("Happy path", func(t *testing.T) { - inputVariables := map[string]*core.Variable{ - "a": { - Type: &core.LiteralType{ - Type: &core.LiteralType_Simple{ - Simple: core.SimpleType_STRING, + inputVariables := []*core.VariableMapEntry{ + { + Name: "a", + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRING, + }, }, }, }, @@ -127,8 +135,11 @@ func TestMakeLiteralForVariables(t *testing.T) { }) t.Run("Invalid Variable", func(t *testing.T) { - inputVariables := map[string]*core.Variable{ - "a": nil, + inputVariables := []*core.VariableMapEntry{ + { + Name: "a", + Var: nil, + }, } _, err := MakeLiteralForVariables(inputValues, inputVariables) @@ -136,9 +147,12 @@ func TestMakeLiteralForVariables(t *testing.T) { }) t.Run("Invalid Type", func(t *testing.T) { - inputVariables := map[string]*core.Variable{ - "a": { - Type: nil, + inputVariables := []*core.VariableMapEntry{ + { + Name: "a", + Var: &core.Variable{ + Type: nil, + }, }, } diff --git a/flytectl/cmd/get/execution_util.go b/flytectl/cmd/get/execution_util.go index 65b72175a5..8469e68e1d 100644 --- a/flytectl/cmd/get/execution_util.go +++ b/flytectl/cmd/get/execution_util.go @@ -58,8 +58,8 @@ func CreateAndWriteExecConfigForWorkflow(wlp *admin.LaunchPlan, fileName string) return WriteExecConfigToFile(executionConfig, fileName) } -func TaskInputs(task *admin.Task) map[string]*core.Variable { - taskInputs := map[string]*core.Variable{} +func TaskInputs(task *admin.Task) []*core.VariableMapEntry { + taskInputs := []*core.VariableMapEntry{} if task == nil || task.Closure == nil { return taskInputs } @@ -81,10 +81,10 @@ func TaskInputs(task *admin.Task) map[string]*core.Variable { func ParamMapForTask(task *admin.Task) (map[string]yaml.Node, error) { taskInputs := TaskInputs(task) paramMap := make(map[string]yaml.Node, len(taskInputs)) - for k, v := range taskInputs { - varTypeValue, err := coreutils.MakeDefaultLiteralForType(v.Type) + for _, e := range taskInputs { + varTypeValue, err := coreutils.MakeDefaultLiteralForType(e.Var.Type) if err != nil { - fmt.Println("error creating default value for literal type ", v.Type) + fmt.Println("error creating default value for literal type ", e.Var.Type) return nil, err } var nativeLiteral interface{} @@ -92,11 +92,11 @@ func ParamMapForTask(task *admin.Task) (map[string]yaml.Node, error) { return nil, err } - if k == v.Description { + if e.Name == e.Var.Description { // a: # a isn't very helpful - paramMap[k], err = getCommentedYamlNode(nativeLiteral, "") + paramMap[e.Name], err = getCommentedYamlNode(nativeLiteral, "") } else { - paramMap[k], err = getCommentedYamlNode(nativeLiteral, v.Description) + paramMap[e.Name], err = getCommentedYamlNode(nativeLiteral, e.Var.Description) } if err != nil { return nil, err @@ -105,8 +105,8 @@ func ParamMapForTask(task *admin.Task) (map[string]yaml.Node, error) { return paramMap, nil } -func WorkflowParams(lp *admin.LaunchPlan) map[string]*core.Parameter { - workflowParams := map[string]*core.Parameter{} +func WorkflowParams(lp *admin.LaunchPlan) []*core.ParameterMapEntry { + workflowParams := []*core.ParameterMapEntry{} if lp == nil || lp.Spec == nil { return workflowParams } @@ -119,10 +119,10 @@ func WorkflowParams(lp *admin.LaunchPlan) map[string]*core.Parameter { func ParamMapForWorkflow(lp *admin.LaunchPlan) (map[string]yaml.Node, error) { workflowParams := WorkflowParams(lp) paramMap := make(map[string]yaml.Node, len(workflowParams)) - for k, v := range workflowParams { - varTypeValue, err := coreutils.MakeDefaultLiteralForType(v.Var.Type) + for _, e := range workflowParams { + varTypeValue, err := coreutils.MakeDefaultLiteralForType(e.Parameter.Var.Type) if err != nil { - fmt.Println("error creating default value for literal type ", v.Var.Type) + fmt.Println("error creating default value for literal type ", e.Parameter.Var.Type) return nil, err } var nativeLiteral interface{} @@ -130,16 +130,16 @@ func ParamMapForWorkflow(lp *admin.LaunchPlan) (map[string]yaml.Node, error) { return nil, err } // Override if there is a default value - if paramsDefault, ok := v.Behavior.(*core.Parameter_Default); ok { + if paramsDefault, ok := e.Parameter.Behavior.(*core.Parameter_Default); ok { if nativeLiteral, err = coreutils.ExtractFromLiteral(paramsDefault.Default); err != nil { return nil, err } } - if k == v.Var.Description { + if e.Name == e.Parameter.Var.Description { // a: # a isn't very helpful - paramMap[k], err = getCommentedYamlNode(nativeLiteral, "") + paramMap[e.Name], err = getCommentedYamlNode(nativeLiteral, "") } else { - paramMap[k], err = getCommentedYamlNode(nativeLiteral, v.Var.Description) + paramMap[e.Name], err = getCommentedYamlNode(nativeLiteral, e.Parameter.Var.Description) } if err != nil { diff --git a/flytectl/cmd/get/execution_util_test.go b/flytectl/cmd/get/execution_util_test.go index 2d98c8b834..5a2c53aee7 100644 --- a/flytectl/cmd/get/execution_util_test.go +++ b/flytectl/cmd/get/execution_util_test.go @@ -11,7 +11,7 @@ import ( ) func TestTaskInputs(t *testing.T) { - taskInputs := map[string]*core.Variable{} + taskInputs := []*core.VariableMapEntry{} t.Run("nil task", func(t *testing.T) { retValue := TaskInputs(nil) assert.Equal(t, taskInputs, retValue) @@ -60,9 +60,14 @@ func createTask() *admin.Task { }, } - variableMap := map[string]*core.Variable{ - "sorted_list1": &sortedListLiteralType, - "sorted_list2": &sortedListLiteralType, + variableMap := []*core.VariableMapEntry{ + { + Name: "sorted_list1", + Var: &sortedListLiteralType, + }, { + Name: "sorted_list2", + Var: &sortedListLiteralType, + }, } inputs := &core.VariableMap{ diff --git a/flytectl/cmd/get/launch_plan.go b/flytectl/cmd/get/launch_plan.go index 6ef85d6cb3..dc19efeaa9 100644 --- a/flytectl/cmd/get/launch_plan.go +++ b/flytectl/cmd/get/launch_plan.go @@ -104,8 +104,8 @@ var launchplanColumns = []printer.Column{ {Header: "Type", JSONPath: "$.closure.compiledTask.template.type"}, {Header: "State", JSONPath: "$.spec.state"}, {Header: "Schedule", JSONPath: "$.spec.entityMetadata.schedule"}, - {Header: "Inputs", JSONPath: "$.closure.expectedInputs.parameters." + printer.DefaultFormattedDescriptionsKey + ".var.description"}, - {Header: "Outputs", JSONPath: "$.closure.expectedOutputs.variables." + printer.DefaultFormattedDescriptionsKey + ".description"}, + {Header: "Inputs", JSONPath: "$.closure.expectedInputs.parameters[0].parameter.var.description"}, + {Header: "Outputs", JSONPath: "$.closure.expectedOutputs.variables[0].var.description"}, } // Column structure for get all launchplans diff --git a/flytectl/cmd/get/launch_plan_test.go b/flytectl/cmd/get/launch_plan_test.go index e6a7cbec01..cf944cf5e6 100644 --- a/flytectl/cmd/get/launch_plan_test.go +++ b/flytectl/cmd/get/launch_plan_test.go @@ -39,9 +39,10 @@ func getLaunchPlanSetup() { // TODO: migrate to new command context from testutils cmdCtx = cmdCore.NewCommandContext(mockClient, u.MockOutStream) argsLp = []string{"launchplan1"} - parameterMap := map[string]*core.Parameter{ - "numbers": { - Var: &core.Variable{ + parameterMap := []*core.ParameterMapEntry{ + { + Name: "numbers", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_CollectionType{ CollectionType: &core.LiteralType{ @@ -51,21 +52,21 @@ func getLaunchPlanSetup() { }, }, }, - Description: "short desc", - }, + Description: "short desc"}}, }, - "numbers_count": { - Var: &core.Variable{ + { + Name: "numbers_count", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_INTEGER, }, - }, - Description: "long description will be truncated in table", - }, + }, Description: "long description will be truncated in table", + }}, }, - "run_local_at_count": { - Var: &core.Variable{ + { + Name: "run_local_at_count", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_INTEGER, @@ -73,21 +74,21 @@ func getLaunchPlanSetup() { }, Description: "run_local_at_count", }, - Behavior: &core.Parameter_Default{ - Default: &core.Literal{ - Value: &core.Literal_Scalar{ - Scalar: &core.Scalar{ - Value: &core.Scalar_Primitive{ - Primitive: &core.Primitive{ - Value: &core.Primitive_Integer{ - Integer: 10, + Behavior: &core.Parameter_Default{ + Default: &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 10, + }, }, }, }, }, }, - }, - }, + }}, }, } launchPlan1 := &admin.LaunchPlan{ @@ -257,80 +258,98 @@ func TestGetLaunchPlanFunc(t *testing.T) { }, "spec": { "defaultInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] } }, "closure": { "expectedInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] }, "createdAt": "1970-01-01T00:00:01Z" } @@ -342,80 +361,98 @@ func TestGetLaunchPlanFunc(t *testing.T) { }, "spec": { "defaultInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] } }, "closure": { "expectedInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] }, "createdAt": "1970-01-01T00:00:00Z" } @@ -440,80 +477,98 @@ func TestGetLaunchPlanFuncLatest(t *testing.T) { }, "spec": { "defaultInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] } }, "closure": { "expectedInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] }, "createdAt": "1970-01-01T00:00:01Z" } @@ -537,80 +592,98 @@ func TestGetLaunchPlanWithVersion(t *testing.T) { }, "spec": { "defaultInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] } }, "closure": { "expectedInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] }, "createdAt": "1970-01-01T00:00:01Z" } @@ -625,7 +698,214 @@ func TestGetLaunchPlans(t *testing.T) { argsLp = []string{} err = getLaunchPlanFunc(ctx, argsLp, cmdCtx) assert.Nil(t, err) - tearDownAndVerify(t, `[{"id": {"name": "launchplan1","version": "v2"},"spec": {"defaultInputs": {"parameters": {"numbers": {"var": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "short desc"}},"numbers_count": {"var": {"type": {"simple": "INTEGER"},"description": "long description will be truncated in table"}},"run_local_at_count": {"var": {"type": {"simple": "INTEGER"},"description": "run_local_at_count"},"default": {"scalar": {"primitive": {"integer": "10"}}}}}}},"closure": {"expectedInputs": {"parameters": {"numbers": {"var": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "short desc"}},"numbers_count": {"var": {"type": {"simple": "INTEGER"},"description": "long description will be truncated in table"}},"run_local_at_count": {"var": {"type": {"simple": "INTEGER"},"description": "run_local_at_count"},"default": {"scalar": {"primitive": {"integer": "10"}}}}}},"createdAt": "1970-01-01T00:00:01Z"}},{"id": {"name": "launchplan1","version": "v1"},"spec": {"defaultInputs": {"parameters": {"numbers": {"var": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "short desc"}},"numbers_count": {"var": {"type": {"simple": "INTEGER"},"description": "long description will be truncated in table"}},"run_local_at_count": {"var": {"type": {"simple": "INTEGER"},"description": "run_local_at_count"},"default": {"scalar": {"primitive": {"integer": "10"}}}}}}},"closure": {"expectedInputs": {"parameters": {"numbers": {"var": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "short desc"}},"numbers_count": {"var": {"type": {"simple": "INTEGER"},"description": "long description will be truncated in table"}},"run_local_at_count": {"var": {"type": {"simple": "INTEGER"},"description": "run_local_at_count"},"default": {"scalar": {"primitive": {"integer": "10"}}}}}},"createdAt": "1970-01-01T00:00:00Z"}}]`) + tearDownAndVerify(t, `[ + { + "id": { + "name": "launchplan1", + "version": "v2" + }, + "spec": { + "defaultInputs": { + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } + } + }, + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } + } + }, + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + ] + } + }, + "closure": { + "expectedInputs": { + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } + } + }, + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } + } + }, + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + ] + }, + "createdAt": "1970-01-01T00:00:01Z" + } + }, + { + "id": { + "name": "launchplan1", + "version": "v1" + }, + "spec": { + "defaultInputs": { + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } + } + }, + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } + } + }, + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + ] + } + }, + "closure": { + "expectedInputs": { + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } + } + }, + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } + } + }, + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + ] + }, + "createdAt": "1970-01-01T00:00:00Z" + } + } +]`) } func TestGetLaunchPlansWithExecFile(t *testing.T) { @@ -647,80 +927,98 @@ func TestGetLaunchPlansWithExecFile(t *testing.T) { }, "spec": { "defaultInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] } }, "closure": { "expectedInputs": { - "parameters": { - "numbers": { - "var": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "short desc" + "parameters": [ + { + "name": "numbers", + "parameter": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "short desc" + } } }, - "numbers_count": { - "var": { - "type": { - "simple": "INTEGER" - }, - "description": "long description will be truncated in table" + { + "name": "numbers_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "long description will be truncated in table" + } } }, - "run_local_at_count": { - "var": { - "type": { - "simple": "INTEGER" + { + "name": "run_local_at_count", + "parameter": { + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "run_local_at_count" }, - "description": "run_local_at_count" - }, - "default": { - "scalar": { - "primitive": { - "integer": "10" + "default": { + "scalar": { + "primitive": { + "integer": "10" + } } } } } - } + ] }, "createdAt": "1970-01-01T00:00:01Z" } diff --git a/flytectl/cmd/get/task.go b/flytectl/cmd/get/task.go index 63b490ae82..e3ce88e348 100644 --- a/flytectl/cmd/get/task.go +++ b/flytectl/cmd/get/task.go @@ -3,6 +3,8 @@ package get import ( "context" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flytectl/cmd/config" taskConfig "github.com/flyteorg/flytectl/cmd/config/subcommand/task" cmdCore "github.com/flyteorg/flytectl/cmd/core" @@ -97,8 +99,8 @@ var taskColumns = []printer.Column{ {Header: "Version", JSONPath: "$.id.version"}, {Header: "Name", JSONPath: "$.id.name"}, {Header: "Type", JSONPath: "$.closure.compiledTask.template.type"}, - {Header: "Inputs", JSONPath: "$.closure.compiledTask.template.interface.inputs.variables." + printer.DefaultFormattedDescriptionsKey + ".description"}, - {Header: "Outputs", JSONPath: "$.closure.compiledTask.template.interface.outputs.variables." + printer.DefaultFormattedDescriptionsKey + ".description"}, + {Header: "Inputs", JSONPath: "$.closure.compiledTask.template.interface.inputs.variables[0].var.description"}, + {Header: "Outputs", JSONPath: "$.closure.compiledTask.template.interface.outputs.variables[0].var.description"}, {Header: "Discoverable", JSONPath: "$.closure.compiledTask.template.metadata.discoverable"}, {Header: "Discovery Version", JSONPath: "$.closure.compiledTask.template.metadata.discoveryVersion"}, {Header: "Created At", JSONPath: "$.closure.createdAt"}, @@ -133,6 +135,14 @@ func TaskToTableProtoMessages(l []*admin.Task) []proto.Message { return messages } +func VariableMapEntriesToMap(mapFieldEntries []*core.VariableMapEntry) (variableMap map[string]*core.Variable) { + variableMap = map[string]*core.Variable{} + for _, e := range mapFieldEntries { + variableMap[e.Name] = e.Var + } + return +} + func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { taskPrinter := printer.Printer{} var tasks []*admin.Task diff --git a/flytectl/cmd/get/task_test.go b/flytectl/cmd/get/task_test.go index 0133c20f46..dfed9cea1f 100644 --- a/flytectl/cmd/get/task_test.go +++ b/flytectl/cmd/get/task_test.go @@ -54,9 +54,14 @@ func getTaskSetup() { }, Description: "var description", } - variableMap := map[string]*core.Variable{ - "sorted_list1": &sortedListLiteralType, - "sorted_list2": &sortedListLiteralType, + variableMap := []*core.VariableMapEntry{ + { + Name: "sorted_list1", + Var: &sortedListLiteralType, + }, { + Name: "sorted_list2", + Var: &sortedListLiteralType, + }, } task1 := &admin.Task{ @@ -256,24 +261,30 @@ func TestGetTaskFunc(t *testing.T) { "template": { "interface": { "inputs": { - "variables": { - "sorted_list1": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } }, - "sorted_list2": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } } - } + ] } } } @@ -291,24 +302,30 @@ func TestGetTaskFunc(t *testing.T) { "template": { "interface": { "inputs": { - "variables": { - "sorted_list1": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } }, - "sorted_list2": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } } - } + ] } } } @@ -363,24 +380,30 @@ func TestGetTaskFuncLatest(t *testing.T) { "template": { "interface": { "inputs": { - "variables": { - "sorted_list1": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } }, - "sorted_list2": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } } - } + ] } } } @@ -412,24 +435,30 @@ func TestGetTaskWithVersion(t *testing.T) { "template": { "interface": { "inputs": { - "variables": { - "sorted_list1": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } }, - "sorted_list2": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } } - } + ] } } } @@ -447,7 +476,90 @@ func TestGetTasks(t *testing.T) { mockClient.OnGetTaskMatch(ctx, objectGetRequestTask).Return(task2, nil) err = getTaskFunc(ctx, argsTask, cmdCtx) assert.Nil(t, err) - tearDownAndVerify(t, `[{"id": {"name": "task1","version": "v2"},"closure": {"compiledTask": {"template": {"interface": {"inputs": {"variables": {"sorted_list1": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "var description"},"sorted_list2": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "var description"}}}}}},"createdAt": "1970-01-01T00:00:01Z"}},{"id": {"name": "task1","version": "v1"},"closure": {"compiledTask": {"template": {"interface": {"inputs": {"variables": {"sorted_list1": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "var description"},"sorted_list2": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "var description"}}}}}},"createdAt": "1970-01-01T00:00:00Z"}}]`) + tearDownAndVerify(t, `[ + { + "id": { + "name": "task1", + "version": "v2" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } + }, + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } + } + ] + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } + }, + { + "id": { + "name": "task1", + "version": "v1" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } + }, + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } + } + ] + } + } + } + }, + "createdAt": "1970-01-01T00:00:00Z" + } + } +]`) } func TestGetTasksFilters(t *testing.T) { @@ -459,7 +571,47 @@ func TestGetTasksFilters(t *testing.T) { mockClient.OnListTasksMatch(ctx, resourceListFilterRequestTask).Return(taskListFilterResponse, nil) err = getTaskFunc(ctx, argsTask, cmdCtx) assert.Nil(t, err) - tearDownAndVerify(t, `{"id": {"name": "task1","version": "v1"},"closure": {"compiledTask": {"template": {"interface": {"inputs": {"variables": {"sorted_list1": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "var description"},"sorted_list2": {"type": {"collectionType": {"simple": "INTEGER"}},"description": "var description"}}}}}},"createdAt": "1970-01-01T00:00:00Z"}}`) + tearDownAndVerify(t, `{ + "id": { + "name": "task1", + "version": "v1" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } + }, + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } + } + ] + } + } + } + }, + "createdAt": "1970-01-01T00:00:00Z" + } +}`) } func TestGetTaskWithExecFile(t *testing.T) { @@ -484,24 +636,30 @@ func TestGetTaskWithExecFile(t *testing.T) { "template": { "interface": { "inputs": { - "variables": { - "sorted_list1": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + "variables": [ + { + "name": "sorted_list1", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } }, - "sorted_list2": { - "type": { - "collectionType": { - "simple": "INTEGER" - } - }, - "description": "var description" + { + "name": "sorted_list2", + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "var description" + } } - } + ] } } } diff --git a/flytectl/cmd/get/workflow.go b/flytectl/cmd/get/workflow.go index 2670b7f10d..f461aac9e3 100644 --- a/flytectl/cmd/get/workflow.go +++ b/flytectl/cmd/get/workflow.go @@ -87,8 +87,8 @@ Usage var workflowColumns = []printer.Column{ {Header: "Version", JSONPath: "$.id.version"}, {Header: "Name", JSONPath: "$.id.name"}, - {Header: "Inputs", JSONPath: "$.closure.compiledWorkflow.primary.template.interface.inputs.variables." + printer.DefaultFormattedDescriptionsKey + ".description"}, - {Header: "Outputs", JSONPath: "$.closure.compiledWorkflow.primary.template.interface.outputs.variables." + printer.DefaultFormattedDescriptionsKey + ".description"}, + {Header: "Inputs", JSONPath: "$.closure.compiledWorkflow.primary.template.interface.inputs.variables[0].var.description"}, + {Header: "Outputs", JSONPath: "$.closure.compiledWorkflow.primary.template.interface.outputs.variables[0].var.description"}, {Header: "Created At", JSONPath: "$.closure.createdAt"}, } diff --git a/flytectl/cmd/get/workflow_test.go b/flytectl/cmd/get/workflow_test.go index 46ad846cdd..54c826d423 100644 --- a/flytectl/cmd/get/workflow_test.go +++ b/flytectl/cmd/get/workflow_test.go @@ -38,31 +38,35 @@ func getWorkflowSetup() { Domain: domainValue, }, } - - variableMap := map[string]*core.Variable{ - "var1": { - Type: &core.LiteralType{ - Type: &core.LiteralType_CollectionType{ - CollectionType: &core.LiteralType{ - Type: &core.LiteralType_Simple{ - Simple: core.SimpleType_INTEGER, + variableMap := []*core.VariableMapEntry{ + { + Name: "var1", + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, }, }, }, + Description: "var1", }, - Description: "var1", - }, - "var2": { - Type: &core.LiteralType{ - Type: &core.LiteralType_CollectionType{ - CollectionType: &core.LiteralType{ - Type: &core.LiteralType_Simple{ - Simple: core.SimpleType_INTEGER, + }, { + Name: "var2", + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, }, }, }, + Description: "var2 long descriptions probably needs truncate", }, - Description: "var2 long descriptions probably needs truncate", }, } workflow1 = &admin.Workflow{ diff --git a/flytectl/go.mod b/flytectl/go.mod index 533ac14d2f..1873b540e2 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 - github.com/flyteorg/flyteidl v0.19.24 + github.com/flyteorg/flyteidl v0.20.2 github.com/flyteorg/flytestdlib v0.3.34 github.com/ghodss/yaml v1.0.0 github.com/golang/protobuf v1.4.3 diff --git a/flytectl/go.sum b/flytectl/go.sum index c99b90cda6..1c820fdecf 100644 --- a/flytectl/go.sum +++ b/flytectl/go.sum @@ -345,8 +345,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.19.24 h1:aiBKTFNKZpIvOfcI051I33LnhQl54N3dhkaubucOg+E= -github.com/flyteorg/flyteidl v0.19.24/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= +github.com/flyteorg/flyteidl v0.20.2 h1:3DDj1y9Axmb35SskN/h2nRgohWhGBPGxmJSX7b/Y2rk= +github.com/flyteorg/flyteidl v0.20.2/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/flyteorg/flytestdlib v0.3.34 h1:OOuV03X8c1AWInzBU6IRsqpEF6y8WDJngbPcdL4VktY= github.com/flyteorg/flytestdlib v0.3.34/go.mod h1:7cDWkY3v7xsoesFcDdu6DSW5Q2U2W5KlHUbUHSwBG1Q= diff --git a/flytectl/pkg/ext/launch_plan_fetcher_test.go b/flytectl/pkg/ext/launch_plan_fetcher_test.go index 39d0121c93..e38d0d054e 100644 --- a/flytectl/pkg/ext/launch_plan_fetcher_test.go +++ b/flytectl/pkg/ext/launch_plan_fetcher_test.go @@ -27,9 +27,10 @@ func getLaunchPlanFetcherSetup() { adminClient = new(mocks.AdminServiceClient) adminFetcherExt = AdminFetcherExtClient{AdminClient: adminClient} - parameterMap := map[string]*core.Parameter{ - "numbers": { - Var: &core.Variable{ + parameterMap := []*core.ParameterMapEntry{ + { + Name: "numbers", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_CollectionType{ CollectionType: &core.LiteralType{ @@ -39,40 +40,42 @@ func getLaunchPlanFetcherSetup() { }, }, }, - }, + }}, }, - "numbers_count": { - Var: &core.Variable{ + { + Name: "numbers_count", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_INTEGER, }, }, - }, + }}, }, - "run_local_at_count": { - Var: &core.Variable{ + { + Name: "run_local_at_count", + Parameter: &core.Parameter{Var: &core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_Simple{ Simple: core.SimpleType_INTEGER, }, }, }, - Behavior: &core.Parameter_Default{ - Default: &core.Literal{ - Value: &core.Literal_Scalar{ - Scalar: &core.Scalar{ - Value: &core.Scalar_Primitive{ - Primitive: &core.Primitive{ - Value: &core.Primitive_Integer{ - Integer: 10, + Behavior: &core.Parameter_Default{ + Default: &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 10, + }, }, }, }, }, }, - }, - }, + }}, }, } launchPlan1 = &admin.LaunchPlan{ diff --git a/flytectl/pkg/ext/task_fetcher_test.go b/flytectl/pkg/ext/task_fetcher_test.go index 0bfc9ef8dd..50d14e2144 100644 --- a/flytectl/pkg/ext/task_fetcher_test.go +++ b/flytectl/pkg/ext/task_fetcher_test.go @@ -41,9 +41,14 @@ func getTaskFetcherSetup() { }, }, } - variableMap := map[string]*core.Variable{ - "sorted_list1": &sortedListLiteralType, - "sorted_list2": &sortedListLiteralType, + variableMap := []*core.VariableMapEntry{ + { + Name: "sorted_list1", + Var: &sortedListLiteralType, + }, { + Name: "sorted_list2", + Var: &sortedListLiteralType, + }, } task1 := &admin.Task{ diff --git a/flytectl/pkg/ext/workflow_fetcher_test.go b/flytectl/pkg/ext/workflow_fetcher_test.go index 9ae96612a6..c8da9b2373 100644 --- a/flytectl/pkg/ext/workflow_fetcher_test.go +++ b/flytectl/pkg/ext/workflow_fetcher_test.go @@ -38,9 +38,14 @@ func getWorkflowFetcherSetup() { }, }, } - variableMap := map[string]*core.Variable{ - "sorted_list1": &sortedListLiteralType, - "sorted_list2": &sortedListLiteralType, + variableMap := []*core.VariableMapEntry{ + { + Name: "sorted_list1", + Var: &sortedListLiteralType, + }, { + Name: "sorted_list2", + Var: &sortedListLiteralType, + }, } var compiledTasks []*core.CompiledTask diff --git a/flytectl/pkg/printer/printer.go b/flytectl/pkg/printer/printer.go index 75d67208eb..0f11d15ac7 100644 --- a/flytectl/pkg/printer/printer.go +++ b/flytectl/pkg/printer/printer.go @@ -6,7 +6,6 @@ import ( "fmt" "net/url" "os" - "sort" "strings" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -170,50 +169,37 @@ func printJSONYaml(format OutputFormat, v interface{}) error { return nil } -func FormatVariableDescriptions(variableMap map[string]*core.Variable) { - keys := make([]string, 0, len(variableMap)) - // sort the keys for testing and consistency with other output formats - for k := range variableMap { - keys = append(keys, k) - } - sort.Strings(keys) - +func FormatVariableDescriptions(variableMap []*core.VariableMapEntry) { var descriptions []string - for _, k := range keys { - v := variableMap[k] + for _, e := range variableMap { + if e.Var == nil { + continue + } // a: a isn't very helpful - if k != v.Description { - descriptions = append(descriptions, getTruncatedLine(fmt.Sprintf("%s: %s", k, v.Description))) + if e.Name != e.Var.Description { + descriptions = append(descriptions, getTruncatedLine(fmt.Sprintf("%s: %s", e.Name, e.Var.Description))) } else { - descriptions = append(descriptions, getTruncatedLine(k)) + descriptions = append(descriptions, getTruncatedLine(e.Name)) } } - variableMap[DefaultFormattedDescriptionsKey] = &core.Variable{Description: strings.Join(descriptions, "\n")} + variableMap[0] = &core.VariableMapEntry{Var: &core.Variable{Description: strings.Join(descriptions, "\n")}} } -func FormatParameterDescriptions(parameterMap map[string]*core.Parameter) { - keys := make([]string, 0, len(parameterMap)) - // sort the keys for testing and consistency with other output formats - for k := range parameterMap { - keys = append(keys, k) - } - sort.Strings(keys) - +func FormatParameterDescriptions(parameterMap []*core.ParameterMapEntry) { var descriptions []string - for _, k := range keys { - v := parameterMap[k] - if v.Var == nil { + for _, e := range parameterMap { + if e.Parameter == nil || e.Parameter.Var == nil { continue } // a: a isn't very helpful - if k != v.Var.Description { - descriptions = append(descriptions, getTruncatedLine(fmt.Sprintf("%s: %s", k, v.Var.Description))) + if e.Name != e.Parameter.Var.Description { + descriptions = append(descriptions, getTruncatedLine(fmt.Sprintf("%s: %s", e.Name, e.Parameter.Var.Description))) } else { - descriptions = append(descriptions, getTruncatedLine(k)) + descriptions = append(descriptions, getTruncatedLine(e.Name)) } } - parameterMap[DefaultFormattedDescriptionsKey] = &core.Parameter{Var: &core.Variable{Description: strings.Join(descriptions, "\n")}} + parameterMap[0] = &core.ParameterMapEntry{Parameter: &core.Parameter{Var: &core.Variable{Description: strings.Join(descriptions, "\n")}}} } func getTruncatedLine(line string) string { diff --git a/flytectl/pkg/printer/printer_test.go b/flytectl/pkg/printer/printer_test.go index 6d5441b9af..1d0ee4e15c 100644 --- a/flytectl/pkg/printer/printer_test.go +++ b/flytectl/pkg/printer/printer_test.go @@ -150,9 +150,15 @@ func TestPrint(t *testing.T) { }, }, } - variableMap := map[string]*core.Variable{ - "sorted_list1": &sortedListLiteralType, - "sorted_list2": &sortedListLiteralType, + variableMap := []*core.VariableMapEntry{ + { + Name: "sorted_list1", + Var: &sortedListLiteralType, + }, + { + Name: "sorted_list2", + Var: &sortedListLiteralType, + }, } var compiledTasks []*core.CompiledTask @@ -275,14 +281,26 @@ func TestFormatVariableDescriptions(t *testing.T) { barVar := &core.Variable{ Description: "bar", } - variableMap := map[string]*core.Variable{ - "var1": fooVar, - "var2": barVar, - "foo": fooVar, - "bar": barVar, + variableMap := []*core.VariableMapEntry{ + { + Name: "var1", + Var: fooVar, + }, + { + Name: "var2", + Var: barVar, + }, + { + Name: "foo", + Var: fooVar, + }, + { + Name: "bar", + Var: barVar, + }, } FormatVariableDescriptions(variableMap) - assert.Equal(t, "bar\nfoo\nvar1: foo\nvar2: bar", variableMap[DefaultFormattedDescriptionsKey].Description) + assert.Equal(t, "var1: foo\nvar2: bar\nfoo\nbar", variableMap[0].Var.Description) } func TestFormatParameterDescriptions(t *testing.T) { @@ -297,13 +315,28 @@ func TestFormatParameterDescriptions(t *testing.T) { }, } emptyParam := &core.Parameter{} - paramMap := map[string]*core.Parameter{ - "var1": fooParam, - "var2": barParam, - "foo": fooParam, - "bar": barParam, - "empty": emptyParam, + paramMap := []*core.ParameterMapEntry{ + { + Name: "var1", + Parameter: fooParam, + }, + { + Name: "var2", + Parameter: barParam, + }, + { + Name: "foo", + Parameter: fooParam, + }, + { + Name: "bar", + Parameter: barParam, + }, + { + Name: "empty", + Parameter: emptyParam, + }, } FormatParameterDescriptions(paramMap) - assert.Equal(t, "bar\nfoo\nvar1: foo\nvar2: bar", paramMap[DefaultFormattedDescriptionsKey].Var.Description) + assert.Equal(t, "var1: foo\nvar2: bar\nfoo\nbar", paramMap[0].Parameter.Var.Description) } diff --git a/flytectl/pkg/visualize/testdata/compiled_closure_branch_nested.json b/flytectl/pkg/visualize/testdata/compiled_closure_branch_nested.json index baae3d9926..6b7330e1d7 100644 --- a/flytectl/pkg/visualize/testdata/compiled_closure_branch_nested.json +++ b/flytectl/pkg/visualize/testdata/compiled_closure_branch_nested.json @@ -11,24 +11,30 @@ "metadata": {}, "interface": { "inputs": { - "variables": { - "my_input": { - "type": { - "simple": "FLOAT" - }, - "description": "my_input" + "variables": [ + { + "name": "my_input", + "var": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } } - } + ] }, "outputs": { - "variables": { - "o0": { - "type": { - "simple": "FLOAT" - }, - "description": "o0" + "variables": [ + { + "name": "o0", + "var": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } } - } + ] } }, "nodes": [ @@ -227,7 +233,7 @@ ], "error": { "failedNodeId": "inner_fractions", - "message": "Only \u003c0.7 allowed" + "message": "Only <0.7 allowed" } } } @@ -424,24 +430,30 @@ }, "interface": { "inputs": { - "variables": { - "n": { - "type": { - "simple": "FLOAT" - }, - "description": "n" + "variables": [ + { + "name": "n", + "var": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } } - } + ] }, "outputs": { - "variables": { - "o0": { - "type": { - "simple": "FLOAT" - }, - "description": "o0" + "variables": [ + { + "name": "o0", + "var": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } } - } + ] } }, "container": { @@ -497,24 +509,30 @@ }, "interface": { "inputs": { - "variables": { - "n": { - "type": { - "simple": "FLOAT" - }, - "description": "n" + "variables": [ + { + "name": "n", + "var": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } } - } + ] }, "outputs": { - "variables": { - "o0": { - "type": { - "simple": "FLOAT" - }, - "description": "o0" + "variables": [ + { + "name": "o0", + "var": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } } - } + ] } }, "container": { diff --git a/flytectl/pkg/visualize/testdata/compiled_subworkflows.json b/flytectl/pkg/visualize/testdata/compiled_subworkflows.json index 8bbf441367..b46a334701 100644 --- a/flytectl/pkg/visualize/testdata/compiled_subworkflows.json +++ b/flytectl/pkg/visualize/testdata/compiled_subworkflows.json @@ -11,36 +11,48 @@ "metadata": {}, "interface": { "inputs": { - "variables": { - "a": { - "type": { - "simple": "INTEGER" - }, - "description": "a" + "variables": [ + { + "name": "a", + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } } - } + ] }, "outputs": { - "variables": { - "o0": { - "type": { - "simple": "INTEGER" - }, - "description": "o0" + "variables": [ + { + "name": "o0", + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } }, - "o1": { - "type": { - "simple": "STRING" - }, - "description": "o1" + { + "name": "o1", + "var": { + "type": { + "simple": "STRING" + }, + "description": "o1" + } }, - "o2": { - "type": { - "simple": "STRING" - }, - "description": "o2" + { + "name": "o2", + "var": { + "type": { + "simple": "STRING" + }, + "description": "o2" + } } - } + ] } }, "nodes": [ @@ -222,30 +234,39 @@ "metadata": {}, "interface": { "inputs": { - "variables": { - "a": { - "type": { - "simple": "INTEGER" - }, - "description": "a" + "variables": [ + { + "name": "a", + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } } - } + ] }, "outputs": { - "variables": { - "o0": { - "type": { - "simple": "STRING" - }, - "description": "o0" + "variables": [ + { + "name": "o0", + "var": { + "type": { + "simple": "STRING" + }, + "description": "o0" + } }, - "o1": { - "type": { - "simple": "STRING" - }, - "description": "o1" + { + "name": "o1", + "var": { + "type": { + "simple": "STRING" + }, + "description": "o1" + } } - } + ] } }, "nodes": [ @@ -419,30 +440,39 @@ }, "interface": { "inputs": { - "variables": { - "a": { - "type": { - "simple": "INTEGER" - }, - "description": "a" + "variables": [ + { + "name": "a", + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } } - } + ] }, "outputs": { - "variables": { - "c": { - "type": { - "simple": "STRING" - }, - "description": "c" + "variables": [ + { + "name": "c", + "var": { + "type": { + "simple": "STRING" + }, + "description": "c" + } }, - "t1_int_output": { - "type": { - "simple": "INTEGER" - }, - "description": "t1_int_output" + { + "name": "t1_int_output", + "var": { + "type": { + "simple": "INTEGER" + }, + "description": "t1_int_output" + } } - } + ] } }, "container": { @@ -478,5 +508,4 @@ } } ] -} - +} \ No newline at end of file