diff --git a/flytectl/boilerplate/lyft/golang_support_tools/tools.go b/flytectl/boilerplate/lyft/golang_support_tools/tools.go index 88ff645233..a256d868f1 100644 --- a/flytectl/boilerplate/lyft/golang_support_tools/tools.go +++ b/flytectl/boilerplate/lyft/golang_support_tools/tools.go @@ -5,6 +5,6 @@ package tools import ( _ "github.com/alvaroloes/enumer" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" - _ "github.com/lyft/flytestdlib/cli/pflags" + _ "github.com/flyteorg/flytestdlib/cli/pflags" _ "github.com/vektra/mockery/cmd/mockery" ) diff --git a/flytectl/boilerplate/lyft/golang_test_targets/download_tooling.sh b/flytectl/boilerplate/lyft/golang_test_targets/download_tooling.sh index ab56c7e481..bc51af5646 100755 --- a/flytectl/boilerplate/lyft/golang_test_targets/download_tooling.sh +++ b/flytectl/boilerplate/lyft/golang_test_targets/download_tooling.sh @@ -17,7 +17,7 @@ set -e # In the format of ":" or ":" if no cli tools=( "github.com/vektra/mockery/cmd/mockery" - "github.com/lyft/flytestdlib/cli/pflags" + "github.com/flyteorg/flytestdlib/cli/pflags" "github.com/golangci/golangci-lint/cmd/golangci-lint" "github.com/alvaroloes/enumer" ) diff --git a/flytectl/cmd/create/create.go b/flytectl/cmd/create/create.go index 5fbf0b9914..ad1f6ac33c 100644 --- a/flytectl/cmd/create/create.go +++ b/flytectl/cmd/create/create.go @@ -26,6 +26,8 @@ func RemoteCreateCommand() *cobra.Command { createResourcesFuncs := map[string]cmdcore.CommandEntry{ "project": {CmdFunc: createProjectsCommand, Aliases: []string{"projects"}, ProjectDomainNotRequired: true, PFlagProvider: projectConfig, Short: projectShort, Long: projectLong}, + "execution": {CmdFunc: createExecutionCommand, Aliases: []string{"executions"}, ProjectDomainNotRequired: false, PFlagProvider: executionConfig, Short: executionShort, + Long: executionLong}, } cmdcore.AddCommands(createCmd, createResourcesFuncs) return createCmd diff --git a/flytectl/cmd/create/create_test.go b/flytectl/cmd/create/create_test.go index fb0d58a535..d2e4711a31 100644 --- a/flytectl/cmd/create/create_test.go +++ b/flytectl/cmd/create/create_test.go @@ -1,23 +1,43 @@ package create import ( + "context" "sort" "testing" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/stretchr/testify/assert" ) +const testDataFolder = "../testdata/" + +var ( + err error + ctx context.Context + mockClient *mocks.AdminServiceClient + args []string + cmdCtx cmdCore.CommandContext +) +var setup = testutils.Setup +var tearDownAndVerify = testutils.TearDownAndVerify + func TestCreateCommand(t *testing.T) { createCommand := RemoteCreateCommand() assert.Equal(t, createCommand.Use, "create") assert.Equal(t, createCommand.Short, "Used for creating various flyte resources including tasks/workflows/launchplans/executions/project.") - assert.Equal(t, len(createCommand.Commands()), 1) + assert.Equal(t, len(createCommand.Commands()), 2) cmdNouns := createCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - assert.Equal(t, cmdNouns[0].Use, "project") - assert.Equal(t, cmdNouns[0].Aliases, []string{"projects"}) - assert.Equal(t, cmdNouns[0].Short, "Create project resources") + assert.Equal(t, cmdNouns[0].Use, "execution") + assert.Equal(t, cmdNouns[0].Aliases, []string{"executions"}) + assert.Equal(t, cmdNouns[0].Short, executionShort) + assert.Equal(t, cmdNouns[1].Use, "project") + assert.Equal(t, cmdNouns[1].Aliases, []string{"projects"}) + assert.Equal(t, cmdNouns[1].Short, "Create project resources") } diff --git a/flytectl/cmd/create/execution.go b/flytectl/cmd/create/execution.go new file mode 100644 index 0000000000..b687790e12 --- /dev/null +++ b/flytectl/cmd/create/execution.go @@ -0,0 +1,129 @@ +package create + +import ( + "context" + "fmt" + + "github.com/flyteorg/flytectl/cmd/config" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + executionShort = "Create execution resources" + executionLong = ` +Create the executions for given workflow/task in a project and domain. + +There are three steps in generating an execution. + +- Generate the execution spec file using the get command. +- Update the inputs for the execution if needed. +- Run the execution by passing in the generated yaml file. + +The spec file should be generated first and then run the execution using the spec file. +You can reference the flytectl get task for more details + +:: + + flytectl get tasks -d development -p flytectldemo core.advanced.run_merge_sort.merge --version v2 --execFile execution_spec.yaml + +The generated file would look similar to this + +.. code-block:: yaml + + iamRoleARN: "" + inputs: + sorted_list1: + - 0 + sorted_list2: + - 0 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + task: core.advanced.run_merge_sort.merge + version: "v2" + + +The generated file can be modified to change the input values. + +.. code-block:: yaml + + iamRoleARN: 'arn:aws:iam::12345678:role/defaultrole' + inputs: + sorted_list1: + - 2 + - 4 + - 6 + sorted_list2: + - 1 + - 3 + - 5 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + task: core.advanced.run_merge_sort.merge + version: "v2" + +And then can be passed through the command line. +Notice the source and target domain/projects can be different. +The root project and domain flags of -p and -d should point to task/launch plans project/domain. + +:: + + flytectl create execution --execFile execution_spec.yaml -p flytectldemo -d development --targetProject flytesnacks + +Usage +` +) + +//go:generate pflags ExecutionConfig --default-var executionConfig + +// ExecutionConfig hold configuration for create execution flags and configuration of the actual task or workflow to be launched. +type ExecutionConfig struct { + // pflag section + ExecFile string `json:"execFile,omitempty" pflag:",file for the execution params.If not specified defaults to <_name>.execution_spec.yaml"` + TargetDomain string `json:"targetDomain" pflag:",project where execution needs to be created.If not specified configured domain would be used."` + TargetProject string `json:"targetProject" pflag:",project where execution needs to be created.If not specified configured project would be used."` + KubeServiceAcct string `json:"kubeServiceAcct" pflag:",kubernetes service account AuthRole for launching execution."` + IamRoleARN string `json:"iamRoleARN" pflag:",iam role ARN AuthRole for launching execution."` + // Non plfag section is read from the execution config generated by get task/launchplan + Workflow string `json:"workflow,omitempty"` + Task string `json:"task,omitempty"` + Version string `json:"version"` + Inputs map[string]interface{} `json:"inputs"` +} + +type ExecutionParams struct { + name string + isTask bool +} + +var ( + executionConfig = &ExecutionConfig{} +) + +func createExecutionCommand(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var execParams ExecutionParams + var err error + sourceProject := config.GetConfig().Project + sourceDomain := config.GetConfig().Domain + if execParams, err = readConfigAndValidate(config.GetConfig().Project, config.GetConfig().Domain); err != nil { + return err + } + var executionRequest *admin.ExecutionCreateRequest + if execParams.isTask { + if executionRequest, err = createExecutionRequestForTask(ctx, execParams.name, sourceProject, sourceDomain, cmdCtx); err != nil { + return err + } + } else { + if executionRequest, err = createExecutionRequestForWorkflow(ctx, execParams.name, sourceProject, sourceDomain, cmdCtx); err != nil { + return err + } + } + exec, _err := cmdCtx.AdminClient().CreateExecution(ctx, executionRequest) + if _err != nil { + return _err + } + fmt.Printf("execution identifier %v\n", exec.Id) + return nil +} diff --git a/flytectl/cmd/create/execution_test.go b/flytectl/cmd/create/execution_test.go new file mode 100644 index 0000000000..cb9fa4cc15 --- /dev/null +++ b/flytectl/cmd/create/execution_test.go @@ -0,0 +1,166 @@ +package create + +import ( + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// This function needs to be called after testutils.Steup() +func createExecutionSetup() { + ctx = testutils.Ctx + cmdCtx = testutils.CmdCtx + mockClient = testutils.MockClient + sortedListLiteralType := core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + } + variableMap := map[string]*core.Variable{ + "sorted_list1": &sortedListLiteralType, + "sorted_list2": &sortedListLiteralType, + } + + task1 := &admin.Task{ + Id: &core.Identifier{ + Name: "task1", + Version: "v2", + }, + Closure: &admin.TaskClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledTask: &core.CompiledTask{ + Template: &core.TaskTemplate{ + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: variableMap, + }, + }, + }, + }, + }, + } + mockClient.OnGetTaskMatch(ctx, mock.Anything).Return(task1, nil) + parameterMap := map[string]*core.Parameter{ + "numbers": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + }, + }, + "numbers_count": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + "run_local_at_count": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + Behavior: &core.Parameter_Default{ + Default: &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 10, + }, + }, + }, + }, + }, + }, + }, + }, + } + launchPlan1 := &admin.LaunchPlan{ + Id: &core.Identifier{ + Name: "core.advanced.run_merge_sort.merge_sort", + Version: "v3", + }, + Spec: &admin.LaunchPlanSpec{ + DefaultInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + Closure: &admin.LaunchPlanClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 0, Nanos: 0}, + ExpectedInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + } + objectGetRequest := &admin.ObjectGetRequest{ + Id: &core.Identifier{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + Name: "core.advanced.run_merge_sort.merge_sort", + Version: "v3", + }, + } + mockClient.OnGetLaunchPlanMatch(ctx, objectGetRequest).Return(launchPlan1, nil) +} +func TestCreateLaunchPlanExecutionFunc(t *testing.T) { + setup() + createExecutionSetup() + executionCreateResponseLP := &admin.ExecutionCreateResponse{ + Id: &core.WorkflowExecutionIdentifier{ + Project: "flytesnacks", + Domain: "development", + Name: "f652ea3596e7f4d80a0e", + }, + } + mockClient.OnCreateExecutionMatch(ctx, mock.Anything).Return(executionCreateResponseLP, nil) + executionConfig.ExecFile = testDataFolder + "launchplan_execution_spec.yaml" + err = createExecutionCommand(ctx, args, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "CreateExecution", ctx, mock.Anything) + tearDownAndVerify(t, `execution identifier project:"flytesnacks" domain:"development" name:"f652ea3596e7f4d80a0e"`) +} + +func TestCreateTaskExecutionFunc(t *testing.T) { + setup() + createExecutionSetup() + executionCreateResponseTask := &admin.ExecutionCreateResponse{ + Id: &core.WorkflowExecutionIdentifier{ + Project: "flytesnacks", + Domain: "development", + Name: "ff513c0e44b5b4a35aa5", + }, + } + mockClient.OnCreateExecutionMatch(ctx, mock.Anything).Return(executionCreateResponseTask, nil) + executionConfig.ExecFile = testDataFolder + "task_execution_spec.yaml" + err = createExecutionCommand(ctx, args, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "CreateExecution", ctx, mock.Anything) + tearDownAndVerify(t, `execution identifier project:"flytesnacks" domain:"development" name:"ff513c0e44b5b4a35aa5" `) +} diff --git a/flytectl/cmd/create/execution_util.go b/flytectl/cmd/create/execution_util.go new file mode 100644 index 0000000000..acb3f595d8 --- /dev/null +++ b/flytectl/cmd/create/execution_util.go @@ -0,0 +1,148 @@ +package create + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "strings" + + cmdCore "github.com/flyteorg/flytectl/cmd/core" + cmdGet "github.com/flyteorg/flytectl/cmd/get" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/google/uuid" + "sigs.k8s.io/yaml" +) + +func createExecutionRequestForWorkflow(ctx context.Context, workflowName string, project string, domain string, cmdCtx cmdCore.CommandContext) (*admin.ExecutionCreateRequest, error) { + var lp *admin.LaunchPlan + var err error + // Fetch the launch plan + if lp, err = cmdGet.FetchLPVersion(ctx, workflowName, executionConfig.Version, project, domain, cmdCtx); err != nil { + return nil, err + } + // Create workflow params literal map + var paramLiterals map[string]*core.Literal + workflowParams := cmdGet.WorkflowParams(lp) + if paramLiterals, err = MakeLiteralForParams(executionConfig.Inputs, workflowParams); err != nil { + return nil, err + } + var inputs = &core.LiteralMap{ + Literals: paramLiterals, + } + ID := lp.Id + return createExecutionRequest(ID, inputs, nil), nil +} + +func createExecutionRequestForTask(ctx context.Context, taskName string, project string, domain string, cmdCtx cmdCore.CommandContext) (*admin.ExecutionCreateRequest, error) { + var task *admin.Task + var err error + // Fetch the task + if task, err = cmdGet.FetchTaskVersion(ctx, taskName, executionConfig.Version, project, domain, cmdCtx); err != nil { + return nil, err + } + // Create task variables literal map + var variableLiterals map[string]*core.Literal + taskInputs := cmdGet.TaskInputs(task) + if variableLiterals, err = MakeLiteralForVariables(executionConfig.Inputs, taskInputs); err != nil { + return nil, err + } + var inputs = &core.LiteralMap{ + Literals: variableLiterals, + } + var authRole *admin.AuthRole + if executionConfig.KubeServiceAcct != "" { + authRole = &admin.AuthRole{Method: &admin.AuthRole_KubernetesServiceAccount{ + KubernetesServiceAccount: executionConfig.KubeServiceAcct}} + } else { + authRole = &admin.AuthRole{Method: &admin.AuthRole_AssumableIamRole{ + AssumableIamRole: executionConfig.IamRoleARN}} + } + ID := &core.Identifier{ + ResourceType: core.ResourceType_TASK, + Project: project, + Domain: domain, + Name: task.Id.Name, + Version: task.Id.Version, + } + return createExecutionRequest(ID, inputs, authRole), nil +} + +func createExecutionRequest(ID *core.Identifier, inputs *core.LiteralMap, authRole *admin.AuthRole) *admin.ExecutionCreateRequest { + return &admin.ExecutionCreateRequest{ + Project: executionConfig.TargetProject, + Domain: executionConfig.TargetDomain, + Name: "f" + strings.ReplaceAll(uuid.New().String(), "-", "")[:19], + Spec: &admin.ExecutionSpec{ + LaunchPlan: ID, + Metadata: &admin.ExecutionMetadata{ + Mode: admin.ExecutionMetadata_MANUAL, + Principal: "sdk", + Nesting: 0, + }, + AuthRole: authRole, + }, + Inputs: inputs, + } +} + +func readExecConfigFromFile(fileName string) (*ExecutionConfig, error) { + data, _err := ioutil.ReadFile(fileName) + if _err != nil { + return nil, fmt.Errorf("unable to read from %v yaml file", fileName) + } + executionConfigRead := ExecutionConfig{} + if _err = yaml.Unmarshal(data, &executionConfigRead); _err != nil { + return nil, _err + } + return &executionConfigRead, nil +} + +func resolveOverrides(readExecutionConfig *ExecutionConfig, project string, domain string) { + if executionConfig.KubeServiceAcct != "" { + readExecutionConfig.KubeServiceAcct = executionConfig.KubeServiceAcct + } + if executionConfig.IamRoleARN != "" { + readExecutionConfig.IamRoleARN = executionConfig.IamRoleARN + } + if executionConfig.TargetProject != "" { + readExecutionConfig.TargetProject = executionConfig.TargetProject + } + if executionConfig.TargetDomain != "" { + readExecutionConfig.TargetDomain = executionConfig.TargetDomain + } + // Use the root project and domain to launch the task/workflow if target is unspecified + if executionConfig.TargetProject == "" { + readExecutionConfig.TargetProject = project + } + if executionConfig.TargetDomain == "" { + readExecutionConfig.TargetDomain = domain + } +} + +func readConfigAndValidate(project string, domain string) (ExecutionParams, error) { + executionParams := ExecutionParams{} + if executionConfig.ExecFile == "" { + return executionParams, errors.New("executionConfig can't be empty. Run the flytectl get task/launchplan to generate the config") + } + var readExecutionConfig *ExecutionConfig + var err error + if readExecutionConfig, err = readExecConfigFromFile(executionConfig.ExecFile); err != nil { + return executionParams, err + } + resolveOverrides(readExecutionConfig, project, domain) + // Update executionConfig pointer to readExecutionConfig as it contains all the updates. + executionConfig = readExecutionConfig + isTask := readExecutionConfig.Task != "" + isWorkflow := readExecutionConfig.Workflow != "" + if isTask == isWorkflow { + return executionParams, errors.New("either one of task or workflow name should be specified to launch an execution") + } + name := readExecutionConfig.Task + if !isTask { + name = readExecutionConfig.Workflow + } + return ExecutionParams{name: name, isTask: isTask}, nil +} diff --git a/flytectl/cmd/create/executionconfig_flags.go b/flytectl/cmd/create/executionconfig_flags.go new file mode 100755 index 0000000000..4a9774dff0 --- /dev/null +++ b/flytectl/cmd/create/executionconfig_flags.go @@ -0,0 +1,50 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package create + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (ExecutionConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (ExecutionConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in ExecutionConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg ExecutionConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("ExecutionConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(executionConfig.ExecFile),fmt.Sprintf("%v%v", prefix, "execFile"), executionConfig.ExecFile, "file for the execution params.If not specified defaults to <_name>.execution_spec.yaml") + cmdFlags.StringVar(&(executionConfig.TargetDomain),fmt.Sprintf("%v%v", prefix, "targetDomain"), executionConfig.TargetDomain, "project where execution needs to be created.If not specified configured domain would be used.") + cmdFlags.StringVar(&(executionConfig.TargetProject),fmt.Sprintf("%v%v", prefix, "targetProject"), executionConfig.TargetProject, "project where execution needs to be created.If not specified configured project would be used.") + cmdFlags.StringVar(&(executionConfig.KubeServiceAcct),fmt.Sprintf("%v%v", prefix, "kubeServiceAcct"), executionConfig.KubeServiceAcct, "kubernetes service account AuthRole for launching execution.") + cmdFlags.StringVar(&(executionConfig.IamRoleARN),fmt.Sprintf("%v%v", prefix, "iamRoleARN"), executionConfig.IamRoleARN, "iam role ARN AuthRole for launching execution.") + return cmdFlags +} diff --git a/flytectl/cmd/create/executionconfig_flags_test.go b/flytectl/cmd/create/executionconfig_flags_test.go new file mode 100755 index 0000000000..bf7ab9c47b --- /dev/null +++ b/flytectl/cmd/create/executionconfig_flags_test.go @@ -0,0 +1,212 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package create + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsExecutionConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementExecutionConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsExecutionConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookExecutionConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementExecutionConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_ExecutionConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookExecutionConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_ExecutionConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_ExecutionConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_ExecutionConfig(val, result)) +} + +func testDecodeSlice_ExecutionConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_ExecutionConfig(vStringSlice, result)) +} + +func TestExecutionConfig_GetPFlagSet(t *testing.T) { + val := ExecutionConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestExecutionConfig_SetFlags(t *testing.T) { + actual := ExecutionConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_execFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("execFile"); err == nil { + assert.Equal(t, string(executionConfig.ExecFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("execFile", testValue) + if vString, err := cmdFlags.GetString("execFile"); err == nil { + testDecodeJson_ExecutionConfig(t, fmt.Sprintf("%v", vString), &actual.ExecFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_targetDomain", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("targetDomain"); err == nil { + assert.Equal(t, string(executionConfig.TargetDomain), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("targetDomain", testValue) + if vString, err := cmdFlags.GetString("targetDomain"); err == nil { + testDecodeJson_ExecutionConfig(t, fmt.Sprintf("%v", vString), &actual.TargetDomain) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_targetProject", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("targetProject"); err == nil { + assert.Equal(t, string(executionConfig.TargetProject), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("targetProject", testValue) + if vString, err := cmdFlags.GetString("targetProject"); err == nil { + testDecodeJson_ExecutionConfig(t, fmt.Sprintf("%v", vString), &actual.TargetProject) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_kubeServiceAcct", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("kubeServiceAcct"); err == nil { + assert.Equal(t, string(executionConfig.KubeServiceAcct), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("kubeServiceAcct", testValue) + if vString, err := cmdFlags.GetString("kubeServiceAcct"); err == nil { + testDecodeJson_ExecutionConfig(t, fmt.Sprintf("%v", vString), &actual.KubeServiceAcct) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_iamRoleARN", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("iamRoleARN"); err == nil { + assert.Equal(t, string(executionConfig.IamRoleARN), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("iamRoleARN", testValue) + if vString, err := cmdFlags.GetString("iamRoleARN"); err == nil { + testDecodeJson_ExecutionConfig(t, fmt.Sprintf("%v", vString), &actual.IamRoleARN) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/flytectl/cmd/create/project_test.go b/flytectl/cmd/create/project_test.go index 77c3cb7578..8f4bb309b0 100644 --- a/flytectl/cmd/create/project_test.go +++ b/flytectl/cmd/create/project_test.go @@ -1,48 +1,26 @@ package create import ( - "bytes" - "context" - "io" - "log" - "os" + "fmt" "testing" - cmdCore "github.com/flyteorg/flytectl/cmd/core" - "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flytectl/cmd/testutils" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) const projectValue = "dummyProject" var ( - reader *os.File - writer *os.File - err error - ctx context.Context - mockClient *mocks.AdminServiceClient - mockOutStream io.Writer - args []string - cmdCtx cmdCore.CommandContext projectRegisterRequest *admin.ProjectRegisterRequest - stdOut *os.File - stderr *os.File ) -func setup() { - reader, writer, err = os.Pipe() - if err != nil { - panic(err) - } - stdOut = os.Stdout - stderr = os.Stderr - os.Stdout = writer - os.Stderr = writer - log.SetOutput(writer) - mockClient = new(mocks.AdminServiceClient) - mockOutStream = writer - cmdCtx = cmdCore.NewCommandContext(mockClient, mockOutStream) +func createProjectSetup() { + ctx = testutils.Ctx + cmdCtx = testutils.CmdCtx + mockClient = testutils.MockClient projectRegisterRequest = &admin.ProjectRegisterRequest{ Project: &admin.Project{ Id: projectValue, @@ -53,21 +31,15 @@ func setup() { }, }, } + projectConfig.ID = "" + projectConfig.Name = "" + projectConfig.Labels = map[string]string{} + projectConfig.Description = "" } - -func teardownAndVerify(t *testing.T, expectedLog string) { - writer.Close() - os.Stdout = stdOut - os.Stderr = stderr - var buf bytes.Buffer - if _, err := io.Copy(&buf, reader); err != nil { - assert.Equal(t, expectedLog, buf.String()) - } -} - func TestCreateProjectFunc(t *testing.T) { setup() - defer teardownAndVerify(t, "project Created successfully") + createProjectSetup() + defer tearDownAndVerify(t, "project Created successfully") projectConfig.ID = projectValue projectConfig.Name = projectValue projectConfig.Labels = map[string]string{} @@ -80,23 +52,25 @@ func TestCreateProjectFunc(t *testing.T) { func TestEmptyProjectID(t *testing.T) { setup() - defer teardownAndVerify(t, "project ID is required flag") + createProjectSetup() + defer tearDownAndVerify(t, "") projectConfig.Name = projectValue projectConfig.Labels = map[string]string{} mockClient.OnRegisterProjectMatch(ctx, projectRegisterRequest).Return(nil, nil) err := createProjectsCommand(ctx, args, cmdCtx) - assert.Nil(t, err) - mockClient.AssertCalled(t, "RegisterProject", ctx, projectRegisterRequest) + assert.Equal(t, fmt.Errorf("project ID is required flag"), err) + mockClient.AssertNotCalled(t, "RegisterProject", ctx, mock.Anything) } func TestEmptyProjectName(t *testing.T) { setup() - defer teardownAndVerify(t, "project ID is required flag") + createProjectSetup() + defer tearDownAndVerify(t, "") projectConfig.ID = projectValue projectConfig.Labels = map[string]string{} projectConfig.Description = "" mockClient.OnRegisterProjectMatch(ctx, projectRegisterRequest).Return(nil, nil) err := createProjectsCommand(ctx, args, cmdCtx) - assert.Nil(t, err) - mockClient.AssertCalled(t, "RegisterProject", ctx, projectRegisterRequest) + assert.Equal(t, fmt.Errorf("project name is required flag"), err) + mockClient.AssertNotCalled(t, "RegisterProject", ctx, mock.Anything) } diff --git a/flytectl/cmd/create/serialization_utils.go b/flytectl/cmd/create/serialization_utils.go new file mode 100644 index 0000000000..b9759ca7c1 --- /dev/null +++ b/flytectl/cmd/create/serialization_utils.go @@ -0,0 +1,29 @@ +package create + +import ( + "github.com/flyteorg/flyteidl/clients/go/coreutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" +) + +// TODO: Move all functions to flyteidl +func MakeLiteralForVariables(serialize map[string]interface{}, variables map[string]*core.Variable) (map[string]*core.Literal, error) { + result := make(map[string]*core.Literal) + var err error + for k, v := range variables { + if result[k], err = coreutils.MakeLiteralForType(v.Type, serialize[k]); err != nil { + return nil, err + } + } + return result, nil +} + +func MakeLiteralForParams(serialize map[string]interface{}, parameters map[string]*core.Parameter) (map[string]*core.Literal, error) { + result := make(map[string]*core.Literal) + var err error + for k, v := range parameters { + if result[k], err = coreutils.MakeLiteralForType(v.GetVar().Type, serialize[k]); err != nil { + return nil, err + } + } + return result, nil +} diff --git a/flytectl/cmd/get/execution.go b/flytectl/cmd/get/execution.go index 8cde1b7b57..f7d91812c1 100644 --- a/flytectl/cmd/get/execution.go +++ b/flytectl/cmd/get/execution.go @@ -50,8 +50,8 @@ Usage var executionColumns = []printer.Column{ {Header: "Name", JSONPath: "$.id.name"}, - {Header: "Workflow Name", JSONPath: "$.closure.workflowId.name"}, - {Header: "Type", JSONPath: "$.closure.workflowId.resourceType"}, + {Header: "Launch Plan Name", JSONPath: "$.spec.launchPlan.name"}, + {Header: "Type", JSONPath: "$.spec.launchPlan.resourceType"}, {Header: "Phase", JSONPath: "$.closure.phase"}, {Header: "Started", JSONPath: "$.closure.startedAt"}, {Header: "Elapsed Time", JSONPath: "$.closure.duration"}, diff --git a/flytectl/cmd/get/execution_test.go b/flytectl/cmd/get/execution_test.go index d414f32145..5940c3d17a 100644 --- a/flytectl/cmd/get/execution_test.go +++ b/flytectl/cmd/get/execution_test.go @@ -14,15 +14,6 @@ import ( "github.com/stretchr/testify/assert" ) -const projectValue = "dummyProject" -const domainValue = "dummyDomain" -const executionNameValue = "e124" -const launchPlanNameValue = "lp_name" -const launchPlanVersionValue = "lp_version" -const workflowNameValue = "wf_name" -const workflowVersionValue = "wf_version" -const output = "json" - func TestListExecutionFunc(t *testing.T) { ctx := context.Background() config.GetConfig().Project = projectValue diff --git a/flytectl/cmd/get/execution_util.go b/flytectl/cmd/get/execution_util.go new file mode 100644 index 0000000000..da330af39e --- /dev/null +++ b/flytectl/cmd/get/execution_util.go @@ -0,0 +1,128 @@ +package get + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + + cmdUtil "github.com/flyteorg/flytectl/pkg/commandutils" + "github.com/flyteorg/flyteidl/clients/go/coreutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "sigs.k8s.io/yaml" +) + +// ExecutionConfig is duplicated struct from create with the same structure. This is to avoid the circular dependency. +// TODO : replace this with a cleaner design +type ExecutionConfig struct { + TargetDomain string `json:"targetDomain"` + TargetProject string `json:"targetProject"` + KubeServiceAcct string `json:"kubeServiceAcct"` + IamRoleARN string `json:"iamRoleARN"` + Workflow string `json:"workflow,omitempty"` + Task string `json:"task,omitempty"` + Version string `json:"version"` + Inputs map[string]interface{} `json:"inputs"` +} + +func WriteExecConfigToFile(executionConfig ExecutionConfig, fileName string) error { + d, err := yaml.Marshal(executionConfig) + if err != nil { + fmt.Printf("error: %v", err) + } + if _, err = os.Stat(fileName); err == nil { + if !cmdUtil.AskForConfirmation(fmt.Sprintf("warning file %v will be overwritten", fileName)) { + return errors.New("backup the file before continuing") + } + } + return ioutil.WriteFile(fileName, d, 0600) +} + +func CreateAndWriteExecConfigForTask(task *admin.Task, fileName string) error { + var err error + executionConfig := ExecutionConfig{Task: task.Id.Name, Version: task.Id.Version} + if executionConfig.Inputs, err = ParamMapForTask(task); err != nil { + return err + } + return WriteExecConfigToFile(executionConfig, fileName) +} + +func CreateAndWriteExecConfigForWorkflow(wlp *admin.LaunchPlan, fileName string) error { + var err error + executionConfig := ExecutionConfig{Workflow: wlp.Id.Name, Version: wlp.Id.Version} + if executionConfig.Inputs, err = ParamMapForWorkflow(wlp); err != nil { + return err + } + return WriteExecConfigToFile(executionConfig, fileName) +} + +func TaskInputs(task *admin.Task) map[string]*core.Variable { + taskInputs := map[string]*core.Variable{} + if task == nil || task.Closure == nil { + return taskInputs + } + if task.Closure.CompiledTask == nil { + return taskInputs + } + if task.Closure.CompiledTask.Template == nil { + return taskInputs + } + if task.Closure.CompiledTask.Template.Interface == nil { + return taskInputs + } + if task.Closure.CompiledTask.Template.Interface.Inputs == nil { + return taskInputs + } + return task.Closure.CompiledTask.Template.Interface.Inputs.Variables +} + +func ParamMapForTask(task *admin.Task) (map[string]interface{}, error) { + taskInputs := TaskInputs(task) + paramMap := make(map[string]interface{}, len(taskInputs)) + for k, v := range taskInputs { + varTypeValue, err := coreutils.MakeDefaultLiteralForType(v.Type) + if err != nil { + fmt.Println("error creating default value for literal type ", v.Type) + return nil, err + } + if paramMap[k], err = coreutils.ExtractFromLiteral(varTypeValue); err != nil { + return nil, err + } + } + return paramMap, nil +} + +func WorkflowParams(lp *admin.LaunchPlan) map[string]*core.Parameter { + workflowParams := map[string]*core.Parameter{} + if lp == nil || lp.Spec == nil { + return workflowParams + } + if lp.Spec.DefaultInputs == nil { + return workflowParams + } + return lp.Spec.DefaultInputs.Parameters +} + +func ParamMapForWorkflow(lp *admin.LaunchPlan) (map[string]interface{}, error) { + workflowParams := WorkflowParams(lp) + paramMap := make(map[string]interface{}, len(workflowParams)) + for k, v := range workflowParams { + varTypeValue, err := coreutils.MakeDefaultLiteralForType(v.Var.Type) + if err != nil { + fmt.Println("error creating default value for literal type ", v.Var.Type) + return nil, err + } + if paramMap[k], err = coreutils.ExtractFromLiteral(varTypeValue); err != nil { + return nil, err + } + // Override if there is a default value + if paramsDefault, ok := v.Behavior.(*core.Parameter_Default); ok { + if paramMap[k], err = coreutils.ExtractFromLiteral(paramsDefault.Default); err != nil { + return nil, err + } + } + } + return paramMap, nil +} diff --git a/flytectl/cmd/get/get.go b/flytectl/cmd/get/get.go index 8f8cc9c25d..e875a0ca01 100644 --- a/flytectl/cmd/get/get.go +++ b/flytectl/cmd/get/get.go @@ -30,11 +30,11 @@ func CreateGetCommand() *cobra.Command { Short: projectShort, Long: projectLong}, "task": {CmdFunc: getTaskFunc, Aliases: []string{"tasks"}, Short: taskShort, - Long: taskLong}, + Long: taskLong, PFlagProvider: taskConfig}, "workflow": {CmdFunc: getWorkflowFunc, Aliases: []string{"workflows"}, Short: workflowShort, Long: workflowLong}, "launchplan": {CmdFunc: getLaunchPlanFunc, Aliases: []string{"launchplans"}, Short: launchPlanShort, - Long: launchPlanLong}, + Long: launchPlanLong, PFlagProvider: launchPlanConfig}, "execution": {CmdFunc: getExecutionFunc, Aliases: []string{"executions"}, Short: executionShort, Long: executionLong}, } diff --git a/flytectl/cmd/get/get_test.go b/flytectl/cmd/get/get_test.go index 243a6a97da..852994db00 100644 --- a/flytectl/cmd/get/get_test.go +++ b/flytectl/cmd/get/get_test.go @@ -1,13 +1,37 @@ package get import ( + "context" "fmt" "sort" "testing" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/stretchr/testify/assert" ) +const projectValue = "dummyProject" +const domainValue = "dummyDomain" +const output = "json" +const executionNameValue = "e124" +const launchPlanNameValue = "lp_name" +const launchPlanVersionValue = "lp_version" +const workflowNameValue = "wf_name" +const workflowVersionValue = "wf_version" +const testDataFolder = "../testdata/" + +var ( + err error + ctx context.Context + mockClient *mocks.AdminServiceClient + cmdCtx cmdCore.CommandContext +) +var setup = testutils.Setup +var tearDownAndVerify = testutils.TearDownAndVerify + func TestCreateGetCommand(t *testing.T) { getCommand := CreateGetCommand() assert.Equal(t, getCommand.Use, "get") diff --git a/flytectl/cmd/get/launch_plan.go b/flytectl/cmd/get/launch_plan.go index 6d40cec4d4..2b6e8a7fe2 100644 --- a/flytectl/cmd/get/launch_plan.go +++ b/flytectl/cmd/get/launch_plan.go @@ -9,6 +9,7 @@ import ( "github.com/flyteorg/flytectl/pkg/printer" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flytestdlib/logger" + "github.com/golang/protobuf/proto" ) @@ -18,13 +19,13 @@ const ( Retrieves all the launch plans within project and domain.(launchplan,launchplans can be used interchangeably in these commands) :: - bin/flytectl get launchplan -p flytesnacks -d development + flytectl get launchplan -p flytesnacks -d development Retrieves launch plan by name within project and domain. :: - bin/flytectl get launchplan -p flytesnacks -d development core.basic.lp.go_greet + flytectl get launchplan -p flytesnacks -d development core.basic.lp.go_greet Retrieves launchplan by filters. :: @@ -35,18 +36,54 @@ Retrieves all the launchplan within project and domain in yaml format. :: - bin/flytectl get launchplan -p flytesnacks -d development -o yaml + flytectl get launchplan -p flytesnacks -d development -o yaml Retrieves all the launchplan within project and domain in json format :: - bin/flytectl get launchplan -p flytesnacks -d development -o json + flytectl get launchplan -p flytesnacks -d development -o json + +Retrieves a launch plans within project and domain for a version and generate the execution spec file for it to be used for launching the execution using create execution. + +:: + + flytectl get launchplan -d development -p flytectldemo core.advanced.run_merge_sort.merge_sort --execFile execution_spec.yam + +The generated file would look similar to this + +.. code-block:: yaml + + iamRoleARN: "" + inputs: + numbers: + - 0 + numbers_count: 0 + run_local_at_count: 10 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + version: v3 + workflow: core.advanced.run_merge_sort.merge_sort + +Check the create execution section on how to launch one using the generated file. Usage ` ) +//go:generate pflags LaunchPlanConfig --default-var launchPlanConfig +var ( + launchPlanConfig = &LaunchPlanConfig{} +) + +// LaunchPlanConfig +type LaunchPlanConfig struct { + ExecFile string `json:"execFile" pflag:",execution file name to be used for generating execution spec of a single launchplan."` + Version string `json:"version" pflag:",version of the launchplan to be fetched."` + Latest bool `json:"latest" pflag:", flag to indicate to fetch the latest version, version flag will be ignored in this case"` +} + var launchplanColumns = []printer.Column{ {Header: "Version", JSONPath: "$.id.version"}, {Header: "Name", JSONPath: "$.id.name"}, @@ -65,29 +102,24 @@ func LaunchplanToProtoMessages(l []*admin.LaunchPlan) []proto.Message { func getLaunchPlanFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { launchPlanPrinter := printer.Printer{} - + project := config.GetConfig().Project + domain := config.GetConfig().Domain if len(args) == 1 { name := args[0] - launchPlan, err := cmdCtx.AdminClient().ListLaunchPlans(ctx, &admin.ResourceListRequest{ - Limit: 10, - Id: &admin.NamedEntityIdentifier{ - Project: config.GetConfig().Project, - Domain: config.GetConfig().Domain, - Name: name, - }, - }) - if err != nil { + var launchPlans []*admin.LaunchPlan + var err error + if launchPlans, err = FetchLPForName(ctx, name, project, domain, cmdCtx); err != nil { return err } - logger.Debugf(ctx, "Retrieved %v excutions", len(launchPlan.LaunchPlans)) - err = launchPlanPrinter.Print(config.GetConfig().MustOutputFormat(), launchplanColumns, LaunchplanToProtoMessages(launchPlan.LaunchPlans)...) + logger.Debugf(ctx, "Retrieved %v launch plans", len(launchPlans)) + err = launchPlanPrinter.Print(config.GetConfig().MustOutputFormat(), launchplanColumns, LaunchplanToProtoMessages(launchPlans)...) if err != nil { return err } return nil } - launchPlans, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListLaunchPlanIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain}) + launchPlans, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListLaunchPlanIds, adminutils.ListRequest{Project: project, Domain: domain}) if err != nil { return err } diff --git a/flytectl/cmd/get/launch_plan_test.go b/flytectl/cmd/get/launch_plan_test.go new file mode 100644 index 0000000000..54f7f14202 --- /dev/null +++ b/flytectl/cmd/get/launch_plan_test.go @@ -0,0 +1,628 @@ +package get + +import ( + "os" + "testing" + + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + resourceListRequest *admin.ResourceListRequest + objectGetRequest *admin.ObjectGetRequest + namedIDRequest *admin.NamedEntityIdentifierListRequest + launchPlanListResponse *admin.LaunchPlanList + argsLp []string +) + +func getLaunchPlanSetup() { + ctx = testutils.Ctx + cmdCtx = testutils.CmdCtx + mockClient = testutils.MockClient + argsLp = []string{"launchplan1"} + parameterMap := map[string]*core.Parameter{ + "numbers": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + }, + }, + "numbers_count": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + "run_local_at_count": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + Behavior: &core.Parameter_Default{ + Default: &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 10, + }, + }, + }, + }, + }, + }, + }, + }, + } + launchPlan1 := &admin.LaunchPlan{ + Id: &core.Identifier{ + Name: "launchplan1", + Version: "v1", + }, + Spec: &admin.LaunchPlanSpec{ + DefaultInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + Closure: &admin.LaunchPlanClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 0, Nanos: 0}, + ExpectedInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + } + launchPlan2 := &admin.LaunchPlan{ + Id: &core.Identifier{ + Name: "launchplan1", + Version: "v2", + }, + Spec: &admin.LaunchPlanSpec{ + DefaultInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + Closure: &admin.LaunchPlanClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + ExpectedInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + } + + launchPlans := []*admin.LaunchPlan{launchPlan2, launchPlan1} + + resourceListRequest = &admin.ResourceListRequest{ + Id: &admin.NamedEntityIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: argsLp[0], + }, + SortBy: &admin.Sort{ + Key: "created_at", + Direction: admin.Sort_DESCENDING, + }, + Limit: 100, + } + + launchPlanListResponse = &admin.LaunchPlanList{ + LaunchPlans: launchPlans, + } + + objectGetRequest = &admin.ObjectGetRequest{ + Id: &core.Identifier{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Project: projectValue, + Domain: domainValue, + Name: argsLp[0], + Version: "v2", + }, + } + + namedIDRequest = &admin.NamedEntityIdentifierListRequest{ + Project: projectValue, + Domain: domainValue, + SortBy: &admin.Sort{ + Key: "name", + Direction: admin.Sort_ASCENDING, + }, + Limit: 100, + } + + var entities []*admin.NamedEntityIdentifier + id1 := &admin.NamedEntityIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: "launchplan1", + } + id2 := &admin.NamedEntityIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: "launchplan2", + } + entities = append(entities, id1, id2) + namedIdentifierList := &admin.NamedEntityIdentifierList{ + Entities: entities, + } + + mockClient.OnListLaunchPlansMatch(ctx, resourceListRequest).Return(launchPlanListResponse, nil) + mockClient.OnGetLaunchPlanMatch(ctx, objectGetRequest).Return(launchPlan2, nil) + mockClient.OnListLaunchPlanIdsMatch(ctx, namedIDRequest).Return(namedIdentifierList, nil) + + launchPlanConfig.Latest = false + launchPlanConfig.Version = "" + launchPlanConfig.ExecFile = "" +} + +func TestGetLaunchPlanFunc(t *testing.T) { + setup() + getLaunchPlanSetup() + err = getLaunchPlanFunc(ctx, argsLp, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "ListLaunchPlans", ctx, resourceListRequest) + tearDownAndVerify(t, `[ + { + "id": { + "name": "launchplan1", + "version": "v2" + }, + "spec": { + "defaultInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + } + }, + "closure": { + "expectedInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } + }, + { + "id": { + "name": "launchplan1", + "version": "v1" + }, + "spec": { + "defaultInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + } + }, + "closure": { + "expectedInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:00Z" + } + } +]`) +} + +func TestGetLaunchPlanFuncLatest(t *testing.T) { + setup() + getLaunchPlanSetup() + launchPlanConfig.Latest = true + err = getLaunchPlanFunc(ctx, argsLp, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "ListLaunchPlans", ctx, resourceListRequest) + tearDownAndVerify(t, `{ + "id": { + "name": "launchplan1", + "version": "v2" + }, + "spec": { + "defaultInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + } + }, + "closure": { + "expectedInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } +}`) +} + +func TestGetLaunchPlanWithVersion(t *testing.T) { + setup() + getLaunchPlanSetup() + launchPlanConfig.Version = "v2" + err = getLaunchPlanFunc(ctx, argsLp, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "GetLaunchPlan", ctx, objectGetRequest) + tearDownAndVerify(t, `{ + "id": { + "name": "launchplan1", + "version": "v2" + }, + "spec": { + "defaultInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + } + }, + "closure": { + "expectedInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } +}`) +} + +func TestGetLaunchPlans(t *testing.T) { + setup() + getLaunchPlanSetup() + argsLp = []string{} + err = getLaunchPlanFunc(ctx, argsLp, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "ListLaunchPlanIds", ctx, namedIDRequest) + tearDownAndVerify(t, `[ + { + "project": "dummyProject", + "domain": "dummyDomain", + "name": "launchplan1" + }, + { + "project": "dummyProject", + "domain": "dummyDomain", + "name": "launchplan2" + } +]`) +} + +func TestGetLaunchPlansWithExecFile(t *testing.T) { + setup() + getLaunchPlanSetup() + launchPlanConfig.Version = "v2" + launchPlanConfig.ExecFile = testDataFolder + "exec_file" + err = getLaunchPlanFunc(ctx, argsLp, cmdCtx) + os.Remove(launchPlanConfig.ExecFile) + assert.Nil(t, err) + mockClient.AssertCalled(t, "GetLaunchPlan", ctx, objectGetRequest) + tearDownAndVerify(t, `{ + "id": { + "name": "launchplan1", + "version": "v2" + }, + "spec": { + "defaultInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + } + }, + "closure": { + "expectedInputs": { + "parameters": { + "numbers": { + "var": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + }, + "numbers_count": { + "var": { + "type": { + "simple": "INTEGER" + } + } + }, + "run_local_at_count": { + "var": { + "type": { + "simple": "INTEGER" + } + }, + "default": { + "scalar": { + "primitive": { + "integer": "10" + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } +}`) +} diff --git a/flytectl/cmd/get/launch_plan_util.go b/flytectl/cmd/get/launch_plan_util.go new file mode 100644 index 0000000000..e72c6e9094 --- /dev/null +++ b/flytectl/cmd/get/launch_plan_util.go @@ -0,0 +1,90 @@ +package get + +import ( + "context" + "fmt" + + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" +) + +// Reads the launchplan config to drive fetching the correct launch plans. +func FetchLPForName(ctx context.Context, name string, project string, domain string, cmdCtx cmdCore.CommandContext) ([]*admin.LaunchPlan, error) { + var launchPlans []*admin.LaunchPlan + var lp *admin.LaunchPlan + var err error + if launchPlanConfig.Latest { + if lp, err = FetchLPLatestVersion(ctx, name, project, domain, cmdCtx); err != nil { + return nil, err + } + launchPlans = append(launchPlans, lp) + } else if launchPlanConfig.Version != "" { + if lp, err = FetchLPVersion(ctx, name, launchPlanConfig.Version, project, domain, cmdCtx); err != nil { + return nil, err + } + launchPlans = append(launchPlans, lp) + } else { + launchPlans, err = FetchAllVerOfLP(ctx, name, project, domain, cmdCtx) + if err != nil { + return nil, err + } + } + if launchPlanConfig.ExecFile != "" { + // There would be atleast one launchplan object when code reaches here and hence the length assertion is not required. + lp = launchPlans[0] + // Only write the first task from the tasks object. + if err = CreateAndWriteExecConfigForWorkflow(lp, launchPlanConfig.ExecFile); err != nil { + return nil, err + } + } + return launchPlans, nil +} + +func FetchAllVerOfLP(ctx context.Context, lpName string, project string, domain string, cmdCtx cmdCore.CommandContext) ([]*admin.LaunchPlan, error) { + tList, err := cmdCtx.AdminClient().ListLaunchPlans(ctx, &admin.ResourceListRequest{ + Id: &admin.NamedEntityIdentifier{ + Project: project, + Domain: domain, + Name: lpName, + }, + SortBy: &admin.Sort{ + Key: "created_at", + Direction: admin.Sort_DESCENDING, + }, + Limit: 100, + }) + if err != nil { + return nil, err + } + if len(tList.LaunchPlans) == 0 { + return nil, fmt.Errorf("no launchplans retrieved for %v", lpName) + } + return tList.LaunchPlans, nil +} + +func FetchLPLatestVersion(ctx context.Context, name string, project string, domain string, cmdCtx cmdCore.CommandContext) (*admin.LaunchPlan, error) { + // Fetch the latest version of the task. + lpVersions, err := FetchAllVerOfLP(ctx, name, project, domain, cmdCtx) + if err != nil { + return nil, err + } + lp := lpVersions[0] + return lp, nil +} + +func FetchLPVersion(ctx context.Context, name string, version string, project string, domain string, cmdCtx cmdCore.CommandContext) (*admin.LaunchPlan, error) { + lp, err := cmdCtx.AdminClient().GetLaunchPlan(ctx, &admin.ObjectGetRequest{ + Id: &core.Identifier{ + ResourceType: core.ResourceType_LAUNCH_PLAN, + Project: project, + Domain: domain, + Name: name, + Version: version, + }, + }) + if err != nil { + return nil, err + } + return lp, nil +} diff --git a/flytectl/cmd/get/launchplanconfig_flags.go b/flytectl/cmd/get/launchplanconfig_flags.go new file mode 100755 index 0000000000..00becdd073 --- /dev/null +++ b/flytectl/cmd/get/launchplanconfig_flags.go @@ -0,0 +1,48 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package get + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (LaunchPlanConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (LaunchPlanConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in LaunchPlanConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg LaunchPlanConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("LaunchPlanConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(launchPlanConfig.ExecFile), fmt.Sprintf("%v%v", prefix, "execFile"), launchPlanConfig.ExecFile, "execution file name to be used for generating execution spec of a single launchplan.") + cmdFlags.StringVar(&(launchPlanConfig.Version), fmt.Sprintf("%v%v", prefix, "version"), launchPlanConfig.Version, "version of the launchplan to be fetched.") + cmdFlags.BoolVar(&(launchPlanConfig.Latest),fmt.Sprintf("%v%v", prefix, "latest"), launchPlanConfig.Latest, "flag to indicate to fetch the latest version, version flag will be ignored in this case") + return cmdFlags +} diff --git a/flytectl/cmd/get/launchplanconfig_flags_test.go b/flytectl/cmd/get/launchplanconfig_flags_test.go new file mode 100755 index 0000000000..2b9271b278 --- /dev/null +++ b/flytectl/cmd/get/launchplanconfig_flags_test.go @@ -0,0 +1,168 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package get + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsLaunchPlanConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementLaunchPlanConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsLaunchPlanConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookLaunchPlanConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementLaunchPlanConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_LaunchPlanConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookLaunchPlanConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_LaunchPlanConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_LaunchPlanConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_LaunchPlanConfig(val, result)) +} + +func testDecodeSlice_LaunchPlanConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_LaunchPlanConfig(vStringSlice, result)) +} + +func TestLaunchPlanConfig_GetPFlagSet(t *testing.T) { + val := LaunchPlanConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestLaunchPlanConfig_SetFlags(t *testing.T) { + actual := LaunchPlanConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_execFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("execFile"); err == nil { + assert.Equal(t, string(launchPlanConfig.ExecFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("execFile", testValue) + if vString, err := cmdFlags.GetString("execFile"); err == nil { + testDecodeJson_LaunchPlanConfig(t, fmt.Sprintf("%v", vString), &actual.ExecFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_version", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("version"); err == nil { + assert.Equal(t, string(launchPlanConfig.Version), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("version", testValue) + if vString, err := cmdFlags.GetString("version"); err == nil { + testDecodeJson_LaunchPlanConfig(t, fmt.Sprintf("%v", vString), &actual.Version) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_latest", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vBool, err := cmdFlags.GetBool("latest"); err == nil { + assert.Equal(t, bool(launchPlanConfig.Latest), vBool) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("latest", testValue) + if vBool, err := cmdFlags.GetBool("latest"); err == nil { + testDecodeJson_LaunchPlanConfig(t, fmt.Sprintf("%v", vBool), &actual.Latest) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/flytectl/cmd/get/task.go b/flytectl/cmd/get/task.go index ccbd46dd4d..9b98de40d6 100644 --- a/flytectl/cmd/get/task.go +++ b/flytectl/cmd/get/task.go @@ -3,16 +3,14 @@ package get import ( "context" - "github.com/flyteorg/flytestdlib/logger" - "github.com/golang/protobuf/proto" - - "github.com/flyteorg/flytectl/pkg/adminutils" - "github.com/flyteorg/flytectl/pkg/printer" - "github.com/flyteorg/flytectl/cmd/config" cmdCore "github.com/flyteorg/flytectl/cmd/core" - + "github.com/flyteorg/flytectl/pkg/adminutils" + "github.com/flyteorg/flytectl/pkg/printer" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flytestdlib/logger" + + "github.com/golang/protobuf/proto" ) const ( @@ -46,10 +44,46 @@ Retrieves all the tasks within project and domain in json format. bin/flytectl get task -p flytesnacks -d development -o json +Retrieves a tasks within project and domain for a version and generate the execution spec file for it to be used for launching the execution using create execution. + +:: + + bin/flytectl get tasks -d development -p flytesnacks core.advanced.run_merge_sort.merge --execFile execution_spec.yaml --version v2 + +The generated file would look similar to this + +.. code-block:: yaml + + iamRoleARN: "" + inputs: + sorted_list1: + - 0 + sorted_list2: + - 0 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + task: core.advanced.run_merge_sort.merge + version: v2 + +Check the create execution section on how to launch one using the generated file. + Usage ` ) +//go:generate pflags TaskConfig --default-var taskConfig +var ( + taskConfig = &TaskConfig{} +) + +// FilesConfig +type TaskConfig struct { + ExecFile string `json:"execFile" pflag:",execution file name to be used for generating execution spec of a single task."` + Version string `json:"version" pflag:",version of the task to be fetched."` + Latest bool `json:"latest" pflag:", flag to indicate to fetch the latest version, version flag will be ignored in this case"` +} + var taskColumns = []printer.Column{ {Header: "Version", JSONPath: "$.id.version"}, {Header: "Name", JSONPath: "$.id.name"}, @@ -68,31 +102,20 @@ func TaskToProtoMessages(l []*admin.Task) []proto.Message { } func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { - taskPrinter := printer.Printer{} - + project := config.GetConfig().Project + domain := config.GetConfig().Domain if len(args) == 1 { - task, err := cmdCtx.AdminClient().ListTasks(ctx, &admin.ResourceListRequest{ - Id: &admin.NamedEntityIdentifier{ - Project: config.GetConfig().Project, - Domain: config.GetConfig().Domain, - Name: args[0], - }, - // TODO Sorting and limits should be parameters - SortBy: &admin.Sort{ - Key: "created_at", - Direction: admin.Sort_DESCENDING, - }, - Limit: 100, - }) - if err != nil { + name := args[0] + var tasks []*admin.Task + var err error + if tasks, err = FetchTaskForName(ctx, name, project, domain, cmdCtx); err != nil { return err } - logger.Debugf(ctx, "Retrieved Task", task.Tasks) - - return taskPrinter.Print(config.GetConfig().MustOutputFormat(), taskColumns, TaskToProtoMessages(task.Tasks)...) + logger.Debugf(ctx, "Retrieved Task", tasks) + return taskPrinter.Print(config.GetConfig().MustOutputFormat(), taskColumns, TaskToProtoMessages(tasks)...) } - tasks, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListTaskIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain}) + tasks, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListTaskIds, adminutils.ListRequest{Project: project, Domain: domain}) if err != nil { return err } diff --git a/flytectl/cmd/get/task_test.go b/flytectl/cmd/get/task_test.go new file mode 100644 index 0000000000..9f525b39ba --- /dev/null +++ b/flytectl/cmd/get/task_test.go @@ -0,0 +1,368 @@ +package get + +import ( + "os" + "testing" + + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + resourceListRequestTask *admin.ResourceListRequest + objectGetRequestTask *admin.ObjectGetRequest + namedIDRequestTask *admin.NamedEntityIdentifierListRequest + taskListResponse *admin.TaskList + argsTask []string +) + +func getTaskSetup() { + ctx = testutils.Ctx + cmdCtx = testutils.CmdCtx + mockClient = testutils.MockClient + argsTask = []string{"task1"} + sortedListLiteralType := core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + } + variableMap := map[string]*core.Variable{ + "sorted_list1": &sortedListLiteralType, + "sorted_list2": &sortedListLiteralType, + } + + task1 := &admin.Task{ + Id: &core.Identifier{ + Name: "task1", + Version: "v1", + }, + Closure: &admin.TaskClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 0, Nanos: 0}, + CompiledTask: &core.CompiledTask{ + Template: &core.TaskTemplate{ + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: variableMap, + }, + }, + }, + }, + }, + } + + task2 := &admin.Task{ + Id: &core.Identifier{ + Name: "task1", + Version: "v2", + }, + Closure: &admin.TaskClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledTask: &core.CompiledTask{ + Template: &core.TaskTemplate{ + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: variableMap, + }, + }, + }, + }, + }, + } + + tasks := []*admin.Task{task2, task1} + resourceListRequestTask = &admin.ResourceListRequest{ + Id: &admin.NamedEntityIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: argsTask[0], + }, + SortBy: &admin.Sort{ + Key: "created_at", + Direction: admin.Sort_DESCENDING, + }, + Limit: 100, + } + + taskListResponse = &admin.TaskList{ + Tasks: tasks, + } + + objectGetRequestTask = &admin.ObjectGetRequest{ + Id: &core.Identifier{ + ResourceType: core.ResourceType_TASK, + Project: projectValue, + Domain: domainValue, + Name: argsTask[0], + Version: "v2", + }, + } + namedIDRequestTask = &admin.NamedEntityIdentifierListRequest{ + Project: projectValue, + Domain: domainValue, + SortBy: &admin.Sort{ + Key: "name", + Direction: admin.Sort_ASCENDING, + }, + Limit: 100, + } + + var taskEntities []*admin.NamedEntityIdentifier + idTask1 := &admin.NamedEntityIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: "task1", + } + idTask2 := &admin.NamedEntityIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: "task2", + } + taskEntities = append(taskEntities, idTask1, idTask2) + namedIdentifierListTask := &admin.NamedEntityIdentifierList{ + Entities: taskEntities, + } + + mockClient.OnListTasksMatch(ctx, resourceListRequestTask).Return(taskListResponse, nil) + mockClient.OnGetTaskMatch(ctx, objectGetRequestTask).Return(task2, nil) + mockClient.OnListTaskIdsMatch(ctx, namedIDRequestTask).Return(namedIdentifierListTask, nil) + + taskConfig.Latest = false + taskConfig.ExecFile = "" + taskConfig.Version = "" +} + +func TestGetTaskFunc(t *testing.T) { + setup() + getTaskSetup() + err = getTaskFunc(ctx, argsTask, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "ListTasks", ctx, resourceListRequestTask) + tearDownAndVerify(t, `[ + { + "id": { + "name": "task1", + "version": "v2" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": { + "sorted_list1": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + }, + "sorted_list2": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } + }, + { + "id": { + "name": "task1", + "version": "v1" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": { + "sorted_list1": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + }, + "sorted_list2": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:00Z" + } + } +]`) +} + +func TestGetTaskFuncLatest(t *testing.T) { + setup() + getTaskSetup() + taskConfig.Latest = true + err = getTaskFunc(ctx, argsTask, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "ListTasks", ctx, resourceListRequestTask) + tearDownAndVerify(t, `{ + "id": { + "name": "task1", + "version": "v2" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": { + "sorted_list1": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + }, + "sorted_list2": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } +}`) +} + +func TestGetTaskWithVersion(t *testing.T) { + setup() + getTaskSetup() + taskConfig.Version = "v2" + objectGetRequestTask.Id.ResourceType = core.ResourceType_TASK + err = getTaskFunc(ctx, argsTask, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "GetTask", ctx, objectGetRequestTask) + tearDownAndVerify(t, `{ + "id": { + "name": "task1", + "version": "v2" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": { + "sorted_list1": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + }, + "sorted_list2": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } +}`) +} + +func TestGetTasks(t *testing.T) { + setup() + getTaskSetup() + argsTask = []string{} + err = getTaskFunc(ctx, argsTask, cmdCtx) + assert.Nil(t, err) + mockClient.AssertCalled(t, "ListTaskIds", ctx, namedIDRequest) + tearDownAndVerify(t, `[ + { + "project": "dummyProject", + "domain": "dummyDomain", + "name": "task1" + }, + { + "project": "dummyProject", + "domain": "dummyDomain", + "name": "task2" + } +]`) +} + +func TestGetTaskWithExecFile(t *testing.T) { + setup() + getTaskSetup() + taskConfig.Version = "v2" + taskConfig.ExecFile = testDataFolder + "task_exec_file" + err = getTaskFunc(ctx, argsTask, cmdCtx) + os.Remove(taskConfig.ExecFile) + assert.Nil(t, err) + mockClient.AssertCalled(t, "GetTask", ctx, objectGetRequestTask) + tearDownAndVerify(t, `{ + "id": { + "name": "task1", + "version": "v2" + }, + "closure": { + "compiledTask": { + "template": { + "interface": { + "inputs": { + "variables": { + "sorted_list1": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + }, + "sorted_list2": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + } + } + } + } + } + } + }, + "createdAt": "1970-01-01T00:00:01Z" + } +}`) +} diff --git a/flytectl/cmd/get/task_util.go b/flytectl/cmd/get/task_util.go new file mode 100644 index 0000000000..0be9490dc0 --- /dev/null +++ b/flytectl/cmd/get/task_util.go @@ -0,0 +1,93 @@ +package get + +import ( + "context" + "fmt" + + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" +) + +// Reads the task config to drive fetching the correct tasks. +func FetchTaskForName(ctx context.Context, name string, project string, domain string, cmdCtx cmdCore.CommandContext) ([]*admin.Task, error) { + var tasks []*admin.Task + var err error + var task *admin.Task + if taskConfig.Latest { + if task, err = FetchTaskLatestVersion(ctx, name, project, domain, cmdCtx); err != nil { + return nil, err + } + tasks = append(tasks, task) + } else if taskConfig.Version != "" { + if task, err = FetchTaskVersion(ctx, name, taskConfig.Version, project, domain, cmdCtx); err != nil { + return nil, err + } + tasks = append(tasks, task) + } else { + tasks, err = FetchAllVerOfTask(ctx, name, project, domain, cmdCtx) + if err != nil { + return nil, err + } + } + if taskConfig.ExecFile != "" { + // There would be atleast one task object when code reaches here and hence the length assertion is not required. + task = tasks[0] + // Only write the first task from the tasks object. + if err = CreateAndWriteExecConfigForTask(task, taskConfig.ExecFile); err != nil { + return nil, err + } + } + return tasks, nil +} + +func FetchAllVerOfTask(ctx context.Context, name string, project string, domain string, cmdCtx cmdCore.CommandContext) ([]*admin.Task, error) { + tList, err := cmdCtx.AdminClient().ListTasks(ctx, &admin.ResourceListRequest{ + Id: &admin.NamedEntityIdentifier{ + Project: project, + Domain: domain, + Name: name, + }, + SortBy: &admin.Sort{ + Key: "created_at", + Direction: admin.Sort_DESCENDING, + }, + Limit: 100, + }) + if err != nil { + return nil, err + } + if len(tList.Tasks) == 0 { + return nil, fmt.Errorf("no tasks retrieved for %v", name) + } + return tList.Tasks, nil +} + +func FetchTaskLatestVersion(ctx context.Context, name string, project string, domain string, cmdCtx cmdCore.CommandContext) (*admin.Task, error) { + var t *admin.Task + var err error + // Fetch the latest version of the task. + var taskVersions []*admin.Task + taskVersions, err = FetchAllVerOfTask(ctx, name, project, domain, cmdCtx) + if err != nil { + return nil, err + } + t = taskVersions[0] + return t, nil +} + +func FetchTaskVersion(ctx context.Context, name string, version string, project string, domain string, cmdCtx cmdCore.CommandContext) (*admin.Task, error) { + t, err := cmdCtx.AdminClient().GetTask(ctx, &admin.ObjectGetRequest{ + Id: &core.Identifier{ + ResourceType: core.ResourceType_TASK, + Project: project, + Domain: domain, + Name: name, + Version: version, + }, + }) + if err != nil { + return nil, err + } + return t, nil +} diff --git a/flytectl/cmd/get/taskconfig_flags.go b/flytectl/cmd/get/taskconfig_flags.go new file mode 100755 index 0000000000..5379d61df7 --- /dev/null +++ b/flytectl/cmd/get/taskconfig_flags.go @@ -0,0 +1,48 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package get + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (TaskConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (TaskConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in TaskConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg TaskConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("TaskConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(taskConfig.ExecFile),fmt.Sprintf("%v%v", prefix, "execFile"), taskConfig.ExecFile, "execution file name to be used for generating execution spec of a single task.") + cmdFlags.StringVar(&(taskConfig.Version),fmt.Sprintf("%v%v", prefix, "version"), taskConfig.Version, "version of the task to be fetched.") + cmdFlags.BoolVar(&(taskConfig.Latest),fmt.Sprintf("%v%v", prefix, "latest"), taskConfig.Latest, "flag to indicate to fetch the latest version, version flag will be ignored in this case") + return cmdFlags +} diff --git a/flytectl/cmd/get/taskconfig_flags_test.go b/flytectl/cmd/get/taskconfig_flags_test.go new file mode 100755 index 0000000000..b945e78619 --- /dev/null +++ b/flytectl/cmd/get/taskconfig_flags_test.go @@ -0,0 +1,168 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package get + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsTaskConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementTaskConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsTaskConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookTaskConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementTaskConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_TaskConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookTaskConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_TaskConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_TaskConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_TaskConfig(val, result)) +} + +func testDecodeSlice_TaskConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_TaskConfig(vStringSlice, result)) +} + +func TestTaskConfig_GetPFlagSet(t *testing.T) { + val := TaskConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestTaskConfig_SetFlags(t *testing.T) { + actual := TaskConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_execFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("execFile"); err == nil { + assert.Equal(t, string(taskConfig.ExecFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("execFile", testValue) + if vString, err := cmdFlags.GetString("execFile"); err == nil { + testDecodeJson_TaskConfig(t, fmt.Sprintf("%v", vString), &actual.ExecFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_version", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("version"); err == nil { + assert.Equal(t, string(taskConfig.Version), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("version", testValue) + if vString, err := cmdFlags.GetString("version"); err == nil { + testDecodeJson_TaskConfig(t, fmt.Sprintf("%v", vString), &actual.Version) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_latest", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vBool, err := cmdFlags.GetBool("latest"); err == nil { + assert.Equal(t, bool(taskConfig.Latest), vBool) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("latest", testValue) + if vBool, err := cmdFlags.GetBool("latest"); err == nil { + testDecodeJson_TaskConfig(t, fmt.Sprintf("%v", vBool), &actual.Latest) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/flytectl/cmd/register/register_util.go b/flytectl/cmd/register/register_util.go index 1f3ac0e816..031b0e836d 100644 --- a/flytectl/cmd/register/register_util.go +++ b/flytectl/cmd/register/register_util.go @@ -13,9 +13,6 @@ import ( "sort" "strings" - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/proto" - "github.com/flyteorg/flytectl/cmd/config" cmdCore "github.com/flyteorg/flytectl/cmd/core" "github.com/flyteorg/flytectl/pkg/printer" @@ -23,6 +20,9 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/storage" + + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" ) const registrationProjectPattern = "{{ registration.project }}" diff --git a/flytectl/cmd/testdata/launchplan_execution_spec.yaml b/flytectl/cmd/testdata/launchplan_execution_spec.yaml new file mode 100644 index 0000000000..aa23903ed8 --- /dev/null +++ b/flytectl/cmd/testdata/launchplan_execution_spec.yaml @@ -0,0 +1,11 @@ +iamRoleARN: "" +inputs: + numbers: + - 0 + numbers_count: 0 + run_local_at_count: 10 +kubeServiceAcct: "" +targetDomain: "" +targetProject: "" +version: v3 +workflow: core.advanced.run_merge_sort.merge_sort diff --git a/flytectl/cmd/testdata/task_execution_spec.yaml b/flytectl/cmd/testdata/task_execution_spec.yaml new file mode 100644 index 0000000000..aba4e46d6d --- /dev/null +++ b/flytectl/cmd/testdata/task_execution_spec.yaml @@ -0,0 +1,15 @@ +iamRoleARN: "" +inputs: + sorted_list1: + - 0 + - 2 + - 4 + sorted_list2: + - 1 + - 3 + - 5 +kubeServiceAcct: "" +targetDomain: "development" +targetProject: "flytesnacks" +task: core.advanced.run_merge_sort.merge +version: v2 diff --git a/flytectl/cmd/testutils/test_utils.go b/flytectl/cmd/testutils/test_utils.go new file mode 100644 index 0000000000..28d999ba73 --- /dev/null +++ b/flytectl/cmd/testutils/test_utils.go @@ -0,0 +1,62 @@ +package testutils + +import ( + "bytes" + "context" + "io" + "log" + "os" + "strings" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + + "github.com/stretchr/testify/assert" +) + +const projectValue = "dummyProject" +const domainValue = "dummyDomain" +const output = "json" + +var ( + reader *os.File + writer *os.File + Err error + Ctx context.Context + MockClient *mocks.AdminServiceClient + mockOutStream io.Writer + CmdCtx cmdCore.CommandContext + stdOut *os.File + stderr *os.File +) + +func Setup() { + Ctx = context.Background() + reader, writer, Err = os.Pipe() + if Err != nil { + panic(Err) + } + stdOut = os.Stdout + stderr = os.Stderr + os.Stdout = writer + os.Stderr = writer + log.SetOutput(writer) + MockClient = new(mocks.AdminServiceClient) + mockOutStream = writer + CmdCtx = cmdCore.NewCommandContext(MockClient, mockOutStream) + config.GetConfig().Project = projectValue + config.GetConfig().Domain = domainValue + config.GetConfig().Output = output +} + +func TearDownAndVerify(t *testing.T, expectedLog string) { + writer.Close() + os.Stdout = stdOut + os.Stderr = stderr + var buf bytes.Buffer + if _, err := io.Copy(&buf, reader); err == nil { + assert.Equal(t, strings.Trim(expectedLog, "\n "), strings.Trim(buf.String(), "\n ")) + } +} diff --git a/flytectl/cmd/update/project_test.go b/flytectl/cmd/update/project_test.go index 36e703b15e..6b0491a375 100644 --- a/flytectl/cmd/update/project_test.go +++ b/flytectl/cmd/update/project_test.go @@ -13,6 +13,7 @@ import ( cmdCore "github.com/flyteorg/flytectl/cmd/core" "github.com/flyteorg/flyteidl/clients/go/admin/mocks" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/stretchr/testify/assert" ) diff --git a/flytectl/docs/source/gen/flytectl_create.rst b/flytectl/docs/source/gen/flytectl_create.rst index dbd0218ae4..a7528e911e 100644 --- a/flytectl/docs/source/gen/flytectl_create.rst +++ b/flytectl/docs/source/gen/flytectl_create.rst @@ -71,5 +71,6 @@ SEE ALSO ~~~~~~~~ * :doc:`flytectl` - flyetcl CLI tool +* :doc:`flytectl_create_execution` - Create execution resources * :doc:`flytectl_create_project` - Create project resources diff --git a/flytectl/docs/source/gen/flytectl_create_execution.rst b/flytectl/docs/source/gen/flytectl_create_execution.rst new file mode 100644 index 0000000000..01b7f0e2b1 --- /dev/null +++ b/flytectl/docs/source/gen/flytectl_create_execution.rst @@ -0,0 +1,140 @@ +.. _flytectl_create_execution: + +flytectl create execution +------------------------- + +Create execution resources + +Synopsis +~~~~~~~~ + + + +Create the executions for given workflow/task in a project and domain. + +There are three steps in generating an execution. + +- Generate the execution spec file using the get command. +- Update the inputs for the execution if needed. +- Run the execution by passing in the generated yaml file. + +The spec file should be generated first and then run the execution using the spec file. +You can reference the flytectl get task for more details + +:: + + flytectl get tasks -d development -p flytectldemo core.advanced.run_merge_sort.merge --version v2 --execFile execution_spec.yaml + +The generated file would look similar to this + +.. code-block:: yaml + + iamRoleARN: "" + inputs: + sorted_list1: + - 0 + sorted_list2: + - 0 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + task: core.advanced.run_merge_sort.merge + version: "v2" + + +The generated file can be modified to change the input values. + +.. code-block:: yaml + + iamRoleARN: 'arn:aws:iam::12345678:role/defaultrole' + inputs: + sorted_list1: + - 2 + - 4 + - 6 + sorted_list2: + - 1 + - 3 + - 5 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + task: core.advanced.run_merge_sort.merge + version: "v2" + +And then can be passed through the command line. +Notice the source and target domain/projects can be different. +The root project and domain flags of -p and -d should point to task/launch plans project/domain. + +:: + + flytectl create execution --execFile execution_spec.yaml -p flytectldemo -d development --targetProject flytesnacks + +Usage + + +:: + + flytectl create execution [flags] + +Options +~~~~~~~ + +:: + + --execFile string file for the execution params.If not specified defaults to <_name>.execution_spec.yaml + -h, --help help for execution + --iamRoleARN string iam role ARN AuthRole for launching execution. + --kubeServiceAcct string kubernetes service account AuthRole for launching execution. + --targetDomain string project where execution needs to be created.If not specified configured domain would be used. + --targetProject string project where execution needs to be created.If not specified configured project would be used. + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' + --admin.clientId string Client ID + --admin.clientSecretLocation string File containing the client secret + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string Your IDPs token endpoint + --admin.useAuth Whether or not to try to authenticate with options below + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_create` - Used for creating various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/flytectl/docs/source/gen/flytectl_get_launchplan.rst b/flytectl/docs/source/gen/flytectl_get_launchplan.rst index ab832ed6b5..2746270b21 100644 --- a/flytectl/docs/source/gen/flytectl_get_launchplan.rst +++ b/flytectl/docs/source/gen/flytectl_get_launchplan.rst @@ -13,13 +13,13 @@ Synopsis Retrieves all the launch plans within project and domain.(launchplan,launchplans can be used interchangeably in these commands) :: - bin/flytectl get launchplan -p flytesnacks -d development + flytectl get launchplan -p flytesnacks -d development Retrieves launch plan by name within project and domain. :: - bin/flytectl get launchplan -p flytesnacks -d development core.basic.lp.go_greet + flytectl get launchplan -p flytesnacks -d development core.basic.lp.go_greet Retrieves launchplan by filters. :: @@ -30,13 +30,37 @@ Retrieves all the launchplan within project and domain in yaml format. :: - bin/flytectl get launchplan -p flytesnacks -d development -o yaml + flytectl get launchplan -p flytesnacks -d development -o yaml Retrieves all the launchplan within project and domain in json format :: - bin/flytectl get launchplan -p flytesnacks -d development -o json + flytectl get launchplan -p flytesnacks -d development -o json + +Retrieves a launch plans within project and domain for a version and generate the execution spec file for it to be used for launching the execution using create execution. + +:: + + flytectl get launchplan -d development -p flytectldemo core.advanced.run_merge_sort.merge_sort --execFile execution_spec.yam + +The generated file would look similar to this + +.. code-block:: yaml + + iamRoleARN: "" + inputs: + numbers: + - 0 + numbers_count: 0 + run_local_at_count: 10 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + workflow: core.advanced.run_merge_sort.merge + version: "v3" + +Check the create execution section on how to launch one using the generated file. Usage @@ -50,7 +74,10 @@ Options :: - -h, --help help for launchplan + --execFile string execution file name to be used for generating execution spec of a single launchplan. + -h, --help help for launchplan + --latest flag to indicate to fetch the latest version, version flag will be ignored in this case + --version string version of the launchplan to be fetched. Options inherited from parent commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/flytectl/docs/source/gen/flytectl_get_task.rst b/flytectl/docs/source/gen/flytectl_get_task.rst index 3037d96055..245de2b4f3 100644 --- a/flytectl/docs/source/gen/flytectl_get_task.rst +++ b/flytectl/docs/source/gen/flytectl_get_task.rst @@ -38,6 +38,30 @@ Retrieves all the tasks within project and domain in json format. bin/flytectl get task -p flytesnacks -d development -o json +Retrieves a tasks within project and domain for a version and generate the execution spec file for it to be used for launching the execution using create execution. + +:: + + bin/flytectl get tasks -d development -p flytesnacks core.advanced.run_merge_sort.merge --execFile execution_spec.yaml --version v2 + +The generated file would look similar to this + +.. code-block:: yaml + + iamRoleARN: "" + inputs: + sorted_list1: + - 0 + sorted_list2: + - 0 + kubeServiceAcct: "" + targetDomain: "" + targetProject: "" + task: core.advanced.run_merge_sort.merge + version: "v2" + +Check the create execution section on how to launch one using the generated file. + Usage @@ -50,7 +74,10 @@ Options :: - -h, --help help for task + --execFile string execution file name to be used for generating execution spec of a single task. + -h, --help help for task + --latest flag to indicate to fetch the latest version, version flag will be ignored in this case + --version string version of the task to be fetched. Options inherited from parent commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/flytectl/docs/source/index.rst b/flytectl/docs/source/index.rst index 00a24536a7..9833bf0c50 100644 --- a/flytectl/docs/source/index.rst +++ b/flytectl/docs/source/index.rst @@ -60,6 +60,7 @@ Basic Configuration :caption: Flytectl nouns gen/flytectl_create_project + gen/flytectl_create_execution gen/flytectl_get_execution gen/flytectl_get_project gen/flytectl_get_workflow diff --git a/flytectl/go.mod b/flytectl/go.mod index c07b1b38a1..5723376489 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -8,15 +8,23 @@ require ( github.com/flyteorg/flytestdlib v0.3.13 github.com/ghodss/yaml v1.0.0 github.com/golang/protobuf v1.4.3 + github.com/google/uuid v1.1.2 github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 + github.com/kr/text v0.2.0 // indirect github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27 github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/mapstructure v1.4.1 - github.com/sirupsen/logrus v1.7.0 - github.com/spf13/cobra v1.1.1 + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/sirupsen/logrus v1.8.0 + github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 google.golang.org/grpc v1.35.0 + google.golang.org/protobuf v1.25.0 + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.4.0 + sigs.k8s.io/yaml v1.2.0 ) + +replace github.com/flyteorg/flyteidl => github.com/flyteorg/flyteidl v0.18.21-0.20210317055906-f2ce9eb7bd1f diff --git a/flytectl/go.sum b/flytectl/go.sum index 410cad1d09..a189bc399e 100644 --- a/flytectl/go.sum +++ b/flytectl/go.sum @@ -141,6 +141,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -172,8 +173,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.18.15 h1:sXrlwTRaRjQsXYMNrY/S930SKdKtu4XnpNFEu8I4tn4= -github.com/flyteorg/flyteidl v0.18.15/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= +github.com/flyteorg/flyteidl v0.18.21-0.20210317055906-f2ce9eb7bd1f h1:7qRMZRPQXUVpebBt92msIzQBRtJ4fraWhd75qA6oqaE= +github.com/flyteorg/flyteidl v0.18.21-0.20210317055906-f2ce9eb7bd1f/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= github.com/flyteorg/flytestdlib v0.3.13 h1:5ioA/q3ixlyqkFh5kDaHgmPyTP/AHtqq1K/TIbVLUzM= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= @@ -208,6 +209,7 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -228,6 +230,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -280,6 +283,7 @@ github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= @@ -367,17 +371,19 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27 h1:O664tckOIC4smyHDDJPXAh/YBYYc0Y1O8S5wmZDm3d8= github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27/go.mod h1:f0X1c0za3TbET/rl5ThtCSel0+G3/yZ8iuU9BxnyVK0= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= +github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -426,6 +432,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -518,8 +526,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.0 h1:nfhvjKcUMhBMVqbKHJlk5RPrrfYr/NMo3692g0dwfWU= +github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -536,8 +545,9 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -959,8 +969,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1016,5 +1027,6 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/flytectl/main.go b/flytectl/main.go index 5951864944..89ecd8e65a 100644 --- a/flytectl/main.go +++ b/flytectl/main.go @@ -8,6 +8,6 @@ import ( func main() { if err := cmd.ExecuteCmd(); err != nil { - fmt.Printf("error: %v", err) + fmt.Printf("error: %v\n", err) } } diff --git a/flytectl/pkg/commandutils/command_utils.go b/flytectl/pkg/commandutils/command_utils.go new file mode 100644 index 0000000000..acee518a5f --- /dev/null +++ b/flytectl/pkg/commandutils/command_utils.go @@ -0,0 +1,26 @@ +package commandutils + +import ( + "bufio" + "fmt" + "log" + "os" + "strings" +) + +func AskForConfirmation(s string) bool { + reader := bufio.NewReader(os.Stdin) + for { + fmt.Printf("%s [y/n]: ", s) + response, err := reader.ReadString('\n') + if err != nil { + log.Fatal(err) + } + response = strings.ToLower(strings.TrimSpace(response)) + if response == "y" || response == "yes" { + return true + } else if response == "n" || response == "no" { + return false + } + } +}