diff --git a/flytectl/adminutils/iterator.go b/flytectl/adminutils/iterator.go index 8057491dd9..18368dc012 100644 --- a/flytectl/adminutils/iterator.go +++ b/flytectl/adminutils/iterator.go @@ -3,6 +3,7 @@ package adminutils import ( "context" + "github.com/golang/protobuf/proto" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" "google.golang.org/grpc" ) @@ -60,3 +61,11 @@ func GetAllNamedEntities(ctx context.Context, lister NamedEntityIDLister, req Li } return allEntities, nil } + +func NamedEntityToProtoMessage(l []*admin.NamedEntityIdentifier) []proto.Message { + messages := make([]proto.Message, 0, len(l)) + for _, m := range l { + messages = append(messages, m) + } + return messages +} \ No newline at end of file diff --git a/flytectl/cmd/get/project.go b/flytectl/cmd/get/project.go index 483d310df6..46868b4932 100644 --- a/flytectl/cmd/get/project.go +++ b/flytectl/cmd/get/project.go @@ -3,6 +3,7 @@ package get import ( "context" + "github.com/golang/protobuf/proto" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" "github.com/lyft/flytestdlib/logger" @@ -17,6 +18,14 @@ var projectColumns = []printer.Column{ {"Description", "$.description"}, } +func ProjectToProtoMessages(l []*admin.Project) []proto.Message { + messages := make([]proto.Message, 0, len(l)) + for _, m := range l { + messages = append(messages, m) + } + return messages +} + func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { adminPrinter := printer.Printer{} @@ -29,7 +38,7 @@ func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC logger.Debugf(ctx, "Retrieved %v projects", len(projects.Projects)) for _, v := range projects.Projects { if v.Name == name { - err := adminPrinter.Print(config.GetConfig().MustOutputFormat(), v, projectColumns) + err := adminPrinter.Print(config.GetConfig().MustOutputFormat(), projectColumns, v) if err != nil { return err } @@ -43,5 +52,5 @@ func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC return err } logger.Debugf(ctx, "Retrieved %v projects", len(projects.Projects)) - return adminPrinter.Print(config.GetConfig().MustOutputFormat(), projects.Projects, projectColumns) + return adminPrinter.Print(config.GetConfig().MustOutputFormat(), projectColumns, ProjectToProtoMessages(projects.Projects)...) } diff --git a/flytectl/cmd/get/task.go b/flytectl/cmd/get/task.go index a61a7f8e34..08344462ba 100644 --- a/flytectl/cmd/get/task.go +++ b/flytectl/cmd/get/task.go @@ -3,6 +3,7 @@ package get import ( "context" + "github.com/golang/protobuf/proto" "github.com/lyft/flytestdlib/logger" "github.com/lyft/flytectl/adminutils" @@ -19,7 +20,16 @@ var taskColumns = []printer.Column{ {"Name", "$.id.name"}, {"Type", "$.closure.compiledTask.template.type"}, {"Discoverable", "$.closure.compiledTask.template.metadata.discoverable"}, - {"DiscoveryVersion", "$.closure.compiledTask.template.metadata.discovery_version"}, + {"Discovery Version", "$.closure.compiledTask.template.metadata.discoveryVersion"}, + {"Created At", "$.closure.createdAt"}, +} + +func TaskToProtoMessages(l []*admin.Task) []proto.Message { + messages := make([]proto.Message, 0, len(l)) + for _, m := range l { + messages = append(messages, m) + } + return messages } func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { @@ -33,19 +43,24 @@ func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandConte Domain: config.GetConfig().Domain, Name: args[0], }, - Limit: 1, + // TODO Sorting and limits should be parameters + SortBy: &admin.Sort{ + Key: "created_at", + Direction: admin.Sort_DESCENDING, + }, + Limit: 100, }) if err != nil { return err } logger.Debugf(ctx, "Retrieved Task", task.Tasks) - return taskPrinter.Print(config.GetConfig().MustOutputFormat(), task.Tasks, taskColumns) + return taskPrinter.Print(config.GetConfig().MustOutputFormat(), taskColumns, TaskToProtoMessages(task.Tasks)...) } tasks, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListTaskIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain}) if err != nil { return err } logger.Debugf(ctx, "Retrieved %v Task", len(tasks)) - return taskPrinter.Print(config.GetConfig().MustOutputFormat(), tasks, entityColumns) + return taskPrinter.Print(config.GetConfig().MustOutputFormat(), entityColumns, adminutils.NamedEntityToProtoMessage(tasks)...) } diff --git a/flytectl/cmd/get/workflow.go b/flytectl/cmd/get/workflow.go index 7f45b1ac90..e20e9be560 100644 --- a/flytectl/cmd/get/workflow.go +++ b/flytectl/cmd/get/workflow.go @@ -3,6 +3,7 @@ package get import ( "context" + "github.com/golang/protobuf/proto" "github.com/lyft/flytestdlib/logger" "github.com/lyft/flytectl/adminutils" @@ -16,6 +17,15 @@ import ( var workflowColumns = []printer.Column{ {"Version", "$.id.version"}, {"Name", "$.id.name"}, + {"Created At", "$.closure.createdAt"}, +} + +func WorkflowToProtoMessages(l []*admin.Workflow) []proto.Message { + messages := make([]proto.Message, 0, len(l)) + for _, m := range l { + messages = append(messages, m) + } + return messages } func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { @@ -27,14 +37,19 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC Domain: config.GetConfig().Domain, Name: args[0], }, - Limit: 1, + // TODO Sorting and limits should be parameters + SortBy: &admin.Sort{ + Key: "created_at", + Direction: admin.Sort_DESCENDING, + }, + Limit: 100, }) if err != nil { return err } logger.Debugf(ctx, "Retrieved %v workflows", len(workflows.Workflows)) - return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows.Workflows, workflowColumns) + return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows.Workflows)...) } workflows, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListWorkflowIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain}) @@ -42,5 +57,5 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC return err } logger.Debugf(ctx, "Retrieved %v workflows", len(workflows)) - return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows, entityColumns) + return adminPrinter.Print(config.GetConfig().MustOutputFormat(), entityColumns, adminutils.NamedEntityToProtoMessage(workflows)...) } diff --git a/flytectl/printer/printer.go b/flytectl/printer/printer.go index edee1a1df1..e1d57d3266 100644 --- a/flytectl/printer/printer.go +++ b/flytectl/printer/printer.go @@ -7,8 +7,11 @@ import ( "os" "github.com/ghodss/yaml" + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" "github.com/kataras/tablewriter" "github.com/landoop/tableprinter" + "github.com/lyft/flytestdlib/errors" "github.com/yalp/jsonpath" ) @@ -62,24 +65,23 @@ func extractRow(data interface{}, columns []Column) []string { // Potential performance problem, as it returns all the rows in memory. // We could use the render row, but that may lead to misalignment. // TODO figure out a more optimal way -func projectColumns(input []interface{}, column []Column) ([][]string, error) { - responses := make([][]string, 0, len(input)) - for _, data := range input { - responses = append(responses, extractRow(data, column)) +func projectColumns(rows []interface{}, column []Column) ([][]string, error) { + responses := make([][]string, 0, len(rows)) + for _, row := range rows { + responses = append(responses, extractRow(row, column)) } return responses, nil } -func JSONToTable(b []byte, columns []Column) error { - var jsonRows []interface{} - err := json.Unmarshal(b, &jsonRows) - if err != nil { - return err +func JSONToTable(jsonRows []byte, columns []Column) error { + var rawRows []interface{} + if err := json.Unmarshal(jsonRows, &rawRows); err != nil { + return errors.Wrapf("JSONUnmarshalFailure", err, "failed to unmarshal into []interface{} from json") } - if jsonRows == nil { - return nil + if rawRows == nil { + return errors.Errorf("JSONUnmarshalNil", "expected one row or empty rows, received nil") } - rows, err := projectColumns(jsonRows, columns) + rows, err := projectColumns(rawRows, columns) if err != nil { return err } @@ -103,29 +105,60 @@ func JSONToTable(b []byte, columns []Column) error { return nil } -func (p Printer) Print(format OutputFormat, i interface{}, columns []Column) error { - - buf := new(bytes.Buffer) - encoder := json.NewEncoder(buf) - encoder.SetIndent(empty, tab) +func (p Printer) Print(format OutputFormat, columns []Column, messages ...proto.Message) error { - err := encoder.Encode(i) - if err != nil { - return err + printableMessages := make([]*PrintableProto, 0, len(messages)) + for _, m := range messages { + printableMessages = append(printableMessages, &PrintableProto{Message: m}) } // Factory Method for all printer switch format { - case OutputFormatJSON: // Print protobuf to json - fmt.Println(buf.String()) - case OutputFormatYAML: - v, err := yaml.JSONToYAML(buf.Bytes()) + case OutputFormatJSON, OutputFormatYAML: // Print protobuf to json + buf := new(bytes.Buffer) + encoder := json.NewEncoder(buf) + encoder.SetIndent(empty, tab) + var err error + if len(printableMessages) == 1 { + err = encoder.Encode(printableMessages[0]) + } else { + err = encoder.Encode(printableMessages) + } if err != nil { return err } - fmt.Println(string(v)) + if format == OutputFormatJSON { + fmt.Println(buf.String()) + } else { + v, err := yaml.JSONToYAML(buf.Bytes()) + if err != nil { + return err + } + fmt.Println(string(v)) + } default: // Print table - return JSONToTable(buf.Bytes(), columns) + rows, err := json.Marshal(printableMessages) + if err != nil { + return errors.Wrapf("ProtoToJSONFailure", err, "failed to marshal proto messages") + } + return JSONToTable(rows, columns) } return nil } + +type PrintableProto struct { + proto.Message +} + +var marshaller = jsonpb.Marshaler{ + Indent: tab, +} + +func (p PrintableProto) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + err := marshaller.Marshal(buf, p.Message) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +}