Task details in table and using jsonpb to print data (#15)

flyteorg · Oct 13, 2020 · 1d96987 · 1d96987
1 parent 09013f7
commit 1d96987
Show file tree

Hide file tree

Showing 5 changed files with 116 additions and 35 deletions.
diff --git a/flytectl/adminutils/iterator.go b/flytectl/adminutils/iterator.go
@@ -3,6 +3,7 @@ package adminutils
 import (
 	"context"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin"
 	"google.golang.org/grpc"
 )
@@ -60,3 +61,11 @@ func GetAllNamedEntities(ctx context.Context, lister NamedEntityIDLister, req Li
 	}
 	return allEntities, nil
 }
+
+func NamedEntityToProtoMessage(l []*admin.NamedEntityIdentifier) []proto.Message {
+	messages := make([]proto.Message, 0, len(l))
+	for _, m := range l {
+		messages = append(messages, m)
+	}
+	return messages
+}
diff --git a/flytectl/cmd/get/project.go b/flytectl/cmd/get/project.go
@@ -3,6 +3,7 @@ package get
 import (
 	"context"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin"
 	"github.com/lyft/flytestdlib/logger"
 
@@ -17,6 +18,14 @@ var projectColumns = []printer.Column{
 	{"Description", "$.description"},
 }
 
+func ProjectToProtoMessages(l []*admin.Project) []proto.Message {
+	messages := make([]proto.Message, 0, len(l))
+	for _, m := range l {
+		messages = append(messages, m)
+	}
+	return messages
+}
+
 func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
 	adminPrinter := printer.Printer{}
 
@@ -29,7 +38,7 @@ func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 		logger.Debugf(ctx, "Retrieved %v projects", len(projects.Projects))
 		for _, v := range projects.Projects {
 			if v.Name == name {
-				err := adminPrinter.Print(config.GetConfig().MustOutputFormat(), v, projectColumns)
+				err := adminPrinter.Print(config.GetConfig().MustOutputFormat(), projectColumns, v)
 				if err != nil {
 					return err
 				}
@@ -43,5 +52,5 @@ func getProjectsFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 		return err
 	}
 	logger.Debugf(ctx, "Retrieved %v projects", len(projects.Projects))
-	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), projects.Projects, projectColumns)
+	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), projectColumns, ProjectToProtoMessages(projects.Projects)...)
 }
diff --git a/flytectl/cmd/get/task.go b/flytectl/cmd/get/task.go
@@ -3,6 +3,7 @@ package get
 import (
 	"context"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/lyft/flytestdlib/logger"
 
 	"github.com/lyft/flytectl/adminutils"
@@ -19,7 +20,16 @@ var taskColumns = []printer.Column{
 	{"Name", "$.id.name"},
 	{"Type", "$.closure.compiledTask.template.type"},
 	{"Discoverable", "$.closure.compiledTask.template.metadata.discoverable"},
-	{"DiscoveryVersion", "$.closure.compiledTask.template.metadata.discovery_version"},
+	{"Discovery Version", "$.closure.compiledTask.template.metadata.discoveryVersion"},
+	{"Created At", "$.closure.createdAt"},
+}
+
+func TaskToProtoMessages(l []*admin.Task) []proto.Message {
+	messages := make([]proto.Message, 0, len(l))
+	for _, m := range l {
+		messages = append(messages, m)
+	}
+	return messages
 }
 
 func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
@@ -33,19 +43,24 @@ func getTaskFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandConte
 				Domain:  config.GetConfig().Domain,
 				Name:    args[0],
 			},
-			Limit: 1,
+			// TODO Sorting and limits should be parameters
+			SortBy: &admin.Sort{
+				Key: "created_at",
+				Direction: admin.Sort_DESCENDING,
+			},
+			Limit: 100,
 		})
 		if err != nil {
 			return err
 		}
 		logger.Debugf(ctx, "Retrieved Task", task.Tasks)
 
-		return taskPrinter.Print(config.GetConfig().MustOutputFormat(), task.Tasks, taskColumns)
+		return taskPrinter.Print(config.GetConfig().MustOutputFormat(), taskColumns, TaskToProtoMessages(task.Tasks)...)
 	}
 	tasks, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListTaskIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain})
 	if err != nil {
 		return err
 	}
 	logger.Debugf(ctx, "Retrieved %v Task", len(tasks))
-	return taskPrinter.Print(config.GetConfig().MustOutputFormat(), tasks, entityColumns)
+	return taskPrinter.Print(config.GetConfig().MustOutputFormat(), entityColumns, adminutils.NamedEntityToProtoMessage(tasks)...)
 }
diff --git a/flytectl/cmd/get/workflow.go b/flytectl/cmd/get/workflow.go
@@ -3,6 +3,7 @@ package get
 import (
 	"context"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/lyft/flytestdlib/logger"
 
 	"github.com/lyft/flytectl/adminutils"
@@ -16,6 +17,15 @@ import (
 var workflowColumns = []printer.Column{
 	{"Version", "$.id.version"},
 	{"Name", "$.id.name"},
+	{"Created At", "$.closure.createdAt"},
+}
+
+func WorkflowToProtoMessages(l []*admin.Workflow) []proto.Message {
+	messages := make([]proto.Message, 0, len(l))
+	for _, m := range l {
+		messages = append(messages, m)
+	}
+	return messages
 }
 
 func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
@@ -27,20 +37,25 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 				Domain:  config.GetConfig().Domain,
 				Name:    args[0],
 			},
-			Limit: 1,
+			// TODO Sorting and limits should be parameters
+			SortBy: &admin.Sort{
+				Key: "created_at",
+				Direction: admin.Sort_DESCENDING,
+			},
+			Limit: 100,
 		})
 		if err != nil {
 			return err
 		}
 		logger.Debugf(ctx, "Retrieved %v workflows", len(workflows.Workflows))
 
-		return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows.Workflows, workflowColumns)
+		return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflowColumns, WorkflowToProtoMessages(workflows.Workflows)...)
 	}
 
 	workflows, err := adminutils.GetAllNamedEntities(ctx, cmdCtx.AdminClient().ListWorkflowIds, adminutils.ListRequest{Project: config.GetConfig().Project, Domain: config.GetConfig().Domain})
 	if err != nil {
 		return err
 	}
 	logger.Debugf(ctx, "Retrieved %v workflows", len(workflows))
-	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), workflows, entityColumns)
+	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), entityColumns, adminutils.NamedEntityToProtoMessage(workflows)...)
 }
diff --git a/flytectl/printer/printer.go b/flytectl/printer/printer.go
@@ -7,8 +7,11 @@ import (
 	"os"
 
 	"github.com/ghodss/yaml"
+	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
 	"github.com/kataras/tablewriter"
 	"github.com/landoop/tableprinter"
+	"github.com/lyft/flytestdlib/errors"
 	"github.com/yalp/jsonpath"
 )
 
@@ -62,24 +65,23 @@ func extractRow(data interface{}, columns []Column) []string {
 // Potential performance problem, as it returns all the rows in memory.
 // We could use the render row, but that may lead to misalignment.
 // TODO figure out a more optimal way
-func projectColumns(input []interface{}, column []Column) ([][]string, error) {
-	responses := make([][]string, 0, len(input))
-	for _, data := range input {
-		responses = append(responses, extractRow(data, column))
+func projectColumns(rows []interface{}, column []Column) ([][]string, error) {
+	responses := make([][]string, 0, len(rows))
+	for _, row := range rows {
+		responses = append(responses, extractRow(row, column))
 	}
 	return responses, nil
 }
 
-func JSONToTable(b []byte, columns []Column) error {
-	var jsonRows []interface{}
-	err := json.Unmarshal(b, &jsonRows)
-	if err != nil {
-		return err
+func JSONToTable(jsonRows []byte, columns []Column) error {
+	var rawRows []interface{}
+	if err := json.Unmarshal(jsonRows, &rawRows); err != nil {
+		return errors.Wrapf("JSONUnmarshalFailure", err, "failed to unmarshal into []interface{} from json")
 	}
-	if jsonRows == nil {
-		return nil
+	if rawRows == nil {
+		return errors.Errorf("JSONUnmarshalNil", "expected one row or empty rows, received nil")
 	}
-	rows, err := projectColumns(jsonRows, columns)
+	rows, err := projectColumns(rawRows, columns)
 	if err != nil {
 		return err
 	}
@@ -103,29 +105,60 @@ func JSONToTable(b []byte, columns []Column) error {
 	return nil
 }
 
-func (p Printer) Print(format OutputFormat, i interface{}, columns []Column) error {
-
-	buf := new(bytes.Buffer)
-	encoder := json.NewEncoder(buf)
-	encoder.SetIndent(empty, tab)
+func (p Printer) Print(format OutputFormat, columns []Column, messages ...proto.Message) error {
 
-	err := encoder.Encode(i)
-	if err != nil {
-		return err
+	printableMessages := make([]*PrintableProto, 0, len(messages))
+	for _, m := range messages {
+		printableMessages = append(printableMessages, &PrintableProto{Message: m})
 	}
 
 	// Factory Method for all printer
 	switch format {
-	case OutputFormatJSON: // Print protobuf to json
-		fmt.Println(buf.String())
-	case OutputFormatYAML:
-		v, err := yaml.JSONToYAML(buf.Bytes())
+	case OutputFormatJSON, OutputFormatYAML: // Print protobuf to json
+		buf := new(bytes.Buffer)
+		encoder := json.NewEncoder(buf)
+		encoder.SetIndent(empty, tab)
+		var err error
+		if len(printableMessages) == 1 {
+			err = encoder.Encode(printableMessages[0])
+		} else {
+			err = encoder.Encode(printableMessages)
+		}
 		if err != nil {
 			return err
 		}
-		fmt.Println(string(v))
+		if format == OutputFormatJSON {
+			fmt.Println(buf.String())
+		} else {
+			v, err := yaml.JSONToYAML(buf.Bytes())
+			if err != nil {
+				return err
+			}
+			fmt.Println(string(v))
+		}
 	default: // Print table
-		return JSONToTable(buf.Bytes(), columns)
+		rows, err := json.Marshal(printableMessages)
+		if err != nil {
+			return errors.Wrapf("ProtoToJSONFailure", err, "failed to marshal proto messages")
+		}
+		return JSONToTable(rows, columns)
 	}
 	return nil
 }
+
+type PrintableProto struct {
+	proto.Message
+}
+
+var marshaller = jsonpb.Marshaler{
+	Indent: tab,
+}
+
+func (p PrintableProto) MarshalJSON() ([]byte, error) {
+	buf := new(bytes.Buffer)
+	err := marshaller.Marshal(buf, p.Message)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}