feat(ui,api): workflow v3 preview (#5927)

cli/cdsctl/main.go

@@ -46,6 +46,7 @@ func main() {
 		version(),
 		worker(),
 		workflow(),
+		preview(),
 	})
 	if err := root.Execute(); err != nil {
 		cli.ExitOnError(err)

cli/cdsctl/preview.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/ovh/cds/cli"
+)
+
+var previewCmd = cli.Command{
+	Name:  "preview",
+	Short: "CDS feature preview",
+	Long:  "Preview commands should not be used in production. These commands are subject to breaking changes.",
+}
+
+func preview() *cobra.Command {
+	return cli.NewCommand(previewCmd, nil, []*cobra.Command{
+		cli.NewCommand(workflowV3ValidateCmd, workflowV3ValidateRun, nil),
+		cli.NewCommand(workflowV3ConvertCmd, workflowV3ConvertRun, nil),
+	})
+}

cli/cdsctl/preview_workflowv3_convert.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk/cdsclient"
+	"github.com/ovh/cds/sdk/workflowv3"
+)
+
+var workflowV3ConvertCmd = cli.Command{
+	Name:  "workflowv3-convert",
+	Short: "Convert existing workflow to Workflow V3 files.",
+	Ctx: []cli.Arg{
+		{Name: _ProjectKey},
+		{Name: _WorkflowName},
+	},
+	Flags: []cli.Flag{
+		{
+			Name:  "full",
+			Type:  cli.FlagBool,
+			Usage: "Set the flag to export pipeline, application and environment content.",
+		},
+		{
+			Name:    "format",
+			Type:    cli.FlagString,
+			Usage:   "Specify export format (json or yaml)",
+			Default: "yaml",
+		},
+	},
+}
+
+func workflowV3ConvertRun(v cli.Values) error {
+	isFullExport := v.GetBool("full")
+
+	w, err := client.WorkflowGet(v.GetString(_ProjectKey), v.GetString(_WorkflowName), cdsclient.WithDeepPipelines())
+	if err != nil {
+		return err
+	}
+
+	res := workflowv3.Convert(*w, isFullExport)
+
+	format := v.GetString("format")
+	var buf []byte
+	switch format {
+	case "yaml":
+		buf, err = yaml.Marshal(res)
+	case "json":
+		buf, err = json.Marshal(res)
+	default:
+		return fmt.Errorf("invalid given export format %q", format)
+	}
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(buf))
+
+	return nil
+}

cli/cdsctl/preview_workflowv3_validate.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+
+	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk/workflowv3"
+)
+
+var workflowV3ValidateCmd = cli.Command{
+	Name:  "workflowv3-validate",
+	Short: "Parse and validate given Workflow V3 files.",
+	Ctx: []cli.Arg{
+		{Name: _ProjectKey},
+	},
+	VariadicArgs: cli.Arg{
+		Name: "yaml-file",
+	},
+	Flags: []cli.Flag{
+		{
+			Name: "silent",
+			Type: cli.FlagBool,
+		},
+	},
+}
+
+func workflowV3ValidateRun(v cli.Values) error {
+	projectKey := v.GetString(_ProjectKey)
+
+	if _, err := client.ProjectGet(v.GetString(_ProjectKey)); err != nil {
+		return errors.WithMessage(err, "cannot get project")
+	}
+
+	var files []string
+	filesPath := strings.Split(v.GetString("yaml-file"), ",")
+	for _, p := range filesPath {
+		if err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !info.IsDir() {
+				files = append(files, path)
+			}
+			return nil
+		}); err != nil {
+			return errors.Wrapf(err, "cannot read given path")
+		}
+	}
+
+	workflowIn := workflowv3.NewWorkflow()
+	for i := range files {
+		buf, err := ioutil.ReadFile(files[i])
+		if err != nil {
+			return errors.Wrapf(err, "cannot read file at %q", files[i])
+		}
+		var w workflowv3.Workflow
+		if err := yaml.Unmarshal(buf, &w); err != nil {
+			return errors.Wrapf(err, "cannot unmarshal file %q", files[i])
+		}
+		if err := workflowIn.Add(w); err != nil {
+			return errors.WithMessagef(err, "cannot merge workflow content from file %q", files[i])
+		}
+	}
+
+	silent := v.GetBool("silent")
+	if !silent {
+		fmt.Printf("Workflow read from %d file(s) %q:\n", len(files), files)
+		buf, err := yaml.Marshal(workflowIn)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(buf))
+	}
+
+	if err := workflowV3Validate(projectKey, workflowIn, silent); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func workflowV3Validate(projectKey string, workflowIn workflowv3.Workflow, silent bool) error {
+	// Static validation for workflow
+	extDep, err := workflowIn.Validate()
+	if err != nil {
+		return err
+	}
+
+	if !silent {
+		fmt.Println("Workflow is valid.")
+		buf, err := json.MarshalIndent(extDep, "", "  ")
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Detected external deps:\n%s\n", buf)
+	}
+
+	return nil
+}

engine/api/api_routes.go

@@ -240,6 +240,11 @@ func (api *API) InitRouter() {
 	r.Handle("/project/{key}/workflow/{permWorkflowName}/node/{nodeID}/hook/model", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getWorkflowHookModelsHandler))
 	r.Handle("/project/{key}/workflow/{permWorkflowName}/node/{nodeID}/outgoinghook/model", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getWorkflowOutgoingHookModelsHandler))
 
+	// Workflows v3
+	r.Handle("/project/{permProjectKey}/workflowv3/validate", Scope(sdk.AuthConsumerScopeProject), r.POST(api.postWorkflowV3ValidateHandler))
+	r.Handle("/project/{key}/workflowv3/{permWorkflowName}", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getWorkflowV3Handler))
+	r.Handle("/project/{key}/workflowv3/{permWorkflowName}/run/{number}", Scope(sdk.AuthConsumerScopeRun), r.GET(api.getWorkflowV3RunHandler))
+
 	// Outgoing hook model
 	r.Handle("/workflow/outgoinghook/model", ScopeNone(), r.GET(api.getWorkflowOutgoingHookModelsHandler))
 

engine/api/workflowv3.go

@@ -0,0 +1,186 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/gorilla/mux"
+
+	"github.com/ovh/cds/engine/api/database/gorpmapping"
+	"github.com/ovh/cds/engine/featureflipping"
+	"github.com/ovh/cds/engine/service"
+	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/exportentities"
+	"github.com/ovh/cds/sdk/workflowv3"
+)
+
+func (api *API) postWorkflowV3ValidateHandler() service.Handler {
+	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+		vars := mux.Vars(r)
+		projectKey := vars["permProjectKey"]
+
+		_, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, api.mustDB(), sdk.FeatureWorkflowV3, map[string]string{
+			"project_key": projectKey,
+		})
+		if !enabled {
+			return sdk.WrapError(sdk.ErrForbidden, "workflow v3 is not enabled for project %s", projectKey)
+		}
+
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return sdk.NewError(sdk.ErrWrongRequest, err)
+		}
+		defer r.Body.Close()
+
+		var res workflowv3.ValidationResponse
+
+		contentType := r.Header.Get("Content-Type")
+		if contentType == "" {
+			contentType = http.DetectContentType(body)
+		}
+		format, err := exportentities.GetFormatFromContentType(contentType)
+		if err != nil {
+			res.Error = sdk.ExtractHTTPError(err).Error()
+			return service.WriteJSON(w, res, http.StatusOK)
+		}
+
+		var workflow workflowv3.Workflow
+		if err := exportentities.Unmarshal(body, format, &workflow); err != nil {
+			res.Error = sdk.ExtractHTTPError(sdk.NewErrorFrom(sdk.ErrWrongRequest, "invalid workflow v3 format: %v", err)).Error()
+			return service.WriteJSON(w, res, http.StatusOK)
+		}
+
+		res.Workflow = workflow
+
+		// Static validation for workflow
+		extDep, err := workflow.Validate()
+
+		res.Valid = err == nil
+		if err != nil {
+			res.Error = sdk.ExtractHTTPError(sdk.NewErrorFrom(sdk.ErrWrongRequest, "invalid workflow v3 format: %v", err)).Error()
+		}
+		res.ExternalDependencies = extDep
+
+		return service.WriteJSON(w, res, http.StatusOK)
+	}
+}
+
+type workflowv3ProxyWriter struct {
+	header     http.Header
+	buf        bytes.Buffer
+	statusCode int
+}
+
+func (w *workflowv3ProxyWriter) Header() http.Header {
+	return w.header
+}
+
+func (w *workflowv3ProxyWriter) Write(bs []byte) (int, error) {
+	return w.buf.Write(bs)
+}
+
+func (w *workflowv3ProxyWriter) WriteHeader(statusCode int) {
+	w.statusCode = statusCode
+}
+
+func (api *API) getWorkflowV3Handler() service.Handler {
+	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+		vars := mux.Vars(r)
+		projectKey := vars["key"]
+
+		_, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, api.mustDB(), sdk.FeatureWorkflowV3, map[string]string{
+			"project_key": projectKey,
+		})
+		if !enabled {
+			return sdk.WrapError(sdk.ErrForbidden, "workflow v3 is not enabled for project %s", projectKey)
+		}
+
+		full := service.FormBool(r, "full")
+		format := FormString(r, "format")
+		if format == "" {
+			format = "yaml"
+		}
+		f, err := exportentities.GetFormat(format)
+		if err != nil {
+			return err
+		}
+
+		p := workflowv3ProxyWriter{header: make(http.Header)}
+
+		r.Form = url.Values{}
+		r.Form.Add("withDeepPipelines", "true")
+		if err := api.getWorkflowHandler()(ctx, &p, r); err != nil {
+			return err
+		}
+
+		var wk sdk.Workflow
+		if err := sdk.JSONUnmarshal(p.buf.Bytes(), &wk); err != nil {
+			return sdk.WithStack(err)
+		}
+
+		res := workflowv3.Convert(wk, full)
+
+		buf, err := exportentities.Marshal(res, f)
+		if err != nil {
+			return err
+		}
+		if _, err := w.Write(buf); err != nil {
+			return sdk.WithStack(err)
+		}
+
+		w.Header().Add("Content-Type", f.ContentType())
+		return nil
+	}
+}
+
+func (api *API) getWorkflowV3RunHandler() service.Handler {
+	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+		vars := mux.Vars(r)
+		projectKey := vars["key"]
+
+		_, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, api.mustDB(), sdk.FeatureWorkflowV3, map[string]string{
+			"project_key": projectKey,
+		})
+		if !enabled {
+			return sdk.WrapError(sdk.ErrForbidden, "workflow v3 is not enabled for project %s", projectKey)
+		}
+
+		full := service.FormBool(r, "full")
+		format := FormString(r, "format")
+		if format == "" {
+			format = "yaml"
+		}
+		f, err := exportentities.GetFormat(format)
+		if err != nil {
+			return err
+		}
+
+		p := workflowv3ProxyWriter{header: make(http.Header)}
+
+		if err := api.getWorkflowRunHandler()(ctx, &p, r); err != nil {
+			return err
+		}
+
+		var wkr sdk.WorkflowRun
+		if err := sdk.JSONUnmarshal(p.buf.Bytes(), &wkr); err != nil {
+			return err
+		}
+
+		res := workflowv3.ConvertRun(&wkr, full)
+
+		buf, err := exportentities.Marshal(res, f)
+		if err != nil {
+			return err
+		}
+		if _, err := w.Write(buf); err != nil {
+			return sdk.WithStack(err)
+		}
+
+		w.Header().Add("Content-Type", f.ContentType())
+
+		return nil
+	}
+}