feat(api,ui): init a template from an existing workflow (#3875)

cli/cdsctl/workflow_as_code.go

@@ -332,7 +332,7 @@ func workflowInitRun(c cli.Values) error {
 			return fmt.Errorf("Unable to write application file format: %v", err)
 		}
 
-		appFilePath := filepath.Join(dotCDS, appName+".app.yml")
+		appFilePath := filepath.Join(dotCDS, fmt.Sprintf(exportentities.PullApplicationName, appName))
 		if err := ioutil.WriteFile(appFilePath, b, os.FileMode(0644)); err != nil {
 			return fmt.Errorf("Unable to write application file: %v", err)
 		}
@@ -363,7 +363,7 @@ func workflowInitRun(c cli.Values) error {
 			return fmt.Errorf("Unable to write pipeline file format: %v", err)
 		}
 
-		pipFilePath := filepath.Join(dotCDS, pipName+".pip.yml")
+		pipFilePath := filepath.Join(dotCDS, fmt.Sprintf(exportentities.PullPipelineName, pipName))
 		if err := ioutil.WriteFile(pipFilePath, b, os.FileMode(0644)); err != nil {
 			return fmt.Errorf("Unable to write application file: %v", err)
 		}

engine/api/group/dao.go

@@ -14,7 +14,7 @@ func GetAllByIDs(db gorp.SqlExecutor, ids []int64) ([]sdk.Group, error) {
 		`SELECT * FROM "group" WHERE id = ANY(string_to_array($1, ',')::int[])`,
 		gorpmapping.IDsToQueryString(ids),
 	); err != nil {
-		return nil, sdk.WrapError(err, "Cannot get groups")
+		return nil, sdk.WrapError(err, "cannot get groups")
 	}
 
 	return gs, nil

engine/api/workflow/as_code.go

@@ -42,11 +42,16 @@ func UpdateAsCode(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *
 	}
 
 	// Export workflow
-	buf := new(bytes.Buffer)
-	if err := Pull(ctx, db, store, proj, wf.Name, exportentities.FormatYAML, encryptFunc, u, buf, exportentities.WorkflowSkipIfOnlyOneRepoWebhook); err != nil {
+	pull, err := Pull(ctx, db, store, proj, wf.Name, exportentities.FormatYAML, encryptFunc, u, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
+	if err != nil {
 		return nil, sdk.WrapError(err, "cannot pull workflow")
 	}
 
+	buf := new(bytes.Buffer)
+	if err := pull.Tar(buf); err != nil {
+		return nil, sdk.WrapError(err, "cannot tar pulled workflow")
+	}
+
 	var vcsStrategy = app.RepositoryStrategy
 
 	if vcsStrategy.SSHKey != "" {

engine/api/workflow/workflow_exporter.go

@@ -1,10 +1,9 @@
 package workflow
 
 import (
-	"archive/tar"
 	"bytes"
 	"context"
-	"fmt"
+	"encoding/base64"
 	"io"
 	"reflect"
 
@@ -19,7 +18,6 @@ import (
 	"github.com/ovh/cds/engine/api/workflowtemplate"
 	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/exportentities"
-	"github.com/ovh/cds/sdk/log"
 )
 
 // Export a workflow
@@ -61,29 +59,32 @@ func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts
 }
 
 // Pull a workflow with all it dependencies; it writes a tar buffer in the writer
-func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, encryptFunc sdk.EncryptFunc, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) error {
+func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
+	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
 	ctx, end := observability.Span(ctx, "workflow.Pull")
 	defer end()
 
+	var wp exportentities.WorkflowPulled
+
 	options := LoadOptions{
 		DeepPipeline: true,
 	}
 	wf, errload := Load(ctx, db, cache, proj, name, u, options)
 	if errload != nil {
-		return sdk.WrapError(errload, "Cannot load workflow %s", name)
+		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
 	}
 
 	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
 	if err != nil {
-		return err
+		return wp, err
 	}
 	if i != nil {
 		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
 		if err != nil {
-			return err
+			return wp, err
 		}
 		if err := group.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
-			return err
+			return wp, err
 		}
 	}
 
@@ -96,12 +97,12 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk
 		app := &apps[i]
 		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
 		if errv != nil {
-			return sdk.WrapError(errv, "Cannot load application variables %s", app.Name)
+			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
 		}
 		app.Variable = vars
 
-		if errk := application.LoadAllDecryptedKeys(db, app); errk != nil {
-			return sdk.WrapError(errk, "Cannot load application keys %s", app.Name)
+		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
+			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
 		}
 	}
 
@@ -110,39 +111,21 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk
 		env := &envs[i]
 		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
 		if errv != nil {
-			return sdk.WrapError(errv, "Cannot load environment variables %s", env.Name)
+			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
 		}
 		env.Variable = vars
 
-		if errk := environment.LoadAllDecryptedKeys(db, env); errk != nil {
-			return sdk.WrapError(errk, "Cannot load environment keys %s", env.Name)
+		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
+			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
 		}
 	}
 
-	tw := tar.NewWriter(w)
-	defer func() {
-		if err := tw.Close(); err != nil {
-			log.Error("%v", sdk.WrapError(err, "Unable to close tar writer"))
-		}
-	}()
-
 	buffw := new(bytes.Buffer)
-	size, errw := exportWorkflow(*wf, f, buffw, opts...)
-	if errw != nil {
-		return sdk.WrapError(errw, "Unable to export workflow")
-	}
-
-	hdr := &tar.Header{
-		Name: fmt.Sprintf("%s.yml", wf.Name),
-		Mode: 0644,
-		Size: int64(size),
-	}
-	if err := tw.WriteHeader(hdr); err != nil {
-		return sdk.WrapError(err, "Unable to write workflow header %+v", hdr)
-	}
-	if _, err := io.Copy(tw, buffw); err != nil {
-		return sdk.WrapError(err, "Unable to copy workflow buffer")
+	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
+		return wp, sdk.WrapError(err, "unable to export workflow")
 	}
+	wp.Workflow.Name = wf.Name
+	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())
 
 	var withPermissions bool
 	for _, f := range opts {
@@ -151,63 +134,35 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk
 		}
 	}
 
-	for _, a := range apps {
+	wp.Applications = make([]exportentities.WorkflowPulledItem, len(apps))
+	for i, a := range apps {
 		buff := new(bytes.Buffer)
-		size, err := application.ExportApplication(db, a, f, withPermissions, encryptFunc, buff)
-		if err != nil {
-			return sdk.WrapError(err, "Unable to export app %s", a.Name)
-		}
-		hdr := &tar.Header{
-			Name: fmt.Sprintf("%s.app.yml", a.Name),
-			Mode: 0644,
-			Size: int64(size),
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return sdk.WrapError(err, "Unable to write app header %+v", hdr)
-		}
-		if _, err := io.Copy(tw, buff); err != nil {
-			return sdk.WrapError(err, "Unable to copy app buffer")
+		if _, err := application.ExportApplication(db, a, f, withPermissions, encryptFunc, buff); err != nil {
+			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
 		}
+		wp.Applications[i].Name = a.Name
+		wp.Applications[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
 	}
 
-	for _, e := range envs {
+	wp.Environments = make([]exportentities.WorkflowPulledItem, len(envs))
+	for i, e := range envs {
 		buff := new(bytes.Buffer)
-		size, err := environment.ExportEnvironment(db, e, f, withPermissions, encryptFunc, buff)
-		if err != nil {
-			return sdk.WrapError(err, "Unable to export env %s", e.Name)
-		}
-
-		hdr := &tar.Header{
-			Name: fmt.Sprintf("%s.env.yml", e.Name),
-			Mode: 0644,
-			Size: int64(size),
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return sdk.WrapError(err, "Unable to write env header %+v", hdr)
-		}
-		if _, err := io.Copy(tw, buff); err != nil {
-			return sdk.WrapError(err, "Unable to copy env buffer")
+		if _, err := environment.ExportEnvironment(db, e, f, withPermissions, encryptFunc, buff); err != nil {
+			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
 		}
+		wp.Environments[i].Name = e.Name
+		wp.Environments[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
 	}
 
-	for _, p := range pips {
+	wp.Pipelines = make([]exportentities.WorkflowPulledItem, len(pips))
+	for i, p := range pips {
 		buff := new(bytes.Buffer)
-		size, err := pipeline.ExportPipeline(p, f, withPermissions, buff)
-		if err != nil {
-			return sdk.WrapError(err, "Unable to export pipeline %s", p.Name)
-		}
-		hdr := &tar.Header{
-			Name: fmt.Sprintf("%s.pip.yml", p.Name),
-			Mode: 0644,
-			Size: int64(size),
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return sdk.WrapError(err, "Unable to write pipeline header %+v", hdr)
-		}
-		if _, err := io.Copy(tw, buff); err != nil {
-			return sdk.WrapError(err, "Unable to copy pip buffer")
+		if _, err := pipeline.ExportPipeline(p, f, withPermissions, buff); err != nil {
+			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
 		}
+		wp.Pipelines[i].Name = p.Name
+		wp.Pipelines[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
 	}
 
-	return nil
+	return wp, nil
 }

engine/api/workflow/workflow_exporter_test.go

@@ -132,8 +132,11 @@ func TestPull(t *testing.T) {
 	test.Equal(t, w.Metadata, w1.Metadata)
 	test.Equal(t, w.PurgeTags, w1.PurgeTags)
 
+	pull, err := workflow.Pull(context.TODO(), db, cache, proj, w1.Name, exportentities.FormatYAML, project.EncryptWithBuiltinKey, u)
+	test.NoError(t, err)
+
 	buff := new(bytes.Buffer)
-	test.NoError(t, workflow.Pull(context.TODO(), db, cache, proj, w1.Name, exportentities.FormatYAML, project.EncryptWithBuiltinKey, u, buff))
+	test.NoError(t, pull.Tar(buff))
 
 	// Open the tar archive for reading.
 	r := bytes.NewReader(buff.Bytes())

engine/api/workflow_export.go

@@ -68,14 +68,24 @@ func (api *API) getWorkflowPullHandler() service.Handler {
 			return sdk.WrapError(err, "unable to load projet")
 		}
 
+		pull, err := workflow.Pull(ctx, api.mustDB(), api.Cache, proj, name, exportentities.FormatYAML, project.EncryptWithBuiltinKey, deprecatedGetUser(ctx), opts...)
+		if err != nil {
+			return err
+		}
+
+		// early returns as json if param set
+		if FormBool(r, "json") {
+			return service.WriteJSON(w, pull, http.StatusOK)
+		}
+
 		buf := new(bytes.Buffer)
-		if err := workflow.Pull(ctx, api.mustDB(), api.Cache, proj, name, exportentities.FormatYAML, project.EncryptWithBuiltinKey, deprecatedGetUser(ctx), buf, opts...); err != nil {
-			return sdk.WrapError(err, "getWorkflowPullHandler")
+		if err := pull.Tar(buf); err != nil {
+			return err
 		}
 
 		w.Header().Add("Content-Type", "application/tar")
 		w.WriteHeader(http.StatusOK)
-		_, errC := io.Copy(w, buf)
-		return sdk.WrapError(errC, "getWorkflowPullHandler> Unable to copy content buffer in the response writer")
+		_, err = io.Copy(w, buf)
+		return sdk.WrapError(err, "unable to copy content buffer in the response writer")
 	}
 }

engine/api/workflowtemplate/dao.go

@@ -57,7 +57,7 @@ func GetByID(db gorp.SqlExecutor, id int64) (*sdk.WorkflowTemplate, error) {
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}
-		return nil, sdk.WrapError(err, "Cannot get workflow template")
+		return nil, sdk.WrapError(err, "cannot get workflow template")
 	}
 
 	return &w, nil
@@ -207,7 +207,7 @@ func GetInstanceByWorkflowID(db gorp.SqlExecutor, workflowID int64) (*sdk.Workfl
 		if err == sql.ErrNoRows {
 			return nil, nil
 		}
-		return nil, sdk.WrapError(err, "Cannot get workflow template instance")
+		return nil, sdk.WrapError(err, "cannot get workflow template instance")
 	}
 
 	return &wti, nil

engine/api/workflowtemplate/execute.go

@@ -229,7 +229,7 @@ func Tar(wt *sdk.WorkflowTemplate, res sdk.WorkflowTemplateResult, w io.Writer)
 		return err
 	}
 	if err := tw.WriteHeader(&tar.Header{
-		Name: fmt.Sprintf("%s.yml", wor.Name),
+		Name: fmt.Sprintf(exportentities.PullWorkflowName, wor.Name),
 		Mode: 0644,
 		Size: int64(len(bs)),
 	}); err != nil {
@@ -254,7 +254,7 @@ func Tar(wt *sdk.WorkflowTemplate, res sdk.WorkflowTemplateResult, w io.Writer)
 			return err
 		}
 		if err := tw.WriteHeader(&tar.Header{
-			Name: fmt.Sprintf("%s.pip.yml", pip.Name),
+			Name: fmt.Sprintf(exportentities.PullPipelineName, pip.Name),
 			Mode: 0644,
 			Size: int64(len(bs)),
 		}); err != nil {
@@ -280,7 +280,7 @@ func Tar(wt *sdk.WorkflowTemplate, res sdk.WorkflowTemplateResult, w io.Writer)
 			return err
 		}
 		if err := tw.WriteHeader(&tar.Header{
-			Name: fmt.Sprintf("%s.app.yml", app.Name),
+			Name: fmt.Sprintf(exportentities.PullApplicationName, app.Name),
 			Mode: 0644,
 			Size: int64(len(bs)),
 		}); err != nil {
@@ -306,7 +306,7 @@ func Tar(wt *sdk.WorkflowTemplate, res sdk.WorkflowTemplateResult, w io.Writer)
 			return err
 		}
 		if err := tw.WriteHeader(&tar.Header{
-			Name: fmt.Sprintf("%s.env.yml", env.Name),
+			Name: fmt.Sprintf(exportentities.PullEnvironmentName, env.Name),
 			Mode: 0644,
 			Size: int64(len(bs)),
 		}); err != nil {

engine/api/workflowtemplate/export.go

@@ -46,7 +46,7 @@ func Pull(wt *sdk.WorkflowTemplate, f exportentities.Format, w io.Writer) error
 		return sdk.WrapError(errw, "Unable to export template")
 	}
 	hdr := &tar.Header{
-		Name: fmt.Sprintf("%s.yml", wt.Slug),
+		Name: fmt.Sprintf(exportentities.PullWorkflowName, wt.Slug),
 		Mode: 0644,
 		Size: int64(size),
 	}