feat(ui,api): allow to import a template from url (#3891)

cli/cdsctl/application_group.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/exportentities"
 )
 
@@ -62,7 +63,7 @@ func applicationGroupImportRun(v cli.Values) error {
 		format = "json"
 	}
 
-	if exportentities.IsURL(v.GetString("path")) {
+	if sdk.IsURL(v.GetString("path")) {
 		var err error
 		reader, _, err = exportentities.OpenURL(v.GetString("path"), format)
 		if err != nil {

cli/cdsctl/environment_group.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/exportentities"
 )
 
@@ -61,7 +62,7 @@ func environmentGroupImportRun(v cli.Values) error {
 		format = "json"
 	}
 
-	if exportentities.IsURL(v.GetString("path")) {
+	if sdk.IsURL(v.GetString("path")) {
 		var err error
 		reader, _, err = exportentities.OpenURL(v.GetString("path"), format)
 		if err != nil {

cli/cdsctl/pipeline.go

@@ -127,7 +127,7 @@ func pipelineImportRun(v cli.Values) error {
 		format = "json"
 	}
 
-	if exportentities.IsURL(v.GetString("path")) {
+	if sdk.IsURL(v.GetString("path")) {
 		var err error
 		reader, _, err = exportentities.OpenURL(v.GetString("path"), format)
 		if err != nil {

cli/cdsctl/pipeline_group.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/exportentities"
 )
 
@@ -62,7 +63,7 @@ func pipelineGroupImportRun(v cli.Values) error {
 		format = "json"
 	}
 
-	if exportentities.IsURL(v.GetString("path")) {
+	if sdk.IsURL(v.GetString("path")) {
 		var err error
 		reader, _, err = exportentities.OpenURL(v.GetString("path"), format)
 		if err != nil {

cli/cdsctl/project_group.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/exportentities"
 )
 
@@ -60,7 +61,7 @@ func projectGroupImportRun(v cli.Values) error {
 		format = "json"
 	}
 
-	if exportentities.IsURL(v.GetString("path")) {
+	if sdk.IsURL(v.GetString("path")) {
 		var err error
 		reader, _, err = exportentities.OpenURL(v.GetString("path"), format)
 		if err != nil {

cli/cdsctl/template.go

@@ -9,9 +9,9 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/exportentities"
 )
 
@@ -113,37 +113,9 @@ func templatePushRun(v cli.Values) error {
 	tar := new(bytes.Buffer)
 
 	// if the first args is an url, try to download all files
-	readFromLink := len(files) > 0 && exportentities.IsURL(files[0]) && strings.HasSuffix(files[0], ".yml")
+	readFromLink := len(files) > 0 && sdk.IsURL(files[0]) && strings.HasSuffix(files[0], ".yml")
 	if readFromLink {
-		manifestURL := files[0]
-		baseURL := manifestURL[0:strings.LastIndex(manifestURL, "/")]
-
-		// get the manifest file
-		contentFile, _, err := exportentities.OpenPath(manifestURL)
-		if err != nil {
-			return err
-		}
-		buf := new(bytes.Buffer)
-		if _, err := buf.ReadFrom(contentFile); err != nil {
-			return fmt.Errorf("cannot read from given remote file: %v", err)
-		}
-		var t exportentities.Template
-		if err := yaml.Unmarshal(buf.Bytes(), &t); err != nil {
-			return fmt.Errorf("cannot unmarshal given remote yaml file: %v", err)
-		}
-
-		// get all components of the template
-		paths := []string{t.Workflow}
-		paths = append(paths, t.Pipelines...)
-		paths = append(paths, t.Applications...)
-		paths = append(paths, t.Environments...)
-
-		links := make([]string, len(paths))
-		for i := range paths {
-			links[i] = fmt.Sprintf("%s/%s", baseURL, paths[i])
-		}
-
-		if err := workflowLinksToTarWriter(append(links, manifestURL), tar); err != nil {
+		if err := exportentities.DownloadTemplate(files[0], tar); err != nil {
 			return err
 		}
 	} else {

cli/cdsctl/workflow_push.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 
 	"github.com/ovh/cds/cli"
-	"github.com/ovh/cds/sdk/exportentities"
 )
 
 var workflowPushCmd = cli.Command{
@@ -132,36 +131,3 @@ func workflowFilesToTarWriter(files []string, buf io.Writer) error {
 	// make sure to check the error on Close
 	return tw.Close()
 }
-
-func workflowLinksToTarWriter(links []string, buf io.Writer) error {
-	tw := tar.NewWriter(buf)
-
-	// download and add some files to the archive
-	for _, link := range links {
-		contentFile, _, err := exportentities.OpenPath(link)
-		if err != nil {
-			return err
-		}
-		buf := new(bytes.Buffer)
-		if _, err := buf.ReadFrom(contentFile); err != nil {
-			return err
-		}
-
-		hdr := &tar.Header{
-			Name: filepath.Base(link),
-			Mode: 0600,
-			Size: int64(buf.Len()),
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if n, err := tw.Write(buf.Bytes()); err != nil {
-			return err
-		} else if n == 0 {
-			return fmt.Errorf("nothing to write")
-		}
-	}
-
-	// make sure to check the error on Close
-	return tw.Close()
-}

engine/api/database/gorpmapping/common.go

@@ -46,6 +46,8 @@ func Update(db gorp.SqlExecutor, i interface{}) error {
 	_, err := db.Update(i)
 	if e, ok := err.(*pq.Error); ok {
 		switch e.Code {
+		case ViolateUniqueKeyPGCode:
+			err = sdk.NewError(sdk.ErrInvalidData, e)
 		case StringDataRightTruncation:
 			err = sdk.NewError(sdk.ErrInvalidData, e)
 		}

engine/api/templates.go

@@ -133,48 +133,82 @@ func (api *API) getTemplatesHandler() service.Handler {
 
 func (api *API) postTemplateHandler() service.Handler {
 	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-		var t sdk.WorkflowTemplate
-		if err := service.UnmarshalBody(r, &t); err != nil {
+		var data sdk.WorkflowTemplate
+		if err := service.UnmarshalBody(r, &data); err != nil {
 			return err
 		}
-		if err := t.IsValid(); err != nil {
+		if err := data.IsValid(); err != nil {
 			return err
 		}
-		t.Version = 0
 
-		u := deprecatedGetUser(ctx)
+		var grp *sdk.Group
+		var err error
+		// if imported from url try to download files then overrides request
+		if data.ImportURL != "" {
+			t := new(bytes.Buffer)
+			if err := exportentities.DownloadTemplate(data.ImportURL, t); err != nil {
+				return sdk.NewError(sdk.ErrWrongRequest, err)
+			}
+			wt, err := workflowtemplate.ReadFromTar(tar.NewReader(t))
+			if err != nil {
+				return err
+			}
+			wt.ImportURL = data.ImportURL
+			data = wt
 
-		// check that group exists
-		g, err := group.LoadGroupByID(api.mustDB(), t.GroupID)
-		if err != nil {
-			return err
+			// group name should be set
+			if data.Group == nil {
+				return sdk.NewErrorFrom(sdk.ErrWrongRequest, "missing group name")
+			}
+
+			// check that the user is admin on the given template's group
+			grp, err = group.LoadGroup(api.mustDB(), data.Group.Name)
+			if err != nil {
+				return sdk.NewError(sdk.ErrWrongRequest, err)
+			}
+			data.GroupID = grp.ID
+
+			// check the workflow template extracted
+			if err := data.IsValid(); err != nil {
+				return err
+			}
+		} else {
+			// check that the group exists and user is admin for group id
+			grp, err = group.LoadGroupByID(api.mustDB(), data.GroupID)
+			if err != nil {
+				return err
+			}
 		}
 
-		if err := group.CheckUserIsGroupAdmin(g, u); err != nil {
+		data.Version = 0
+
+		u := deprecatedGetUser(ctx)
+
+		if err := group.CheckUserIsGroupAdmin(grp, u); err != nil {
 			return err
 		}
 
 		// execute template with no instance only to check if parsing is ok
-		if _, err := workflowtemplate.Execute(&t, nil); err != nil {
+		if _, err := workflowtemplate.Execute(&data, nil); err != nil {
 			return err
 		}
 
 		// duplicate couple of group id and slug will failed with sql constraint
-		if err := workflowtemplate.Insert(api.mustDB(), &t); err != nil {
+		if err := workflowtemplate.Insert(api.mustDB(), &data); err != nil {
 			return err
 		}
 
-		event.PublishWorkflowTemplateAdd(t, u)
+		event.PublishWorkflowTemplateAdd(data, u)
 
-		if err := group.AggregateOnWorkflowTemplate(api.mustDB(), &t); err != nil {
+		if err := group.AggregateOnWorkflowTemplate(api.mustDB(), &data); err != nil {
 			return err
 		}
-		if err := workflowtemplate.AggregateAuditsOnWorkflowTemplate(api.mustDB(), &t); err != nil {
+		if err := workflowtemplate.AggregateAuditsOnWorkflowTemplate(api.mustDB(), &data); err != nil {
 			return err
 		}
-		t.Editable = true
+		data.Editable = true
 
-		return service.WriteJSON(w, t, http.StatusOK)
+		return service.WriteJSON(w, data, http.StatusOK)
 	}
 }
 
@@ -211,25 +245,54 @@ func (api *API) putTemplateHandler() service.Handler {
 			return err
 		}
 
+		var grp *sdk.Group
 		var err error
-		ctx, err = api.middlewareTemplate(true)(ctx, w, r)
-		if err != nil {
-			return err
-		}
+		// if imported from url try to download files then overrides request
+		if data.ImportURL != "" {
+			t := new(bytes.Buffer)
+			if err := exportentities.DownloadTemplate(data.ImportURL, t); err != nil {
+				return sdk.NewError(sdk.ErrWrongRequest, err)
+			}
+			wt, err := workflowtemplate.ReadFromTar(tar.NewReader(t))
+			if err != nil {
+				return err
+			}
+			wt.ImportURL = data.ImportURL
+			data = wt
 
-		old := getWorkflowTemplate(ctx)
-		u := deprecatedGetUser(ctx)
+			// group name should be set
+			if data.Group == nil {
+				return sdk.NewErrorFrom(sdk.ErrWrongRequest, "missing group name")
+			}
 
-		// if group id has changed check that the group exists and user is admin for new group id
-		if old.GroupID != data.GroupID {
-			newGroup, err := group.LoadGroupByID(api.mustDB(), data.GroupID)
+			// check that the user is admin on the given template's group
+			grp, err = group.LoadGroup(api.mustDB(), data.Group.Name)
 			if err != nil {
-				return err
+				return sdk.NewError(sdk.ErrWrongRequest, err)
 			}
+			data.GroupID = grp.ID
 
-			if err := group.CheckUserIsGroupAdmin(newGroup, u); err != nil {
+			// check the workflow template extracted
+			if err := data.IsValid(); err != nil {
 				return err
 			}
+		} else {
+			// check that the group exists and user is admin for group id
+			grp, err = group.LoadGroupByID(api.mustDB(), data.GroupID)
+			if err != nil {
+				return err
+			}
+		}
+
+		ctx, err = api.middlewareTemplate(true)(ctx, w, r)
+		if err != nil {
+			return err
+		}
+		old := getWorkflowTemplate(ctx)
+		u := deprecatedGetUser(ctx)
+
+		if err := group.CheckUserIsGroupAdmin(grp, u); err != nil {
+			return err
 		}
 
 		// update fields from request data