feat: lock apply to workflow from template in api

cli/cdsctl/template.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/ovh/cds/cli"
 	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/cdsclient"
 	"github.com/ovh/cds/sdk/exportentities"
 )
 
@@ -304,13 +305,16 @@ func templateDetachRun(v cli.Values) error {
 	projectKey := v.GetString(_ProjectKey)
 	workflowName := v.GetString(_WorkflowName)
 
-	// try to get an existing template instance for current workflow
-	wti, err := client.WorkflowTemplateInstanceGet(projectKey, workflowName)
+	// try to get workflow with a template instance if exists
+	wk, err := client.WorkflowGet(projectKey, workflowName, cdsclient.WithTemplate())
 	if err != nil {
 		return err
 	}
+	if wk.TemplateInstance == nil {
+		return fmt.Errorf("given workflow is was not generated by a template")
+	}
 
-	if err := client.TemplateDeleteInstance(wti.Template.Group.Name, wti.Template.Slug, wti.ID); err != nil {
+	if err := client.TemplateDeleteInstance(wk.TemplateInstance.Template.Group.Name, wk.TemplateInstance.Template.Slug, wk.TemplateInstance.ID); err != nil {
 		return err
 	}
 

cli/cdsctl/template_apply.go

@@ -117,14 +117,18 @@ func templateApplyRun(v cli.Values) error {
 	projectKey := v.GetString(_ProjectKey)
 	workflowName := v.GetString(_WorkflowName)
 
-	var wti *sdk.WorkflowTemplateInstance
+	var existingWorkflow *sdk.Workflow
+	var existingWorkflowTemplateInstance *sdk.WorkflowTemplateInstance
 	var err error
 	if workflowName != "" {
 		// try to get an existing template instance for current workflow
-		wti, err = client.WorkflowTemplateInstanceGet(projectKey, workflowName)
+		existingWorkflow, err = client.WorkflowGet(projectKey, workflowName, cdsclient.WithTemplate())
 		if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
 			return err
 		}
+		if existingWorkflow != nil && existingWorkflow.TemplateInstance != nil {
+			existingWorkflowTemplateInstance = existingWorkflow.TemplateInstance
+		}
 	}
 
 	wt, err := getTemplateFromCLI(v)
@@ -133,8 +137,8 @@ func templateApplyRun(v cli.Values) error {
 	}
 
 	// if no template given from args, and exiting instance try to get it's template
-	if wt == nil && wti != nil {
-		wt = wti.Template
+	if wt == nil && existingWorkflowTemplateInstance != nil {
+		wt = existingWorkflowTemplateInstance.Template
 	}
 
 	// if no template found for workflow or no instance, suggest one
@@ -150,9 +154,9 @@ func templateApplyRun(v cli.Values) error {
 
 	// init params map from previous template instance if exists
 	params := make(map[string]string)
-	if wti != nil {
+	if existingWorkflowTemplateInstance != nil {
 		for _, p := range wt.Parameters {
-			if v, ok := wti.Request.Parameters[p.Key]; ok {
+			if v, ok := existingWorkflowTemplateInstance.Request.Parameters[p.Key]; ok {
 				params[p.Key] = v
 			}
 		}
@@ -168,8 +172,9 @@ func templateApplyRun(v cli.Values) error {
 		params[ps[0]] = strings.Join(ps[1:], "=")
 	}
 
-	importPush := v.GetBool("import-push")
-	importAsCode := v.GetBool("import-as-code")
+	// Import flags are not allowed if an existing ascode workflow exists
+	importPush := v.GetBool("import-push") && (existingWorkflow == nil || existingWorkflow.FromRepository == "")
+	importAsCode := v.GetBool("import-as-code") && (existingWorkflow == nil || existingWorkflow.FromRepository == "")
 	detached := v.GetBool("detach")
 
 	// try to find existing .git repository
@@ -295,7 +300,8 @@ func templateApplyRun(v cli.Values) error {
 			}
 		}
 
-		if !importAsCode && !importPush {
+		// We ask for import only if there is no existing workflow or if exists but not ascode
+		if !importAsCode && !importPush && (existingWorkflow == nil || existingWorkflow.FromRepository == "") {
 			if localRepoURL != "" {
 				importAsCode = cli.AskConfirm(fmt.Sprintf("Import the generated workflow as code to the %s project", projectKey))
 			}
@@ -317,7 +323,7 @@ func templateApplyRun(v cli.Values) error {
 		return fmt.Errorf("Unable to create directory %s: %v", v.GetString("output-dir"), err)
 	}
 
-	// check request before submit
+	// Check request before submit
 	req := sdk.WorkflowTemplateRequest{
 		ProjectKey:   projectKey,
 		WorkflowName: workflowName,
@@ -333,7 +339,7 @@ func templateApplyRun(v cli.Values) error {
 		return err
 	}
 
-	// import or push the generated workflow if one option is set
+	// Import or push the generated workflow if one option is set
 	if importAsCode || importPush {
 		var buf bytes.Buffer
 		tr, err = teeTarReader(tr, &buf)

engine/api/api_routes.go

@@ -425,7 +425,6 @@ func (api *API) InitRouter() {
 	r.Handle("/template/{groupName}/{templateSlug}/instance", Scope(sdk.AuthConsumerScopeTemplate), r.GET(api.getTemplateInstancesHandler))
 	r.Handle("/template/{groupName}/{templateSlug}/instance/{instanceID}", Scope(sdk.AuthConsumerScopeTemplate), r.DELETE(api.deleteTemplateInstanceHandler))
 	r.Handle("/template/{groupName}/{templateSlug}/usage", Scope(sdk.AuthConsumerScopeTemplate), r.GET(api.getTemplateUsageHandler))
-	r.Handle("/project/{key}/workflow/{permWorkflowName}/templateInstance", Scope(sdk.AuthConsumerScopeTemplate), r.GET(api.getTemplateInstanceHandler))
 
 	//Not Found handler
 	r.Mux.NotFoundHandler = http.HandlerFunc(NotFoundHandler)

engine/api/templates.go

@@ -354,7 +354,7 @@ func (api *API) postTemplateApplyHandler() service.Handler {
 		}
 
 		// load project with key
-		p, err := project.Load(api.mustDB(),  req.ProjectKey,
+		p, err := project.Load(api.mustDB(), req.ProjectKey,
 			project.LoadOptions.WithGroups,
 			project.LoadOptions.WithApplications,
 			project.LoadOptions.WithEnvironments,
@@ -373,6 +373,7 @@ func (api *API) postTemplateApplyHandler() service.Handler {
 			},
 		}
 
+		//
 		if !withImport && !req.Detached {
 			buf := new(bytes.Buffer)
 			if err := exportentities.TarWorkflowComponents(ctx, data, buf); err != nil {
@@ -381,6 +382,26 @@ func (api *API) postTemplateApplyHandler() service.Handler {
 			return service.Write(w, buf.Bytes(), http.StatusOK, "application/tar")
 		}
 
+		// In case we want to generated a workflow not detached from the template, we need to check if the template
+		// was not already applied to the same target workflow. If there is already a workflow and it's ascode we will
+		// create a PR from the apply request and we will not execute the template.
+		if withImport && !req.Detached {
+			wti, err := workflowtemplate.LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName(ctx, api.mustDB(), wt.ID, p.ID, req.WorkflowName)
+			if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
+				return err
+			}
+			if wti != nil && wti.WorkflowID != nil {
+				existingWorkflow, err := workflow.LoadByID(ctx, api.mustDB(), api.Cache, *p, *wti.WorkflowID, workflow.LoadOptions{})
+				if err != nil {
+					return err
+				}
+				if existingWorkflow.FromRepository != "" {
+					// TODO we need to create a PR
+					return sdk.NewErrorFrom(sdk.ErrNotImplemented, "update a workflow ascode with template not yet available")
+				}
+			}
+		}
+
 		mods := []workflowtemplate.TemplateRequestModifierFunc{
 			workflowtemplate.TemplateRequestModifiers.DefaultKeys(*p),
 		}
@@ -518,7 +539,7 @@ func (api *API) postTemplateBulkHandler() service.Handler {
 					}
 
 					// load project with key
-					p, err := project.Load(api.mustDB(),  bulk.Operations[i].Request.ProjectKey,
+					p, err := project.Load(api.mustDB(), bulk.Operations[i].Request.ProjectKey,
 						project.LoadOptions.WithGroups,
 						project.LoadOptions.WithApplications,
 						project.LoadOptions.WithEnvironments,
@@ -542,10 +563,38 @@ func (api *API) postTemplateBulkHandler() service.Handler {
 						},
 					}
 
+					// In case we want to update a workflow that is ascode, we want to create a PR instead of pushing directly the new workflow.
+					wti, err := workflowtemplate.LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName(ctx, api.mustDB(), wt.ID, p.ID, data.Template.Name)
+					if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
+						if errD := errorDefer(err); errD != nil {
+							log.Error(ctx, "%v", errD)
+							return
+						}
+						continue
+					}
+					if wti != nil && wti.WorkflowID != nil {
+						existingWorkflow, err := workflow.LoadByID(ctx, api.mustDB(), api.Cache, *p, *wti.WorkflowID, workflow.LoadOptions{})
+						if err != nil {
+							if errD := errorDefer(err); errD != nil {
+								log.Error(ctx, "%v", errD)
+								return
+							}
+							continue
+						}
+						if existingWorkflow.FromRepository != "" {
+							// TODO we need to create a PR
+							if errD := errorDefer(sdk.NewErrorFrom(sdk.ErrNotImplemented, "update a workflow ascode with template not yet available")); errD != nil {
+								log.Error(ctx, "%v", errD)
+								return
+							}
+							continue
+						}
+					}
+
 					mods := []workflowtemplate.TemplateRequestModifierFunc{
 						workflowtemplate.TemplateRequestModifiers.DefaultKeys(*p),
 					}
-					wti, err := workflowtemplate.CheckAndExecuteTemplate(ctx, api.mustDB(), *consumer, *p, &data, mods...)
+					wti, err = workflowtemplate.CheckAndExecuteTemplate(ctx, api.mustDB(), *consumer, *p, &data, mods...)
 					if err != nil {
 						if errD := errorDefer(err); errD != nil {
 							log.Error(ctx, "%v", errD)
@@ -683,37 +732,6 @@ func (api *API) getTemplateInstancesHandler() service.Handler {
 	}
 }
 
-func (api *API) getTemplateInstanceHandler() service.Handler {
-	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-		vars := mux.Vars(r)
-		key := vars["key"]
-		workflowName := vars["permWorkflowName"]
-
-		proj, err := project.Load(api.mustDB(),  key, project.LoadOptions.WithIntegrations)
-		if err != nil {
-			return sdk.WrapError(err, "unable to load projet")
-		}
-
-		wf, err := workflow.Load(ctx, api.mustDB(), api.Cache, *proj, workflowName, workflow.LoadOptions{})
-		if err != nil {
-			if sdk.ErrorIs(err, sdk.ErrNotFound) {
-				return sdk.NewErrorFrom(sdk.ErrNotFound, "cannot load workflow %s", workflowName)
-			}
-			return sdk.WithStack(err)
-		}
-
-		// return the template instance if workflow is a generated one
-		wti, err := workflowtemplate.LoadInstanceByWorkflowID(ctx, api.mustDB(), wf.ID, workflowtemplate.LoadInstanceOptions.WithTemplate)
-		if err != nil {
-			return err
-		}
-
-		wti.Project = proj
-
-		return service.WriteJSON(w, wti, http.StatusOK)
-	}
-}
-
 func (api *API) deleteTemplateInstanceHandler() service.Handler {
 	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
 		vars := mux.Vars(r)

engine/api/workflowtemplate/dao_instance.go

@@ -84,16 +84,6 @@ func LoadInstancesByTemplateIDAndProjectIDs(ctx context.Context, db gorp.SqlExec
 	return getInstances(ctx, db, query, opts...)
 }
 
-// LoadInstancesByTemplateIDAndProjectIDAndRequestWorkflowName returns all workflow template instances by template id, project id and request workflow name.
-func LoadInstancesByTemplateIDAndProjectIDAndRequestWorkflowName(ctx context.Context, db gorp.SqlExecutor, templateID, projectID int64, workflowName string, opts ...LoadInstanceOptionFunc) ([]sdk.WorkflowTemplateInstance, error) {
-	query := gorpmapping.NewQuery(`
-    SELECT *
-    FROM workflow_template_instance
-    WHERE workflow_template_id = $1 AND project_id = $2 AND (request->>'workflow_name')::text = $3
-  `).Args(templateID, projectID, workflowName)
-	return getInstances(ctx, db, query, opts...)
-}
-
 // LoadInstancesByWorkflowIDs returns all workflow template instances by workflow ids.
 func LoadInstancesByWorkflowIDs(ctx context.Context, db gorp.SqlExecutor, workflowIDs []int64, opts ...LoadInstanceOptionFunc) ([]sdk.WorkflowTemplateInstance, error) {
 	query := gorpmapping.NewQuery(`
@@ -124,6 +114,16 @@ func LoadInstanceByIDForTemplateIDAndProjectIDs(ctx context.Context, db gorp.Sql
 	return getInstance(ctx, db, query, opts...)
 }
 
+// LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName return a workflow template instance by template id, project id and request workflow name.
+func LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName(ctx context.Context, db gorp.SqlExecutor, templateID, projectID int64, workflowName string, opts ...LoadInstanceOptionFunc) (*sdk.WorkflowTemplateInstance, error) {
+	query := gorpmapping.NewQuery(`
+    SELECT *
+    FROM workflow_template_instance
+    WHERE workflow_template_id = $1 AND project_id = $2 AND (request->>'workflow_name')::text = $3
+  `).Args(templateID, projectID, workflowName)
+	return getInstance(ctx, db, query, opts...)
+}
+
 // InsertInstanceAudit for workflow template instance in database.
 func InsertInstanceAudit(db gorp.SqlExecutor, awti *sdk.AuditWorkflowTemplateInstance) error {
 	return sdk.WrapError(gorpmapping.Insert(db, awti), "unable to insert audit for workflow template instance %d", awti.WorkflowTemplateInstanceID)

engine/api/workflowtemplate/dao_instance_test.go

@@ -80,7 +80,7 @@ func TestLoad_Instance(t *testing.T) {
 		WorkflowTemplateID:      tmpl.ID,
 		WorkflowTemplateVersion: tmpl.Version,
 		Request: sdk.WorkflowTemplateRequest{
-			WorkflowName: "my-workflow",
+			WorkflowName: "my-workflow-1",
 			ProjectKey:   proj2.Key,
 		},
 	}
@@ -92,7 +92,7 @@ func TestLoad_Instance(t *testing.T) {
 		WorkflowTemplateID:      tmpl.ID,
 		WorkflowTemplateVersion: tmpl.Version,
 		Request: sdk.WorkflowTemplateRequest{
-			WorkflowName: "my-workflow",
+			WorkflowName: "my-workflow-2",
 			ProjectKey:   proj2.Key,
 		},
 	}
@@ -106,14 +106,6 @@ func TestLoad_Instance(t *testing.T) {
 	require.NoError(t, err)
 	assert.Len(t, is, 3)
 
-	is, err = workflowtemplate.LoadInstancesByTemplateIDAndProjectIDAndRequestWorkflowName(context.TODO(), db, tmpl.ID, proj2.ID, "my-unknown-workflow")
-	require.NoError(t, err)
-	assert.Len(t, is, 0)
-
-	is, err = workflowtemplate.LoadInstancesByTemplateIDAndProjectIDAndRequestWorkflowName(context.TODO(), db, tmpl.ID, proj2.ID, "my-workflow")
-	require.NoError(t, err)
-	assert.Len(t, is, 2)
-
 	is, err = workflowtemplate.LoadInstancesByWorkflowIDs(context.TODO(), db, []int64{wk1.ID})
 	require.NoError(t, err)
 	assert.Len(t, is, 1)
@@ -132,4 +124,11 @@ func TestLoad_Instance(t *testing.T) {
 	i, err = workflowtemplate.LoadInstanceByIDForTemplateIDAndProjectIDs(context.TODO(), db, wti2.ID, tmpl.ID, []int64{proj2.ID})
 	require.NoError(t, err)
 	assert.Equal(t, wti2.ID, i.ID)
+
+	i, err = workflowtemplate.LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName(context.TODO(), db, tmpl.ID, proj2.ID, "my-unknown-workflow")
+	assert.Error(t, err)
+
+	i, err = workflowtemplate.LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName(context.TODO(), db, tmpl.ID, proj2.ID, "my-workflow-1")
+	require.NoError(t, err)
+	assert.Equal(t, i.ID, wti2.ID)
 }

engine/api/workflowtemplate/instance.go

@@ -202,23 +202,11 @@ func CheckAndExecuteTemplate(ctx context.Context, db *gorp.DbMap, consumer sdk.A
 	}
 	defer tx.Rollback() // nolint
 
-	var wti *sdk.WorkflowTemplateInstance
-
 	// try to get a instance not assign to a workflow but with the same slug
-	wtis, err := LoadInstancesByTemplateIDAndProjectIDAndRequestWorkflowName(ctx, tx, wt.ID, p.ID, req.WorkflowName)
-	if err != nil {
+	wti, err := LoadInstanceByTemplateIDAndProjectIDAndRequestWorkflowName(ctx, tx, wt.ID, p.ID, req.WorkflowName)
+	if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
 		return nil, err
 	}
-	for _, res := range wtis {
-		if wti == nil {
-			wti = &res
-		} else {
-			// if there are more than one instance found, delete others
-			if err := DeleteInstance(tx, &res); err != nil {
-				return nil, err
-			}
-		}
-	}
 
 	// if a previous instance exist for the same workflow update it, else create a new one
 	var old *sdk.WorkflowTemplateInstance

sdk/cdsclient/client_workflow.go

@@ -445,14 +445,3 @@ func (c *client) WorkflowCachePull(projectKey, integrationName, ref string) (io.
 
 	return bytes.NewBuffer(body), nil
 }
-
-func (c *client) WorkflowTemplateInstanceGet(projectKey, workflowName string) (*sdk.WorkflowTemplateInstance, error) {
-	url := fmt.Sprintf("/project/%s/workflow/%s/templateInstance", projectKey, workflowName)
-
-	var i sdk.WorkflowTemplateInstance
-	if _, err := c.GetJSON(context.Background(), url, &i); err != nil {
-		return nil, err
-	}
-
-	return &i, nil
-}

sdk/cdsclient/interface.go

@@ -319,7 +319,6 @@ type WorkflowClient interface {
 	WorkflowAllHooksList() ([]sdk.NodeHook, error)
 	WorkflowCachePush(projectKey, integrationName, ref string, tarContent io.Reader, size int) error
 	WorkflowCachePull(projectKey, integrationName, ref string) (io.Reader, error)
-	WorkflowTemplateInstanceGet(projectKey, workflowName string) (*sdk.WorkflowTemplateInstance, error)
 	WorkflowTransformAsCode(projectKey, workflowName string) (*sdk.Operation, error)
 	WorkflowTransformAsCodeFollow(projectKey, workflowName string, ope *sdk.Operation) error
 }
@@ -498,6 +497,15 @@ func WithKeys() RequestModifier {
 	}
 }
 
+// WithTemplate allow a provider to retrieve a workflow with template if exists.
+func WithTemplate() RequestModifier {
+	return func(r *http.Request) {
+		q := r.URL.Query()
+		q.Set("withTemplate", "true")
+		r.URL.RawQuery = q.Encode()
+	}
+}
+
 func Format(format string) RequestModifier {
 	return func(r *http.Request) {
 		q := r.URL.Query()