refactor(api): searching entities + add more check on analysis (#7254)

engine/api/entity_search.go

@@ -334,3 +334,106 @@ func (ef *EntityFinder) searchEntity(ctx context.Context, db gorp.SqlExecutor, s
 	}
 	return completePath, "", nil
 }
+
+func (ef *EntityFinder) searchAction(ctx context.Context, db gorp.SqlExecutor, store cache.Store, name string) (*sdk.V2Action, string, string, error) {
+	// Local def
+	if strings.HasPrefix(name, ".cds/actions/") {
+		// Find action from path
+		localAct, has := ef.localActionsCache[name]
+		if !has {
+			actionEntity, err := entity.LoadEntityByPathAndRefAndCommit(ctx, db, ef.currentRepo.ID, name, ef.currentRef, ef.currentSha)
+			if err != nil {
+				return nil, "", fmt.Sprintf("Unable to find action %s", name), nil
+			}
+			if err := yaml.Unmarshal([]byte(actionEntity.Data), &localAct); err != nil {
+				return nil, "", "", err
+			}
+			ef.localActionsCache[name] = localAct
+		}
+		completeName := fmt.Sprintf("%s/%s/%s/%s@%s", ef.currentProject, ef.currentVCS.Name, ef.currentRepo.Name, localAct.Name, ef.currentRef)
+		return &localAct, completeName, "", nil
+	}
+
+	actionName := strings.TrimPrefix(name, "actions/")
+	actionSplit := strings.Split(actionName, "/")
+
+	// If plugins
+	if strings.HasPrefix(name, "actions/") && len(actionSplit) == 1 {
+		// Check plugins
+		if _, has := ef.plugins[actionSplit[0]]; !has {
+			return nil, "", fmt.Sprintf("Action %s doesn't exist", actionSplit[0]), nil
+		}
+		return nil, "", "", nil
+	}
+
+	// Others
+	completePath, msg, err := ef.searchEntity(ctx, db, store, actionName, sdk.EntityTypeAction)
+	if msg != "" || err != nil {
+		return nil, completePath, msg, err
+	}
+	act := ef.actionsCache[completePath]
+	return &act, completePath, msg, err
+}
+
+func (ef *EntityFinder) searchWorkerModel(ctx context.Context, db gorp.SqlExecutor, store cache.Store, name string) (*sdk.EntityWithObject, string, string, error) {
+	// Local def
+	if strings.HasPrefix(name, ".cds/worker-models/") {
+		// Find worker model from path
+		localWM, has := ef.localWorkerModelCache[name]
+		if !has {
+			wmEntity, err := entity.LoadEntityByPathAndRefAndCommit(ctx, db, ef.currentRepo.ID, name, ef.currentRef, ef.currentSha)
+			if err != nil {
+				return nil, "", fmt.Sprintf("Unable to find worker model %s", name), nil
+			}
+			var wm sdk.V2WorkerModel
+			if err := yaml.Unmarshal([]byte(wmEntity.Data), &wm); err != nil {
+				return nil, "", "", err
+			}
+			localWM = sdk.EntityWithObject{Entity: *wmEntity, Model: wm}
+			ef.localWorkerModelCache[name] = localWM
+		}
+		completeName := fmt.Sprintf("%s/%s/%s/%s@%s", ef.currentProject, ef.currentVCS.Name, ef.currentRepo.Name, localWM.Model.Name, ef.currentRef)
+		return &localWM, completeName, "", nil
+	}
+
+	completeName, msg, err := ef.searchEntity(ctx, db, store, name, sdk.EntityTypeWorkerModel)
+	if err != nil {
+		return nil, completeName, "", err
+	}
+	if msg != "" {
+		return nil, completeName, msg, nil
+	}
+	wm := ef.workerModelCache[completeName]
+	return &wm, completeName, "", nil
+}
+
+func (ef *EntityFinder) searchWorkflowTemplate(ctx context.Context, db gorp.SqlExecutor, store cache.Store, name string) (*sdk.EntityWithObject, string, string, error) {
+	if strings.HasPrefix(name, ".cds/workflow-templates/") {
+		// Find tempalte from path
+		localEntity, has := ef.localTemplatesCache[name]
+		if !has {
+			wtEntity, err := entity.LoadEntityByPathAndRefAndCommit(ctx, db, ef.currentRepo.ID, name, ef.currentRef, ef.currentSha)
+			if err != nil {
+				msg := fmt.Sprintf("Unable to find workflow template %s %s %s %s", ef.currentRepo.ID, name, ef.currentRef, ef.currentSha)
+				return nil, "", msg, nil
+			}
+			if err := yaml.Unmarshal([]byte(wtEntity.Data), &localEntity.Template); err != nil {
+				return nil, "", "", sdk.NewErrorFrom(sdk.ErrInvalidData, "unable to read workflow template %s: %v", name, err)
+			}
+			localEntity.Entity = *wtEntity
+			ef.localTemplatesCache[name] = localEntity
+		}
+		completeName := fmt.Sprintf("%s/%s/%s/%s@%s", ef.currentProject, ef.currentVCS.Name, ef.currentRepo.Name, localEntity.Template.Name, ef.currentRef)
+		return &localEntity, completeName, "", nil
+	}
+	completeName, msg, err := ef.searchEntity(ctx, db, store, name, sdk.EntityTypeWorkflowTemplate)
+	if err != nil {
+		return nil, completeName, "", err
+	}
+	if msg != "" {
+		return nil, completeName, msg, nil
+	}
+	e := ef.templatesCache[completeName]
+	return &e, completeName, "", nil
+
+}

engine/api/v2_repository_analyze.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ovh/cds/engine/api/event_v2"
 	"github.com/ovh/cds/engine/api/link"
 	"github.com/ovh/cds/engine/api/operation"
+	"github.com/ovh/cds/engine/api/plugin"
 	"github.com/ovh/cds/engine/api/project"
 	"github.com/ovh/cds/engine/api/rbac"
 	"github.com/ovh/cds/engine/api/repositoriesmanager"
@@ -536,6 +537,14 @@ func (api *API) analyzeRepository(ctx context.Context, projectRepoID string, ana
 
 	ef := NewEntityFinder(proj.Key, analysis.Ref, analysis.Commit, *repo, *vcsProjectWithSecret, *u, analysis.Data.CDSAdminWithMFA, api.Config.WorkflowV2.LibraryProjectKey)
 
+	plugins, err := plugin.LoadAllByType(ctx, api.mustDB(), sdk.GRPCPluginAction)
+	if err != nil {
+		return err
+	}
+	for _, p := range plugins {
+		ef.plugins[p.Name] = p
+	}
+
 	// Transform file content into entities
 	entities, multiErr := api.handleEntitiesFiles(ctx, ef, filesContent, analysis)
 	if multiErr != nil {
@@ -860,27 +869,14 @@ func manageWorkflowHooks(ctx context.Context, db gorpmapper.SqlExecutorWithTx, c
 
 	// If there is a workflow template and non hooks on workflow, check the workflow template
 	if e.Workflow.From != "" && e.Workflow.On == nil {
-		var wkfTmpl sdk.V2WorkflowTemplate
-		if strings.HasPrefix(e.Workflow.From, ".cds/") {
-			tmpl, err := entity.LoadEntityByPathAndRefAndCommit(ctx, db, e.ProjectRepositoryID, e.Workflow.From, e.Ref, e.Commit)
-			if err != nil {
-				return nil, err
-			}
-			if err := yaml.Unmarshal([]byte(tmpl.Data), &wkfTmpl); err != nil {
-				return nil, err
-			}
-		} else {
-			completePath, errMsg, err := ef.searchEntity(ctx, db, cache, e.Workflow.From, sdk.EntityTypeWorkflowTemplate)
-			if err != nil {
-				return nil, err
-			}
-			if errMsg != "" {
-				return nil, sdk.NewErrorFrom(sdk.ErrInvalidData, errMsg)
-			}
-			workflowTemplate := ef.templatesCache[completePath]
-			wkfTmpl = workflowTemplate.Template
+		entTemplate, _, msg, err := ef.searchWorkflowTemplate(ctx, db, cache, e.Workflow.From)
+		if err != nil {
+			return nil, err
+		}
+		if msg != "" {
+			return nil, sdk.NewErrorFrom(sdk.ErrInvalidData, msg)
 		}
-		if _, err := wkfTmpl.Resolve(ctx, &e.Workflow); err != nil {
+		if _, err := entTemplate.Template.Resolve(ctx, &e.Workflow); err != nil {
 			return nil, sdk.NewErrorFrom(sdk.ErrInvalidData, "unable to compute workflow from template: %v", err)
 		}
 	}
@@ -1565,44 +1561,57 @@ func Lint[T sdk.Lintable](ctx context.Context, api *API, o T, ef *EntityFinder)
 	case sdk.V2Workflow:
 		switch {
 		case x.From != "":
-			var tmpl *sdk.V2WorkflowTemplate
-			if strings.HasPrefix(x.From, ".cds/workflow-templates/") {
-				// Retrieve tmpl from current analysis
-				for _, v := range ef.templatesCache {
-					if v.FilePath == x.From {
-						tmpl = &v.Template
-						break
-					}
-				}
-			} else {
-				// Retrieve existing template in DB
-				path, msg, errSearch := ef.searchEntity(ctx, api.mustDB(), api.Cache, x.From, sdk.EntityTypeWorkflowTemplate)
-				if errSearch != nil {
-					err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "unable to retrieve entity %s of type %s: %v", x.From, sdk.EntityTypeWorkflowTemplate, errSearch))
-					break
-				}
-				if msg != "" {
-					err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, msg))
-					break
-				}
-				t := ef.templatesCache[path].Template
-				tmpl = &t
+			entTmpl, _, msg, errSearch := ef.searchWorkflowTemplate(ctx, api.mustDB(), api.Cache, x.From)
+			if errSearch != nil {
+				err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "workflow %s: unable to retrieve template %s of type %s: %v", x.Name, x.From, sdk.EntityTypeWorkflowTemplate, errSearch))
+				break
 			}
-			if tmpl == nil || tmpl.Name == "" {
-				err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "unknown workflow template %s", x.From))
-			} else {
-				// Check required parameters
-				for _, v := range tmpl.Parameters {
-					if wkfP, has := x.Parameters[v.Key]; (!has || len(wkfP) == 0) && v.Required {
-						err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "required template parameter %s is required by template %s", v.Key, x.From))
-					}
+			if msg != "" {
+				err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "workflow %s: %s", x.Name, msg))
+				break
+			}
+			if entTmpl == nil || entTmpl.Template.Name == "" {
+				err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "workflow %s: unknown workflow template %s", x.Name, x.From))
+			}
+			// Check required parameters
+			for _, v := range entTmpl.Template.Parameters {
+				if wkfP, has := x.Parameters[v.Key]; (!has || len(wkfP) == 0) && v.Required {
+					err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "workflow %s: required template parameter %s is missing or empty", x.Name, x.From))
 				}
 			}
+
 		default:
 			sameVCS := x.Repository == nil || x.Repository.VCSServer == ef.currentVCS.Name || x.Repository.VCSServer == ""
 			sameRepo := x.Repository == nil || x.Repository.Name == ef.currentRepo.Name || x.Repository.Name == ""
 			if sameVCS && sameRepo && x.Repository != nil && x.Repository.InsecureSkipSignatureVerify {
-				err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "parameter `insecure-skip-signature-verify`is not allowed if the workflow is defined on the same repository as `workfow.repository.name`. "))
+				err = append(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "workflow %s: parameter `insecure-skip-signature-verify`is not allowed if the workflow is defined on the same repository as `workfow.repository.name`. ", x.Name))
+			}
+			for _, j := range x.Jobs {
+				// Check if worker model exists
+				if !strings.Contains(j.RunsOn.Model, "${{") && len(j.Steps) > 0 {
+					_, _, msg, errSearch := ef.searchWorkerModel(ctx, api.mustDB(), api.Cache, j.RunsOn.Model)
+					if errSearch != nil {
+						err = append(err, errSearch)
+					}
+					if msg != "" {
+						err = append(err, sdk.NewErrorFrom(sdk.ErrInvalidData, "workflow %s: %s", x.Name, msg))
+					}
+				}
+
+				// Check if actions/plugins exists
+				for _, s := range j.Steps {
+					if s.Uses == "" {
+						continue
+					}
+					_, _, msg, errSearch := ef.searchAction(ctx, api.mustDB(), api.Cache, s.Uses)
+					if errSearch != nil {
+						err = append(err, errSearch)
+					}
+					if msg != "" {
+						err = append(err, sdk.NewErrorFrom(sdk.ErrInvalidData, msg))
+					}
+				}
+
 			}
 		}
 	}
@@ -1646,14 +1655,17 @@ func ReadEntityFile[T sdk.Lintable](ctx context.Context, api *API, directory, fi
 		switch t {
 		case sdk.EntityTypeWorkerModel:
 			eo.Model = any(o).(sdk.V2WorkerModel)
+			ef.localWorkerModelCache[eo.Entity.FilePath] = eo
 			ef.workerModelCache[fmt.Sprintf("%s/%s/%s/%s@%s", analysis.ProjectKey, ef.currentVCS.Name, ef.currentRepo.Name, eo.Model.Name, analysis.Ref)] = eo
 		case sdk.EntityTypeAction:
 			eo.Action = any(o).(sdk.V2Action)
+			ef.localActionsCache[eo.Entity.FilePath] = eo.Action
 			ef.actionsCache[fmt.Sprintf("%s/%s/%s/%s@%s", analysis.ProjectKey, ef.currentVCS.Name, ef.currentRepo.Name, eo.Action.Name, analysis.Ref)] = eo.Action
 		case sdk.EntityTypeWorkflow:
 			eo.Workflow = any(o).(sdk.V2Workflow)
 		case sdk.EntityTypeWorkflowTemplate:
 			eo.Template = any(o).(sdk.V2WorkflowTemplate)
+			ef.localTemplatesCache[eo.Entity.FilePath] = eo
 			ef.templatesCache[fmt.Sprintf("%s/%s/%s/%s@%s", analysis.ProjectKey, ef.currentVCS.Name, ef.currentRepo.Name, eo.Template.Name, analysis.Ref)] = eo
 		}
 

engine/api/v2_repository_analyze_test.go

@@ -2459,7 +2459,7 @@ parameters:
 	anal, err := repository.LoadRepositoryAnalysisById(ctx, db, repo.ID, analysis.ID)
 	require.NoError(t, err)
 	require.Equal(t, sdk.RepositoryAnalysisStatusError, anal.Status)
-	require.Equal(t, "unable to find workflow dependency: mytemplate", anal.Data.Error)
+	require.Equal(t, "workflow myworkflow: unable to find workflow dependency: mytemplate", anal.Data.Error)
 }
 func TestAnalyzeGithubUpdateWorkflowNoRight(t *testing.T) {
 	api, db, _ := newTestAPI(t)
@@ -2597,14 +2597,22 @@ GDFkaTe3nUJdYV4=
 
 	servicesClients.EXPECT().DoJSONRequest(gomock.Any(), "POST", "/v2/repository/event/callback", gomock.Any(), gomock.Any()).AnyTimes()
 
-	wkf := `
-    name: myworkflow
-    jobs:
-      root:
-        steps:
-        - run: echo toto`
+	wkf := `name: myworkflow
+jobs:
+  root:
+    runs-on: .cds/worker-models/mymodel.yml 
+    steps:
+    - run: echo toto`
 	encodedWorkflow := base64.StdEncoding.EncodeToString([]byte(wkf))
 
+	wm := `name: mymodel
+osarch: linux/amd64
+type: docker
+spec:
+  image: myimage`
+
+	encodedWorkerModel := base64.StdEncoding.EncodeToString([]byte(wm))
+
 	servicesClients.EXPECT().
 		DoJSONRequest(gomock.Any(), "GET", "/vcs/vcs-server/repos/myrepo/branches/?branch=devBranch&default=false&noCache=true", gomock.Any(), gomock.Any(), gomock.Any()).
 		DoAndReturn(
@@ -2642,6 +2650,10 @@ GDFkaTe3nUJdYV4=
 						IsDirectory: true,
 						Name:        "workflows",
 					},
+					{
+						IsDirectory: true,
+						Name:        "worker-models",
+					},
 				}
 				*(out.(*[]sdk.VCSContent)) = contents
 				return nil, 200, nil
@@ -2662,6 +2674,21 @@ GDFkaTe3nUJdYV4=
 				return nil, 200, nil
 			},
 		).MaxTimes(1)
+	servicesClients.EXPECT().
+		DoJSONRequest(gomock.Any(), "GET", "/vcs/vcs-server/repos/myrepo/contents/.cds%2Fworker-models?commit=zyxwv&offset=0&limit=100", gomock.Any(), gomock.Any(), gomock.Any()).
+		DoAndReturn(
+			func(ctx context.Context, method, path string, in interface{}, out interface{}, _ interface{}) (http.Header, int, error) {
+				contents := []sdk.VCSContent{
+					{
+						IsDirectory: false,
+						IsFile:      true,
+						Name:        "mymodel.yml",
+					},
+				}
+				*(out.(*[]sdk.VCSContent)) = contents
+				return nil, 200, nil
+			},
+		).MaxTimes(1)
 	servicesClients.EXPECT().
 		DoJSONRequest(gomock.Any(), "GET", "/vcs/vcs-server/repos/myrepo/content/.cds%2Fworkflows%2Fmyworkflow.yml?commit=zyxwv", gomock.Any(), gomock.Any(), gomock.Any()).
 		DoAndReturn(
@@ -2678,6 +2705,22 @@ GDFkaTe3nUJdYV4=
 			},
 		).MaxTimes(1)
 
+	servicesClients.EXPECT().
+		DoJSONRequest(gomock.Any(), "GET", "/vcs/vcs-server/repos/myrepo/content/.cds%2Fworker-models%2Fmymodel.yml?commit=zyxwv", gomock.Any(), gomock.Any(), gomock.Any()).
+		DoAndReturn(
+			func(ctx context.Context, method, path string, in interface{}, out interface{}, _ interface{}) (http.Header, int, error) {
+
+				content := sdk.VCSContent{
+					IsDirectory: false,
+					IsFile:      true,
+					Name:        "mymodel.yml",
+					Content:     encodedWorkerModel,
+				}
+				*(out.(*sdk.VCSContent)) = content
+				return nil, 200, nil
+			},
+		).MaxTimes(1)
+
 	servicesClients.EXPECT().DoJSONRequest(gomock.Any(), "GET", "/vcs/vcs-server/repos/myrepo/branches/?branch=&default=true&noCache=true", gomock.Any(), gomock.Any(), gomock.Any()).Times(1).
 		DoAndReturn(
 			func(ctx context.Context, method, path string, in interface{}, out interface{}, _ interface{}) (http.Header, int, error) {
@@ -2694,6 +2737,7 @@ GDFkaTe3nUJdYV4=
 
 	analysisUpdated, err := repository.LoadRepositoryAnalysisById(ctx, db, repo.ID, analysis.ID)
 	require.NoError(t, err)
+	t.Logf(">>>>%+v", analysisUpdated.Data.Error)
 	require.Equal(t, sdk.RepositoryAnalysisStatusSkipped, analysisUpdated.Status)
 
 	entitiesNonHEad, err := entity.LoadByTypeAndRefCommit(context.TODO(), db, repo.ID, sdk.EntityTypeWorkflow, "refs/heads/devBranch", "zyxwv")