fix(api,worker): handle multiple release maturity

contrib/integrations/artifactory/artifactory.go

@@ -211,32 +211,32 @@ func checkArtifactExists(artiClient artifactory.ArtifactoryServicesManager, repo
 }
 
 type executionContext struct {
-	buildInfo         string
-	projectKey        string
-	workflowName      string
-	version           string
-	lowMaturitySuffix string
+	buildInfo                string
+	projectKey               string
+	workflowName             string
+	version                  string
+	defaultLowMaturitySuffix string
 }
 
 type BuildInfoRequest struct {
-	BuildInfoPrefix   string
-	ProjectKey        string
-	WorkflowName      string
-	Version           string
-	AgentName         string
-	TokenName         string
-	RunURL            string
-	GitBranch         string
-	GitMessage        string
-	GitURL            string
-	GitHash           string
-	LowMaturitySuffix string
-	RunResults        []sdk.WorkflowRunResult
+	BuildInfoPrefix          string
+	ProjectKey               string
+	WorkflowName             string
+	Version                  string
+	AgentName                string
+	TokenName                string
+	RunURL                   string
+	GitBranch                string
+	GitMessage               string
+	GitURL                   string
+	GitHash                  string
+	DefaultLowMaturitySuffix string
+	RunResults               []sdk.WorkflowRunResult
 }
 
 func PrepareBuildInfo(ctx context.Context, artiClient artifact_manager.ArtifactManager, r BuildInfoRequest) (*buildinfo.BuildInfo, error) {
 	buildInfoName := fmt.Sprintf("%s/%s/%s", r.BuildInfoPrefix, r.ProjectKey, r.WorkflowName)
-	log.Debug(ctx, "PrepareBuildInfo %q maturity:%q", buildInfoName, r.LowMaturitySuffix)
+	log.Debug(ctx, "PrepareBuildInfo %q maturity:%q", buildInfoName, r.DefaultLowMaturitySuffix)
 
 	buildInfoRequest := &buildinfo.BuildInfo{
 		Properties: map[string]string{},
@@ -266,11 +266,11 @@ func PrepareBuildInfo(ctx context.Context, artiClient artifact_manager.ArtifactM
 	})
 
 	execContext := executionContext{
-		buildInfo:         r.BuildInfoPrefix,
-		lowMaturitySuffix: r.LowMaturitySuffix,
-		workflowName:      r.WorkflowName,
-		version:           r.Version,
-		projectKey:        r.ProjectKey,
+		buildInfo:                r.BuildInfoPrefix,
+		defaultLowMaturitySuffix: r.DefaultLowMaturitySuffix,
+		workflowName:             r.WorkflowName,
+		version:                  r.Version,
+		projectKey:               r.ProjectKey,
 	}
 	modules, err := computeBuildInfoModules(ctx, artiClient, execContext, r.RunResults)
 	if err != nil {
@@ -287,6 +287,18 @@ func computeBuildInfoModules(ctx context.Context, client artifact_manager.Artifa
 		if r.Type != sdk.WorkflowRunResultTypeArtifactManager {
 			continue
 		}
+
+		var currentMaturity string
+		if r.DataSync != nil {
+			latestPromotion := r.DataSync.LatestPromotionOrRelease()
+			if latestPromotion != nil {
+				currentMaturity = latestPromotion.ToMaturity
+			}
+		}
+		if currentMaturity == "" {
+			currentMaturity = execContext.defaultLowMaturitySuffix
+		}
+
 		data, err := r.GetArtifactManager()
 		if err != nil {
 			return nil, err
@@ -312,7 +324,7 @@ func computeBuildInfoModules(ctx context.Context, client artifact_manager.Artifa
 			props := make(map[string]string)
 			parsedUrl, err := url.Parse(client.GetURL())
 			if err != nil {
-				return nil, fmt.Errorf("unable to parse artifactory url [%s]: %v", client.GetURL(), err)
+				return nil, sdk.WrapError(err, "unable to parse artifactory url [%s]: %v", client.GetURL())
 			}
 			urlArtifactory := parsedUrl.Host
 			if parsedUrl.Port() != "" {
@@ -322,7 +334,7 @@ func computeBuildInfoModules(ctx context.Context, client artifact_manager.Artifa
 			mod.Properties = props
 		}
 
-		artifacts, err := retrieveModulesArtifacts(ctx, client, data.RepoName, data.Path, execContext)
+		artifacts, err := retrieveModulesArtifacts(ctx, client, data.RepoName, currentMaturity, data.Path, execContext)
 		if err != nil {
 			return nil, err
 		}
@@ -333,7 +345,7 @@ func computeBuildInfoModules(ctx context.Context, client artifact_manager.Artifa
 	return modules, nil
 }
 
-func retrieveModulesArtifacts(ctx context.Context, client artifact_manager.ArtifactManager, repoName string, path string, execContext executionContext) ([]buildinfo.Artifact, error) {
+func retrieveModulesArtifacts(ctx context.Context, client artifact_manager.ArtifactManager, repoName string, maturity string, path string, execContext executionContext) ([]buildinfo.Artifact, error) {
 	log.Debug(ctx, "retrieve:ModulesArtifacts repoName:%s path:%s execContext:%+v", repoName, path, execContext)
 	fileInfo, err := client.GetFileInfo(repoName, path)
 	if err != nil {
@@ -358,7 +370,7 @@ func retrieveModulesArtifacts(ctx context.Context, client artifact_manager.Artif
 			},
 		}
 		repoSrc := repoName
-		repoSrc += "-" + execContext.lowMaturitySuffix
+		repoSrc += "-" + maturity
 		log.Debug(ctx, "setting properties %+v on repoSrc:%s path:%s", props, repoSrc, props)
 		if err := client.SetProperties(repoSrc, path, props...); err != nil {
 			return nil, err
@@ -374,7 +386,7 @@ func retrieveModulesArtifacts(ctx context.Context, client artifact_manager.Artif
 		artifacts = append(artifacts, currentArtifact)
 	} else {
 		for _, c := range fileInfo.Children {
-			artsChildren, err := retrieveModulesArtifacts(ctx, client, repoName, fmt.Sprintf("%s%s", path, c.Uri), execContext)
+			artsChildren, err := retrieveModulesArtifacts(ctx, client, repoName, maturity, fmt.Sprintf("%s%s", path, c.Uri), execContext)
 			if err != nil {
 				return nil, err
 			}

contrib/integrations/artifactory/plugin-artifactory-build-info/main.go

@@ -40,11 +40,11 @@ type artifactoryBuildInfoPlugin struct {
 }
 
 type executionContext struct {
-	buildInfo         string
-	projectKey        string
-	workflowName      string
-	version           string
-	lowMaturitySuffix string
+	buildInfo                string
+	projectKey               string
+	workflowName             string
+	version                  string
+	defaultLowMaturitySuffix string
 }
 
 func (e *artifactoryBuildInfoPlugin) Manifest(_ context.Context, _ *empty.Empty) (*integrationplugin.IntegrationPluginManifest, error) {
@@ -91,19 +91,19 @@ func (e *artifactoryBuildInfoPlugin) Run(ctx context.Context, opts *integrationp
 	}
 
 	buildInfoRequest, err := art.PrepareBuildInfo(ctx, artifactClient, art.BuildInfoRequest{
-		BuildInfoPrefix:   buildInfo,
-		ProjectKey:        opts.GetOptions()["cds.project"],
-		WorkflowName:      opts.GetOptions()["cds.workflow"],
-		Version:           opts.GetOptions()["cds.version"],
-		AgentName:         workerName,
-		TokenName:         tokenName,
-		RunURL:            runURL,
-		GitBranch:         opts.GetOptions()["git.branch"],
-		GitMessage:        opts.GetOptions()["git.message"],
-		GitURL:            gitUrl,
-		GitHash:           opts.GetOptions()["git.hash"],
-		RunResults:        runResults,
-		LowMaturitySuffix: lowMaturitySuffix,
+		BuildInfoPrefix:          buildInfo,
+		ProjectKey:               opts.GetOptions()["cds.project"],
+		WorkflowName:             opts.GetOptions()["cds.workflow"],
+		Version:                  opts.GetOptions()["cds.version"],
+		AgentName:                workerName,
+		TokenName:                tokenName,
+		RunURL:                   runURL,
+		GitBranch:                opts.GetOptions()["git.branch"],
+		GitMessage:               opts.GetOptions()["git.message"],
+		GitURL:                   gitUrl,
+		GitHash:                  opts.GetOptions()["git.hash"],
+		RunResults:               runResults,
+		DefaultLowMaturitySuffix: lowMaturitySuffix,
 	})
 	if err != nil {
 		return fail("unable to prepare build info: %v", err)

contrib/integrations/artifactory/plugin-artifactory-promote/main.go

@@ -54,15 +54,6 @@ func (e *artifactoryPromotePlugin) Run(ctx context.Context, opts *integrationplu
 	token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactoryConfigToken)]
 
 	artifactList := opts.GetOptions()["artifacts"]
-	srcMaturity := opts.GetOptions()["srcMaturity"]
-	destMaturity := opts.GetOptions()["destMaturity"]
-
-	if srcMaturity == "" {
-		srcMaturity = "snapshot"
-	}
-	if destMaturity == "" {
-		destMaturity = "release"
-	}
 
 	runResult, err := grpcplugins.GetRunResults(e.HTTPPort)
 	if err != nil {
@@ -89,7 +80,6 @@ func (e *artifactoryPromotePlugin) Run(ctx context.Context, opts *integrationplu
 		artRegs = append(artRegs, r)
 	}
 
-	promotedArtifacts := make([]string, 0)
 	for _, r := range runResult {
 		rData, err := r.GetArtifactManager()
 		if err != nil {
@@ -105,17 +95,22 @@ func (e *artifactoryPromotePlugin) Run(ctx context.Context, opts *integrationplu
 		if skip {
 			continue
 		}
+		if r.DataSync == nil {
+			return fail("unable to find an existing promotion for result %s", r.ID)
+		}
+		latestPromotion := r.DataSync.LatestPromotionOrRelease()
+		if latestPromotion == nil {
+			return fail("unable to find an existing promotion for result %s", r.ID)
+		}
 		switch rData.RepoType {
 		case "docker":
-			if err := art.PromoteDockerImage(artiClient, rData, srcMaturity, destMaturity); err != nil {
-				return fail("unable to promote docker image: %s: %v", rData.Name+"-"+destMaturity, err)
+			if err := art.PromoteDockerImage(artiClient, rData, latestPromotion.FromMaturity, latestPromotion.ToMaturity); err != nil {
+				return fail("unable to promote docker image: %s: %v", rData.Name+"-"+latestPromotion.ToMaturity, err)
 			}
-			promotedArtifacts = append(promotedArtifacts, fmt.Sprintf("%s-%s/%s/manifest.json", rData.RepoName, destMaturity, rData.Path))
 		default:
-			if err := art.PromoteFile(artiClient, rData, srcMaturity, destMaturity); err != nil {
+			if err := art.PromoteFile(artiClient, rData, latestPromotion.FromMaturity, latestPromotion.ToMaturity); err != nil {
 				return fail("unable to promote file: %s: %v", rData.Name, err)
 			}
-			promotedArtifacts = append(promotedArtifacts, fmt.Sprintf("%s-%s/%s", rData.RepoName, destMaturity, rData.Path))
 		}
 	}
 	return &integrationplugin.RunResult{

contrib/integrations/artifactory/plugin-artifactory-release/main.go

@@ -71,12 +71,7 @@ func (e *artifactoryReleasePlugin) Run(ctx context.Context, opts *integrationplu
 
 	artifactList := opts.GetOptions()["artifacts"]
 	releaseNote := opts.GetOptions()["releaseNote"]
-	srcMaturity := opts.GetOptions()["srcMaturity"]
 	destMaturity := opts.GetOptions()["destMaturity"]
-
-	if srcMaturity == "" {
-		srcMaturity = "snapshot"
-	}
 	if destMaturity == "" {
 		destMaturity = DefaultHighMaturity
 	}
@@ -149,19 +144,25 @@ func (e *artifactoryReleasePlugin) Run(ctx context.Context, opts *integrationplu
 		} else {
 			fmt.Printf("Result %q to promote\n", name)
 		}
+		if r.DataSync == nil {
+			return fail("unable to find an existing release for result %s", r.ID)
+		}
+		latestPromotion := r.DataSync.LatestPromotionOrRelease()
+		if latestPromotion == nil {
+			return fail("unable to find an existing release for result %s", r.ID)
+		}
 		switch rData.RepoType {
 		case "docker":
-			if err := art.PromoteDockerImage(artiClient, rData, srcMaturity, destMaturity); err != nil {
-				return fail("unable to promote docker image: %s: %v", rData.Name+"-"+destMaturity, err)
+			if err := art.PromoteDockerImage(artiClient, rData, latestPromotion.FromMaturity, latestPromotion.ToMaturity); err != nil {
+				return fail("unable to promote docker image: %s: %v", rData.Name+"-"+latestPromotion.ToMaturity, err)
 			}
-			promotedArtifacts = append(promotedArtifacts, fmt.Sprintf("%s-%s/%s/manifest.json", rData.RepoName, destMaturity, rData.Path))
+			promotedArtifacts = append(promotedArtifacts, fmt.Sprintf("%s-%s/%s/manifest.json", rData.RepoName, latestPromotion.ToMaturity, rData.Path))
 		default:
-			if err := art.PromoteFile(artiClient, rData, srcMaturity, destMaturity); err != nil {
+			if err := art.PromoteFile(artiClient, rData, latestPromotion.FromMaturity, latestPromotion.ToMaturity); err != nil {
 				return fail("unable to promote file: %s: %v", rData.Name, err)
 			}
-			promotedArtifacts = append(promotedArtifacts, fmt.Sprintf("%s-%s/%s", rData.RepoName, destMaturity, rData.Path))
+			promotedArtifacts = append(promotedArtifacts, fmt.Sprintf("%s-%s/%s", rData.RepoName, latestPromotion.ToMaturity, rData.Path))
 		}
-
 	}
 
 	if len(promotedArtifacts) == 0 {

engine/api/api.go

@@ -745,13 +745,13 @@ func (a *API) Serve(ctx context.Context) error {
 		a.WorkflowRunCraft(ctx, 100*time.Millisecond)
 	})
 	a.GoRoutines.RunWithRestart(ctx, "api.repositoryAnalysisPoller", func(ctx context.Context) {
-		a.repositoryAnalysisPoller(ctx, 1*time.Second)
+		a.repositoryAnalysisPoller(ctx, 5*time.Second)
 	})
 	a.GoRoutines.RunWithRestart(ctx, "api.cleanRepositoryAnalysis", func(ctx context.Context) {
 		a.cleanRepositoryAnalysis(ctx, 1*time.Hour)
 	})
 	a.GoRoutines.RunWithRestart(ctx, "workflow.ResyncWorkflowRunResultsRoutine", func(ctx context.Context) {
-		workflow.ResyncWorkflowRunResultsRoutine(ctx, a.mustDB)
+		workflow.ResyncWorkflowRunResultsRoutine(ctx, a.mustDB, 5*time.Second)
 	})
 	if a.Config.Secrets.SnapshotRetentionDelay > 0 {
 		a.GoRoutines.RunWithRestart(ctx, "workflow.CleanSecretsSnapshot", func(ctx context.Context) {