feat(worker): add workflow command add run result (#5805)

cli/cdsctl/workflow_run_result.go

@@ -20,9 +20,9 @@ var workflowRunResultCmd = cli.Command{
 }
 
 type RunResultCli struct {
-	ID   string                    `cli:"id"`
-	Type sdk.WorkflowRunResultType `cli:"type"`
-	Name string                    `cli:"name"`
+	ID   string `cli:"id"`
+	Type string `cli:"type"`
+	Name string `cli:"name"`
 }
 
 func workflowRunResult() *cobra.Command {
@@ -223,6 +223,7 @@ func workflowRunResultList(v cli.Values) (cli.ListResult, error) {
 func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
 	cliresults := make([]RunResultCli, 0, len(results))
 	for _, r := range results {
+		artiType := string(r.Type)
 		var name string
 		switch r.Type {
 		case sdk.WorkflowRunResultTypeCoverage:
@@ -247,7 +248,7 @@ func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
 
 		cliresults = append(cliresults, RunResultCli{
 			ID:   r.ID,
-			Type: r.Type,
+			Type: artiType,
 			Name: name,
 		})
 	}

contrib/integrations/artifactory/plugin-artifactory-download-artifact/main.go

@@ -77,9 +77,9 @@ func createArtifactoryClient(url, token string) (artifactory.ArtifactoryServices
 }
 
 func (e *artifactoryDownloadArtifactPlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) {
-	cdsRepo := opts.GetOptions()["cds.integration.artifact_manager.artifactory.cds_repository"]
-	artifactoryURL := opts.GetOptions()["cds.integration.artifact_manager.artifactory.url"]
-	token := opts.GetOptions()["cds.integration.artifact_manager.artifactory.token"]
+	cdsRepo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigCdsRepository)]
+	artifactoryURL := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigURL)]
+	token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)]
 
 	filePath := opts.GetOptions()[sdk.ArtifactDownloadPluginInputFilePath]
 	path := opts.GetOptions()[sdk.ArtifactDownloadPluginInputDestinationPath]

contrib/integrations/artifactory/plugin-artifactory-download-artifact/plugin-artifactory-download-artifact.yml

@@ -1,5 +1,5 @@
 name: artifactory-download-artifact-plugin
 type: integration-download_artifact
-integration: Artifactory
+integration: ArtifactManager
 author: "OVH SAS"
 description: "OVH Artifactory Download Artifact Plugin"

contrib/integrations/artifactory/plugin-artifactory-upload-artifact/main.go

@@ -80,13 +80,13 @@ func (e *artifactoryUploadArtifactPlugin) createArtifactoryClient(url, token str
 }
 
 func (e *artifactoryUploadArtifactPlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) {
-	cdsRepo := opts.GetOptions()["cds.integration.artifact_manager.artifactory.cds_repository"]
-	artifactoryURL := opts.GetOptions()["cds.integration.artifact_manager.artifactory.url"]
-	token := opts.GetOptions()["cds.integration.artifact_manager.artifactory.token"]
+	cdsRepo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigCdsRepository)]
+	artifactoryURL := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigURL)]
+	token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)]
 	pathToUpload := opts.GetOptions()["cds.integration.artifact_manager.upload.path"]
 	projectKey := opts.GetOptions()["cds.project"]
 	workflowName := opts.GetOptions()["cds.workflow"]
-	buildInfo := opts.GetOptions()["cds.integration.artifact_manager.build.info.path"]
+	buildInfo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigBuildInfoPath)]
 	version := opts.GetOptions()["cds.version"]
 
 	artiClient, err := e.createArtifactoryClient(artifactoryURL, token)
@@ -104,7 +104,7 @@ func (e *artifactoryUploadArtifactPlugin) Run(_ context.Context, opts *integrati
 
 	summary, err := artiClient.UploadFilesWithSummary(params)
 	if err != nil || summary.TotalFailed > 0 {
-		return fail("unable to upload file %s into artifactory %s: %v", pathToUpload, params.Target, err)
+		return fail("unable to upload file %s into artifactory[%s] %s: %v", pathToUpload, artifactoryURL, params.Target, err)
 	}
 	defer summary.Close()
 

contrib/integrations/artifactory/plugin-artifactory-upload-artifact/plugin-artifactory-upload-artifact.yml

@@ -1,5 +1,5 @@
 name: artifactory-upload-artifact-plugin
 type: integration-upload_artifact
-integration: Artifactory
+integration: ArtifactManager
 author: "OVH SAS"
 description: "OVH Artifactory Upload Artifact Plugin"

engine/api/integration/artifact_manager/artifactory/client.go

@@ -0,0 +1,76 @@
+package artifactory
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/jfrog/jfrog-client-go/artifactory"
+	"github.com/jfrog/jfrog-client-go/artifactory/services"
+	"github.com/ovh/cds/sdk"
+)
+
+type FileInfoResponse struct {
+	Checksums         *FileInfoChecksum `json:"checksums"`
+	Created           time.Time         `json:"created"`
+	CreatedBy         string            `json:"createdBy"`
+	DownloadURI       string            `json:"downloadUri"`
+	LastModified      time.Time         `json:"lastModified"`
+	LastUpdated       time.Time         `json:"lastUpdated"`
+	MimeType          string            `json:"mimeType"`
+	ModifiedBy        string            `json:"modifiedBy"`
+	OriginalChecksums *FileInfoChecksum `json:"originalChecksums"`
+	Path              string            `json:"path"`
+	RemoteURL         string            `json:"remoteUrl"`
+	Repo              string            `json:"repo"`
+	Size              string            `json:"size"`
+	URI               string            `json:"uri"`
+}
+
+type FileInfoChecksum struct {
+	Md5    string `json:"md5"`
+	Sha1   string `json:"sha1"`
+	Sha256 string `json:"sha256"`
+}
+
+type Client struct {
+	Asm artifactory.ArtifactoryServicesManager
+}
+
+func (c *Client) GetFileInfo(repoName string, filePath string) (sdk.FileInfo, error) {
+	fi := sdk.FileInfo{}
+	repoDetails := services.RepositoryDetails{}
+	if err := c.Asm.GetRepository(repoName, &repoDetails); err != nil {
+		return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to get repository %s: %v", repoName, err)
+	}
+	fi.Type = repoDetails.PackageType
+
+	fileInfoURL := fmt.Sprintf("%sapi/storage/%s/%s", c.Asm.GetConfig().GetServiceDetails().GetUrl(), repoName, filePath)
+	httpDetails := c.Asm.GetConfig().GetServiceDetails().CreateHttpClientDetails()
+	re, body, _, err := c.Asm.Client().SendGet(fileInfoURL, true, &httpDetails)
+	if err != nil {
+		return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to call artifactory: %v", err)
+	}
+
+	if re.StatusCode >= 400 {
+		return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to call artifactory [HTTP: %d] %s", re.StatusCode, string(body))
+	}
+
+	var resp FileInfoResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fi, sdk.NewErrorFrom(sdk.ErrUnknownError, "unable to read artifactory response %s: %v", string(body), err)
+	}
+
+	if resp.Size != "" {
+		s, err := strconv.ParseInt(resp.Size, 10, 64)
+		if err != nil {
+			return fi, sdk.NewErrorFrom(sdk.ErrInvalidData, "size return by artifactory is not an integer %s: %v", resp.Size, err)
+		}
+		fi.Size = s
+	}
+	if resp.Checksums != nil {
+		fi.Md5 = resp.Checksums.Md5
+	}
+	return fi, nil
+}

engine/api/integration/artifact_manager/interface.go

@@ -0,0 +1,46 @@
+package artifact_manager
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/jfrog/jfrog-client-go/artifactory"
+	"github.com/jfrog/jfrog-client-go/artifactory/auth"
+	"github.com/jfrog/jfrog-client-go/config"
+
+	"github.com/jfrog/jfrog-client-go/utils/log"
+	arti "github.com/ovh/cds/engine/api/integration/artifact_manager/artifactory"
+	"github.com/ovh/cds/sdk"
+)
+
+type ArtifactManager interface {
+	GetFileInfo(repoName string, filePath string) (sdk.FileInfo, error)
+}
+
+func NewClient(managerType, url, token string) (ArtifactManager, error) {
+	switch managerType {
+	case "artifactory":
+		return newArtifactoryClient(url, token)
+	}
+	return nil, fmt.Errorf("artifact Manager %s not implemented", managerType)
+}
+
+func newArtifactoryClient(url string, token string) (ArtifactManager, error) {
+	log.SetLogger(log.NewLogger(log.INFO, os.Stdout))
+	rtDetails := auth.NewArtifactoryDetails()
+	rtDetails.SetUrl(url)
+	rtDetails.SetAccessToken(token)
+	serviceConfig, err := config.NewConfigBuilder().
+		SetServiceDetails(rtDetails).
+		SetThreads(1).
+		SetDryRun(false).
+		Build()
+	if err != nil {
+		return nil, fmt.Errorf("unable to create service config: %v", err)
+	}
+	asm, err := artifactory.New(serviceConfig)
+	if err != nil {
+		return nil, sdk.WrapError(err, "unable to create artifactory client")
+	}
+	return &arti.Client{Asm: asm}, nil
+}

engine/api/integration/builtin.go

@@ -16,6 +16,7 @@ var (
 		sdk.RabbitMQIntegration,
 		sdk.OpenstackIntegration,
 		sdk.AWSIntegration,
+		sdk.ArtifactManagerIntegration,
 	}
 )
 

engine/api/workflow/dao.go

@@ -984,6 +984,7 @@ func checkProjectIntegration(proj sdk.Project, w *sdk.Workflow, n *sdk.Node) err
 
 // checkIntegration checks integration data
 func checkIntegration(proj sdk.Project, w *sdk.Workflow) error {
+	countArtifactManagerIntegration := 0
 	for i := range w.Integrations {
 		workflowIntegration := &w.Integrations[i]
 		found := false
@@ -1000,6 +1001,13 @@ func checkIntegration(proj sdk.Project, w *sdk.Workflow) error {
 		if !found {
 			return sdk.WithData(sdk.ErrIntegrationtNotFound, workflowIntegration.ProjectIntegration.Name)
 		}
+		if workflowIntegration.ProjectIntegration.Model.ArtifactManager {
+			countArtifactManagerIntegration++
+		}
+	}
+
+	if countArtifactManagerIntegration > 1 {
+		return sdk.NewErrorFrom(sdk.ErrInvalidData, "you can't have multiple artifact manager integrations on a workflow")
 	}
 	return nil
 }

engine/api/workflow/dao_test.go

@@ -302,6 +302,15 @@ func TestUpdateWorkflowIntegration(t *testing.T) {
 	wfDb, err := workflow.LoadByID(context.TODO(), db, cache, *proj, w.ID, workflow.LoadOptions{})
 	require.NoError(t, err)
 	require.Equal(t, "newValue", wfDb.Integrations[0].Config["BuildInfo"].Value)
+
+	w.Integrations = append(w.Integrations, sdk.WorkflowProjectIntegration{
+
+		ProjectIntegration:   projInt,
+		ProjectIntegrationID: projInt.ID,
+	})
+	errUpdate := workflow.Update(context.TODO(), db, cache, *proj, &w, workflow.UpdateOptions{})
+	require.NotNil(t, errUpdate)
+	require.Contains(t, errUpdate.Error(), "you can't have multiple artifact manager integrations on a workflow")
 }
 
 func TestInsertComplexeWorkflowAndExport(t *testing.T) {

engine/api/workflow/execute_node_run.go

@@ -588,18 +588,23 @@ func getIntegrationPlugins(db gorp.SqlExecutor, wr *sdk.WorkflowRun, nr *sdk.Wor
 		plugins = append(plugins, *plugin)
 	}
 
-	var artifactManagerModelID int64
-	for _, int := range wr.Workflow.Integrations {
-		if int.ProjectIntegration.Model.ArtifactManager {
-			artifactManagerModelID = int.ProjectIntegration.Model.ID
+	var artifactManagerInteg *sdk.WorkflowProjectIntegration
+	for i := range wr.Workflow.Integrations {
+		if wr.Workflow.Integrations[i].ProjectIntegration.Model.ArtifactManager {
+			artifactManagerInteg = &wr.Workflow.Integrations[i]
 		}
 	}
-	if artifactManagerModelID != 0 {
-		plgs, err := plugin.LoadAllByIntegrationModelID(db, artifactManagerModelID)
+	if artifactManagerInteg != nil {
+		plgs, err := plugin.LoadAllByIntegrationModelID(db, artifactManagerInteg.ProjectIntegration.Model.ID)
 		if err != nil {
 			return nil, sdk.NewErrorFrom(sdk.ErrNotFound, "Cannot find plugin for integration model id %d, %v", projectIntegrationModelID, err)
 		}
-		plugins = append(plugins, plgs...)
+		platform := artifactManagerInteg.ProjectIntegration.Config[sdk.ArtifactManagerConfigPlatform]
+		for _, plg := range plgs {
+			if strings.HasPrefix(plg.Name, fmt.Sprintf("%s-", platform.Value)) {
+				plugins = append(plugins, plg)
+			}
+		}
 	}
 
 	return plugins, nil

engine/api/workflow/workflow_run_results.go

@@ -2,12 +2,15 @@ package workflow
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"strconv"
 	"time"
 
 	"github.com/go-gorp/gorp"
 
 	"github.com/ovh/cds/engine/api/database/gorpmapping"
+	"github.com/ovh/cds/engine/api/integration/artifact_manager"
 	"github.com/ovh/cds/engine/cache"
 	"github.com/ovh/cds/engine/gorpmapper"
 	"github.com/ovh/cds/sdk"
@@ -118,7 +121,7 @@ func CanUploadRunResult(ctx context.Context, db *gorp.DbMap, store cache.Store,
 	return true, nil
 }
 
-func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResult) error {
+func AddResult(ctx context.Context, db *gorp.DbMap, store cache.Store, wr *sdk.WorkflowRun, runResult *sdk.WorkflowRunResult) error {
 	var cacheKey string
 	switch runResult.Type {
 	case sdk.WorkflowRunResultTypeArtifact:
@@ -135,7 +138,7 @@ func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResu
 		}
 	case sdk.WorkflowRunResultTypeArtifactManager:
 		var err error
-		cacheKey, err = verifyAddResultArtifactManager(store, runResult)
+		cacheKey, err = verifyAddResultArtifactManager(ctx, db, store, wr, runResult)
 		if err != nil {
 			return err
 		}
@@ -158,14 +161,56 @@ func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResu
 	return sdk.WithStack(store.Delete(cacheKey))
 }
 
-func verifyAddResultArtifactManager(store cache.Store, runResult *sdk.WorkflowRunResult) (string, error) {
+// Check validity of the request + complete runResuklt with md5,size,type
+func verifyAddResultArtifactManager(ctx context.Context, db gorp.SqlExecutor, store cache.Store, wr *sdk.WorkflowRun, runResult *sdk.WorkflowRunResult) (string, error) {
 	artResult, err := runResult.GetArtifactManager()
 	if err != nil {
 		return "", err
 	}
+
+	// Check file in integration
+	var artiInteg *sdk.WorkflowProjectIntegration
+	for i := range wr.Workflow.Integrations {
+		if !wr.Workflow.Integrations[i].ProjectIntegration.Model.ArtifactManager {
+			continue
+		}
+		artiInteg = &wr.Workflow.Integrations[i]
+	}
+	if artiInteg == nil {
+		return "", sdk.NewErrorFrom(sdk.ErrInvalidData, "you cannot add a artifact manager run result without an integration")
+	}
+	secrets, err := loadRunSecretWithDecryption(ctx, db, wr.ID, []string{fmt.Sprintf(SecretProjIntegrationContext, artiInteg.ProjectIntegrationID)})
+	if err != nil {
+		return "", err
+	}
+	var artifactManagerToken string
+	for _, s := range secrets {
+		if s.Name == fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken) {
+			artifactManagerToken = s.Value
+			break
+		}
+	}
+	if artifactManagerToken == "" {
+		return "", sdk.NewErrorFrom(sdk.ErrNotFound, "unable to find artifact manager token")
+	}
+	artifactClient, err := artifact_manager.NewClient(artiInteg.ProjectIntegration.Config[sdk.ArtifactManagerConfigPlatform].Value, artiInteg.ProjectIntegration.Config[sdk.ArtifactManagerConfigURL].Value, artifactManagerToken)
+	if err != nil {
+		return "", err
+	}
+	fileInfo, err := artifactClient.GetFileInfo(artResult.RepoName, artResult.Path)
+	if err != nil {
+		return "", err
+	}
+	artResult.Size = fileInfo.Size
+	artResult.MD5 = fileInfo.Md5
+	artResult.RepoType = fileInfo.Type
+
 	if err := artResult.IsValid(); err != nil {
 		return "", err
 	}
+	dataUpdated, _ := json.Marshal(artResult)
+	runResult.DataRaw = dataUpdated
+
 	cacheKey := GetRunResultKey(runResult.WorkflowRunID, runResult.Type, artResult.Name)
 	b, err := store.Exist(cacheKey)
 	if err != nil {