diff --git a/contrib/integrations/artifactory/plugin-artifactory-release/main.go b/contrib/integrations/artifactory/plugin-artifactory-release/main.go index e6e0ee506c..9df8502223 100644 --- a/contrib/integrations/artifactory/plugin-artifactory-release/main.go +++ b/contrib/integrations/artifactory/plugin-artifactory-release/main.go @@ -120,6 +120,10 @@ func (e *artifactoryReleasePlugin) Run(_ context.Context, opts *integrationplugi promotedArtifacts := make([]string, 0) for _, r := range runResult { + // static-file type does not need to be released + if r.Type == sdk.WorkflowRunResultTypeStaticFile { + continue + } rData, err := r.GetArtifactManager() if err != nil { return fail("unable to read result %s: %v", r.ID, err) diff --git a/contrib/integrations/artifactory/plugin-artifactory-upload-artifact/main.go b/contrib/integrations/artifactory/plugin-artifactory-upload-artifact/main.go index f25887cc8d..70519a6eef 100644 --- a/contrib/integrations/artifactory/plugin-artifactory-upload-artifact/main.go +++ b/contrib/integrations/artifactory/plugin-artifactory-upload-artifact/main.go @@ -63,15 +63,15 @@ func (e *artifactoryUploadArtifactPlugin) Manifest(_ context.Context, _ *empty.E } func (e *artifactoryUploadArtifactPlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) { - cdsRepo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigCdsRepository)] - artifactoryURL := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigURL)] - token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)] - pathToUpload := opts.GetOptions()["cds.integration.artifact_manager.upload.path"] + prefix := "cds.integration.artifact_manager" + cdsRepo := opts.GetOptions()[fmt.Sprintf("%s.%s", prefix, sdk.ArtifactManagerConfigCdsRepository)] + artifactoryURL := opts.GetOptions()[fmt.Sprintf("%s.%s", prefix, sdk.ArtifactManagerConfigURL)] + token := opts.GetOptions()[fmt.Sprintf("%s.%s", prefix, sdk.ArtifactManagerConfigToken)] + pathToUpload := opts.GetOptions()[fmt.Sprintf("%s.upload.path", prefix)] projectKey := opts.GetOptions()["cds.project"] workflowName := opts.GetOptions()["cds.workflow"] version := opts.GetOptions()["cds.version"] - - buildInfo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigBuildInfoPrefix)] + buildInfo := opts.GetOptions()[fmt.Sprintf("%s.%s", prefix, sdk.ArtifactManagerConfigBuildInfoPrefix)] artiClient, err := art.CreateArtifactoryClient(artifactoryURL, token) if err != nil { diff --git a/engine/api/workflow/dao_node_run.go b/engine/api/workflow/dao_node_run.go index 299e042825..75bb990106 100644 --- a/engine/api/workflow/dao_node_run.go +++ b/engine/api/workflow/dao_node_run.go @@ -766,7 +766,7 @@ func PreviousNodeRunVCSInfos(ctx context.Context, db gorp.SqlExecutor, projectKe argPrevious = append(argPrevious, envID) queryPrevious += "AND w_node_context.environment_id = $5" } - queryPrevious += fmt.Sprintf(" ORDER BY workflow_node_run.num DESC LIMIT 1") + queryPrevious += " ORDER BY workflow_node_run.num DESC LIMIT 1" errPrev := db.QueryRow(queryPrevious, argPrevious...).Scan(&prevBranch, &prevTag, &prevHash, &prevRepository, &previousBuildNumber) if errPrev == sql.ErrNoRows { diff --git a/engine/api/workflow/workflow_run_event_test.go b/engine/api/workflow/workflow_run_event_test.go index 6e15698d2b..262f8508ba 100644 --- a/engine/api/workflow/workflow_run_event_test.go +++ b/engine/api/workflow/workflow_run_event_test.go @@ -91,6 +91,7 @@ func TestResyncCommitStatusNotifDisabled(t *testing.T) { } allSrv, err := services.LoadAll(context.TODO(), db) + assert.NoError(t, err) for _, s := range allSrv { if err := services.Delete(db, &s); err != nil { t.Fatalf("unable to delete service: %v", err) diff --git a/engine/api/workflow/workflow_run_results.go b/engine/api/workflow/workflow_run_results.go index baa73f2b2f..52b7089b40 100644 --- a/engine/api/workflow/workflow_run_results.go +++ b/engine/api/workflow/workflow_run_results.go @@ -91,6 +91,12 @@ func CanUploadRunResult(ctx context.Context, db *gorp.DbMap, store cache.Store, return false, err } fileName = refArt.Name + case sdk.WorkflowRunResultTypeStaticFile: + refArt, err := result.GetStaticFile() + if err != nil { + return false, err + } + fileName = refArt.Name } if fileName != runResultCheck.Name { @@ -142,8 +148,14 @@ func AddResult(ctx context.Context, db *gorp.DbMap, store cache.Store, wr *sdk.W if err != nil { return err } + case sdk.WorkflowRunResultTypeStaticFile: + var err error + cacheKey, err = verifyAddResultStaticFile(store, runResult) + if err != nil { + return err + } default: - return sdk.WrapError(sdk.ErrInvalidData, "unkonwn result type %s", runResult.Type) + return sdk.WrapError(sdk.ErrInvalidData, "unknown result type %s", runResult.Type) } tx, err := db.Begin() @@ -262,6 +274,26 @@ func verifyAddResultArtifact(store cache.Store, runResult *sdk.WorkflowRunResult return cacheKey, nil } +func verifyAddResultStaticFile(store cache.Store, runResult *sdk.WorkflowRunResult) (string, error) { + staticFileRunResult, err := runResult.GetStaticFile() + if err != nil { + return "", err + } + if err := staticFileRunResult.IsValid(); err != nil { + return "", err + } + + cacheKey := GetRunResultKey(runResult.WorkflowRunID, runResult.Type, staticFileRunResult.Name) + b, err := store.Exist(cacheKey) + if err != nil { + return cacheKey, err + } + if !b { + return cacheKey, sdk.WrapError(sdk.ErrForbidden, "unable to upload an unchecked static-file") + } + return cacheKey, nil +} + func insertResult(tx gorpmapper.SqlExecutorWithTx, runResult *sdk.WorkflowRunResult) error { runResult.ID = sdk.UUID() runResult.Created = time.Now() diff --git a/engine/api/workflow/workflow_run_results_test.go b/engine/api/workflow/workflow_run_results_test.go index 9840bef031..ad2de2f69d 100644 --- a/engine/api/workflow/workflow_run_results_test.go +++ b/engine/api/workflow/workflow_run_results_test.go @@ -132,6 +132,7 @@ func TestCanUploadArtifactAlreadyExist(t *testing.T) { Perm: 0777, } bts, err := json.Marshal(artiData) + require.NoError(t, err) result.DataRaw = bts cacheKey := workflow.GetRunResultKey(result.WorkflowRunID, sdk.WorkflowRunResultTypeArtifact, artiData.Name) @@ -177,6 +178,7 @@ func TestCanUploadArtifactAlreadyExistInMoreRecentSubNum(t *testing.T) { Perm: 0777, } bts, err := json.Marshal(artiData) + require.NoError(t, err) result.DataRaw = bts cacheKey := workflow.GetRunResultKey(result.WorkflowRunID, sdk.WorkflowRunResultTypeArtifact, artiData.Name) @@ -235,6 +237,7 @@ func TestCanUploadArtifactAlreadyExistInAPreviousSubNum(t *testing.T) { Perm: 0777, } bts, err := json.Marshal(artiData) + require.NoError(t, err) result.DataRaw = bts cacheKey := workflow.GetRunResultKey(result.WorkflowRunID, sdk.WorkflowRunResultTypeArtifact, artiData.Name) @@ -247,3 +250,76 @@ func TestCanUploadArtifactAlreadyExistInAPreviousSubNum(t *testing.T) { _, err = workflow.CanUploadRunResult(ctx, db.DbMap, store, workflowRun, artifactRef) require.NoError(t, err) } + +func TestCanUploadStaticFile(t *testing.T) { + ctx := context.Background() + db, store := test.SetupPG(t) + + _, _, workflowRun, nodeRun, jobRun := createRunNodeRunAndJob(t, db, store) + + staticFileRef := sdk.WorkflowRunResultCheck{ + RunJobID: jobRun.ID, + RunNodeID: nodeRun.ID, + RunID: workflowRun.ID, + Name: "my title static file", + ResultType: sdk.WorkflowRunResultTypeStaticFile, + } + + result := sdk.WorkflowRunResult{ + ID: sdk.UUID(), + Created: time.Now(), + WorkflowNodeRunID: nodeRun.ID, + WorkflowRunID: workflowRun.ID, + SubNum: 0, + WorkflowRunJobID: jobRun.ID + 1, + Type: sdk.WorkflowRunResultTypeStaticFile, + } + artiData := sdk.WorkflowRunResultStaticFile{ + Name: "my title static file", + RemoteURL: "https://foo/bar", + } + bts, err := json.Marshal(artiData) + require.NoError(t, err) + result.DataRaw = bts + + cacheKey := workflow.GetRunResultKey(result.WorkflowRunID, sdk.WorkflowRunResultTypeStaticFile, artiData.Name) + require.NoError(t, store.SetWithTTL(cacheKey, true, 60)) + require.NoError(t, workflow.AddResult(ctx, db.DbMap, store, &workflowRun, &result)) + b, err := store.Exist(cacheKey) + require.NoError(t, err) + require.False(t, b) + + _, err = workflow.CanUploadRunResult(ctx, db.DbMap, store, workflowRun, staticFileRef) + require.True(t, sdk.ErrorIs(err, sdk.ErrConflictData)) + require.Contains(t, err.Error(), "artifact my title static file has already been uploaded") +} +func TestCanUploadStaticFileInvalid(t *testing.T) { + ctx := context.Background() + db, store := test.SetupPG(t) + + _, _, workflowRun, nodeRun, jobRun := createRunNodeRunAndJob(t, db, store) + + result := sdk.WorkflowRunResult{ + ID: sdk.UUID(), + Created: time.Now(), + WorkflowNodeRunID: nodeRun.ID, + WorkflowRunID: workflowRun.ID, + SubNum: 0, + WorkflowRunJobID: jobRun.ID + 1, + Type: sdk.WorkflowRunResultTypeStaticFile, + } + artiData := sdk.WorkflowRunResultStaticFile{ + Name: "my title static file", + RemoteURL: "", + } + bts, err := json.Marshal(artiData) + require.NoError(t, err) + result.DataRaw = bts + + cacheKey := workflow.GetRunResultKey(result.WorkflowRunID, sdk.WorkflowRunResultTypeStaticFile, artiData.Name) + require.NoError(t, store.SetWithTTL(cacheKey, true, 60)) + + err = workflow.AddResult(ctx, db.DbMap, store, &workflowRun, &result) + require.Contains(t, err.Error(), "missing remote url") + require.Error(t, err) +} diff --git a/engine/api/workflow_run.go b/engine/api/workflow_run.go index e7c52c2297..eb0f1ed692 100644 --- a/engine/api/workflow_run.go +++ b/engine/api/workflow_run.go @@ -1121,6 +1121,20 @@ func (api *API) initWorkflowRun(ctx context.Context, projKey string, wf *sdk.Wor } workflow.ResyncNodeRunsWithCommits(ctx, api.mustDB(), api.Cache, *p, report) + api.initWorkflowRunPurge(ctx, wf) + + // Update parent + for i := range report.WorkflowRuns() { + run := &report.WorkflowRuns()[i] + if err := api.updateParentWorkflowRun(ctx, run); err != nil { + log.Error(ctx, "unable to update parent workflow run: %v", err) + } + } + + return report +} + +func (api *API) initWorkflowRunPurge(ctx context.Context, wf *sdk.Workflow) { _, enabled := featureflipping.IsEnabled(ctx, gorpmapping.Mapper, api.mustDB(), sdk.FeaturePurgeName, map[string]string{"project_key": wf.ProjectKey}) if !enabled { // Purge workflow run @@ -1142,16 +1156,6 @@ func (api *API) initWorkflowRun(ctx context.Context, projKey string, wf *sdk.Wor workflow.CountWorkflowRunsMarkToDelete(ctx, api.mustDB(), api.Metrics.WorkflowRunsMarkToDelete) }) } - - // Update parent - for i := range report.WorkflowRuns() { - run := &report.WorkflowRuns()[i] - if err := api.updateParentWorkflowRun(ctx, run); err != nil { - log.Error(ctx, "unable to update parent workflow run: %v", err) - } - } - - return report } func saveWorkflowRunSecrets(ctx context.Context, db *gorp.DbMap, projID int64, wr sdk.WorkflowRun, secrets *workflow.PushSecrets) error { diff --git a/engine/worker/cmd_cds_version_set.go b/engine/worker/cmd_cds_version_set.go index cfc6982a32..1701663888 100644 --- a/engine/worker/cmd_cds_version_set.go +++ b/engine/worker/cmd_cds_version_set.go @@ -17,10 +17,6 @@ import ( "github.com/ovh/cds/sdk" ) -var ( - cmdCDSSetVersionValue string -) - func cmdCDSVersionSet() *cobra.Command { c := &cobra.Command{ Use: "set-version", diff --git a/engine/worker/cmd_run_result.go b/engine/worker/cmd_run_result.go index 619f7351b7..264e8de7cb 100644 --- a/engine/worker/cmd_run_result.go +++ b/engine/worker/cmd_run_result.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "strconv" @@ -32,9 +33,49 @@ func cdmAddRunResult() *cobra.Command { Long: `Inside a job, add a run result`, } c.AddCommand(cmdRunResultAddArtifactIntegration()) + c.AddCommand(cmdRunResultAddStaticFile()) return c } +func cmdRunResultAddStaticFile() *cobra.Command { + c := &cobra.Command{ + Use: "static-file", + Short: "worker run-result add static-file ", + Long: `Inside a job, add a run result of type static-file: +Worker Command: + + worker run-result add static-file + +Example: + + worker run-result add static-file the-title https://the-remote-url/somewhere/index.html +`, + Run: addStaticFileRunResultCmd(), + } + return c +} + +func addStaticFileRunResultCmd() func(cmd *cobra.Command, args []string) { + return func(cmd *cobra.Command, args []string) { + if len(args) != 2 { + sdk.Exit("missing arguments. Cmd: worker run-result add static-file ") + } + + name := args[0] + remoteURL, err := url.Parse(args[1]) + if err != nil { + sdk.Exit("remote url invalid:%v url:%v", err, remoteURL) + } + + payload := sdk.WorkflowRunResultStaticFile{ + Name: name, + RemoteURL: remoteURL.String(), + } + data, _ := json.Marshal(payload) + addRunResult(data, sdk.WorkflowRunResultTypeStaticFile) + } +} + func cmdRunResultAddArtifactIntegration() *cobra.Command { c := &cobra.Command{ Use: "artifact-manager", @@ -63,16 +104,6 @@ func addArtifactManagerRunResultCmd() func(cmd *cobra.Command, args []string) { repositoryName := args[1] filePath := args[2] - portS := os.Getenv(internal.WorkerServerPort) - if portS == "" { - sdk.Exit("%s not found, are you running inside a CDS worker job?\n", internal.WorkerServerPort) - } - - port, errPort := strconv.Atoi(portS) - if errPort != nil { - sdk.Exit("cannot parse '%s' as a port number", portS) - } - payload := sdk.WorkflowRunResultArtifactManager{ Name: fileName, Perm: 0, @@ -81,35 +112,49 @@ func addArtifactManagerRunResultCmd() func(cmd *cobra.Command, args []string) { } data, _ := json.Marshal(payload) - req, errRequest := http.NewRequest("POST", fmt.Sprintf("http://127.0.0.1:%d/run-result/add", port), bytes.NewBuffer(data)) - if errRequest != nil { - sdk.Exit("cannot add run result (Request): %s\n", errRequest) - } - client := http.DefaultClient - resp, errDo := client.Do(req) - if errDo != nil { - sdk.Exit("cannot post worker artifacts (Do): %s\n", errDo) - } - defer resp.Body.Close() // nolint - - if resp.StatusCode >= 300 { - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - sdk.Exit("cannot add run result HTTP %v\n", err) - } - cdsError := sdk.DecodeError(body) - if cdsError != nil { - sdk.Exit("adding run result failed: %v\n", cdsError) - } else { - sdk.Exit("adding run result failed: %s\n", body) - } - } + addRunResult(data, sdk.WorkflowRunResultTypeArtifactManager) + } +} - // step: read the response body - respBody, err := ioutil.ReadAll(resp.Body) +func addRunResult(data []byte, stype sdk.WorkflowRunResultType) { + portS := os.Getenv(internal.WorkerServerPort) + if portS == "" { + sdk.Exit("%s not found, are you running inside a CDS worker job?\n", internal.WorkerServerPort) + } + + port, errPort := strconv.Atoi(portS) + if errPort != nil { + sdk.Exit("cannot parse '%s' as a port number", portS) + } + + req, errRequest := http.NewRequest("POST", fmt.Sprintf("http://127.0.0.1:%d/run-result/add/%s", port, stype), bytes.NewBuffer(data)) + if errRequest != nil { + sdk.Exit("cannot add run result (Request): %s\n", errRequest) + } + client := http.DefaultClient + resp, errDo := client.Do(req) + if errDo != nil { + sdk.Exit("cannot post worker run-result (Do): %s\n", errDo) + } + defer resp.Body.Close() // nolint + + if resp.StatusCode >= 300 { + body, err := ioutil.ReadAll(resp.Body) if err != nil { - sdk.Exit("add run result failed ReadAll: %v\n", err) + sdk.Exit("cannot add run result HTTP %v\n", err) + } + cdsError := sdk.DecodeError(body) + if cdsError != nil { + sdk.Exit("adding run result failed: %v\n", cdsError) + } else { + sdk.Exit("adding run result failed: %s\n", body) } - fmt.Println(string(respBody)) } + + // step: read the response body + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + sdk.Exit("add run result failed ReadAll: %v\n", err) + } + fmt.Println(string(respBody)) } diff --git a/engine/worker/internal/action/builtin_artifact_download.go b/engine/worker/internal/action/builtin_artifact_download.go index 27f13961e2..80ee92e9e8 100644 --- a/engine/worker/internal/action/builtin_artifact_download.go +++ b/engine/worker/internal/action/builtin_artifact_download.go @@ -200,7 +200,6 @@ func GetArtifactFromIntegrationPlugin(ctx context.Context, wk workerruntime.Runt res.Status = sdk.StatusFail res.Reason = err.Error() } - return }(opts) // Be kind with the artifact manager if len(runResults) > 1 { @@ -260,7 +259,7 @@ func runGRPCIntegrationPlugin(ctx context.Context, wk workerruntime.Runtime, bin return fmt.Errorf("error deploying application: %v", err) } - if strings.ToUpper(result.Status) != strings.ToUpper(sdk.StatusSuccess) { + if !strings.EqualFold(result.Status, sdk.StatusSuccess) { return fmt.Errorf("plugin execution failed %s: %s", result.Status, result.Details) } return nil diff --git a/engine/worker/internal/action/builtin_deploy_application.go b/engine/worker/internal/action/builtin_deploy_application.go index 7c7a0b6057..f8c4595715 100644 --- a/engine/worker/internal/action/builtin_deploy_application.go +++ b/engine/worker/internal/action/builtin_deploy_application.go @@ -77,7 +77,7 @@ func RunDeployApplication(ctx context.Context, wk workerruntime.Runtime, _ sdk.A wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Details: %s", res.Details)) wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Status: %s", res.Status)) - if strings.ToUpper(res.Status) == strings.ToUpper(sdk.StatusSuccess) { + if strings.EqualFold(res.Status, sdk.StatusSuccess) { integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) return sdk.Result{ Status: sdk.StatusSuccess, diff --git a/engine/worker/internal/action/builtin_key_install_test.go b/engine/worker/internal/action/builtin_key_install_test.go index 58e812df7e..e8fccaa1a8 100644 --- a/engine/worker/internal/action/builtin_key_install_test.go +++ b/engine/worker/internal/action/builtin_key_install_test.go @@ -116,7 +116,7 @@ func TestRunInstallKeyAction_Absolute(t *testing.T) { }, } secrets := []sdk.Variable{ - sdk.Variable{ + { ID: 1, Name: "cds.key.proj-mykey.priv", Value: "test", diff --git a/engine/worker/internal/action/builtin_promote.go b/engine/worker/internal/action/builtin_promote.go index 285aa75f07..ad0d2577ba 100644 --- a/engine/worker/internal/action/builtin_promote.go +++ b/engine/worker/internal/action/builtin_promote.go @@ -86,7 +86,7 @@ func RunPromote(ctx context.Context, wk workerruntime.Runtime, a sdk.Action, _ [ wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Details: %s", res.Details)) wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Status: %s", res.Status)) - if strings.ToUpper(res.Status) == strings.ToUpper(sdk.StatusSuccess) { + if strings.EqualFold(res.Status, sdk.StatusSuccess) { integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) return sdk.Result{ Status: sdk.StatusSuccess, diff --git a/engine/worker/internal/action/builtin_release.go b/engine/worker/internal/action/builtin_release.go index 85941b45b8..c6e2482b9e 100644 --- a/engine/worker/internal/action/builtin_release.go +++ b/engine/worker/internal/action/builtin_release.go @@ -86,7 +86,7 @@ func RunRelease(ctx context.Context, wk workerruntime.Runtime, a sdk.Action, _ [ wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Details: %s", res.Details)) wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Status: %s", res.Status)) - if strings.ToUpper(res.Status) == strings.ToUpper(sdk.StatusSuccess) { + if strings.EqualFold(res.Status, sdk.StatusSuccess) { integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) return sdk.Result{ Status: sdk.StatusSuccess, diff --git a/engine/worker/internal/handler_artifacts.go b/engine/worker/internal/handler_artifacts.go index 6cbb395181..3ae5162a3c 100644 --- a/engine/worker/internal/handler_artifacts.go +++ b/engine/worker/internal/handler_artifacts.go @@ -43,7 +43,7 @@ func artifactsHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc { buildNumberString := sdk.ParameterValue(wk.currentJob.params, "cds.run.number") reqArgs.Number, errN = strconv.ParseInt(buildNumberString, 10, 64) if errN != nil { - newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("Cannot parse '%s' as run number: %s", buildNumberString, errN)) + newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("cannot parse '%s' as run number: %s", buildNumberString, errN)) writeError(w, r, newError) return } @@ -52,14 +52,14 @@ func artifactsHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc { projectKey := sdk.ParameterValue(wk.currentJob.params, "cds.project") artifacts, err := wk.client.WorkflowRunArtifacts(projectKey, reqArgs.Workflow, reqArgs.Number) if err != nil { - newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("Cannot list artifacts with worker artifacts: %s", err)) + newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("cannot list artifacts with worker artifacts: %s", err)) writeError(w, r, newError) return } regexp, errp := regexp.Compile(reqArgs.Pattern) if errp != nil { - newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("Invalid pattern %s : %s", reqArgs.Pattern, errp)) + newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("invalid pattern %s : %s", reqArgs.Pattern, errp)) writeError(w, r, newError) return } diff --git a/engine/worker/internal/handler_run_result.go b/engine/worker/internal/handler_run_result.go index bec15eb47d..105ec6c9ad 100644 --- a/engine/worker/internal/handler_run_result.go +++ b/engine/worker/internal/handler_run_result.go @@ -36,58 +36,81 @@ func getRunResultHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFun } } -func addRunResulthandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc { +func addRunResultArtifactManagerHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ctx := workerruntime.SetJobID(ctx, wk.currentJob.wJob.ID) - ctx = workerruntime.SetStepOrder(ctx, wk.currentJob.currentStepIndex) - ctx = workerruntime.SetStepName(ctx, wk.currentJob.currentStepName) + addRunResult(ctx, wk, w, r, sdk.WorkflowRunResultTypeArtifactManager) + } +} + +func addRunResultStaticFileHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + addRunResult(ctx, wk, w, r, sdk.WorkflowRunResultTypeStaticFile) + } +} - data, errRead := ioutil.ReadAll(r.Body) - if errRead != nil { - newError := sdk.NewError(sdk.ErrWrongRequest, errRead) +func addRunResult(ctx context.Context, wk *CurrentWorker, w http.ResponseWriter, r *http.Request, stype sdk.WorkflowRunResultType) { + ctx = workerruntime.SetJobID(ctx, wk.currentJob.wJob.ID) + ctx = workerruntime.SetStepOrder(ctx, wk.currentJob.currentStepIndex) + ctx = workerruntime.SetStepName(ctx, wk.currentJob.currentStepName) + + data, err := ioutil.ReadAll(r.Body) + if err != nil { + newError := sdk.NewError(sdk.ErrWrongRequest, err) + writeError(w, r, newError) + return + } + defer r.Body.Close() //nolint + + var name string + switch stype { + case sdk.WorkflowRunResultTypeStaticFile: + var reqArgs sdk.WorkflowRunResultStaticFile + if err := sdk.JSONUnmarshal(data, &reqArgs); err != nil { + newError := sdk.NewError(sdk.ErrWrongRequest, err) writeError(w, r, newError) return } - defer r.Body.Close() //nolint - + name = reqArgs.Name + case sdk.WorkflowRunResultTypeArtifactManager: var reqArgs sdk.WorkflowRunResultArtifactManager if err := sdk.JSONUnmarshal(data, &reqArgs); err != nil { newError := sdk.NewError(sdk.ErrWrongRequest, err) writeError(w, r, newError) return } - runID, runNodeID, runJobID := wk.GetJobIdentifiers() - runResultCheck := sdk.WorkflowRunResultCheck{ - RunJobID: runJobID, - RunNodeID: runNodeID, - RunID: runID, - Name: reqArgs.Name, - ResultType: sdk.WorkflowRunResultTypeArtifactManager, - } - code, err := wk.Client().QueueWorkflowRunResultCheck(ctx, runJobID, runResultCheck) - if err != nil { - if code == 409 { - writeError(w, r, sdk.NewErrorFrom(sdk.ErrInvalidData, "unable to upload the same file twice")) - return - } - writeError(w, r, sdk.WrapError(err, "unable to check run result %s", reqArgs.Name)) - return - - } + name = reqArgs.Name + } - addRunRequest := sdk.WorkflowRunResult{ - Type: sdk.WorkflowRunResultTypeArtifactManager, - DataRaw: data, - Created: time.Now(), - WorkflowRunJobID: runJobID, - WorkflowRunID: runID, - WorkflowNodeRunID: runNodeID, - } - if err := wk.client.QueueWorkflowRunResultsAdd(ctx, wk.currentJob.wJob.ID, addRunRequest); err != nil { - newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("cannot add run result: %s", err)) - writeError(w, r, newError) + runID, runNodeID, runJobID := wk.GetJobIdentifiers() + runResultCheck := sdk.WorkflowRunResultCheck{ + RunJobID: runJobID, + RunNodeID: runNodeID, + RunID: runID, + Name: name, + ResultType: stype, + } + code, err := wk.Client().QueueWorkflowRunResultCheck(ctx, runJobID, runResultCheck) + if err != nil { + if code == 409 { + writeError(w, r, sdk.NewErrorFrom(sdk.ErrInvalidData, "unable to upload the same file twice")) return } - writeJSON(w, nil, http.StatusOK) + writeError(w, r, sdk.WrapError(err, "unable to check run result %s", name)) + return + } + + addRunRequest := sdk.WorkflowRunResult{ + Type: stype, + DataRaw: data, + Created: time.Now(), + WorkflowRunJobID: runJobID, + WorkflowRunID: runID, + WorkflowNodeRunID: runNodeID, + } + if err := wk.client.QueueWorkflowRunResultsAdd(ctx, wk.currentJob.wJob.ID, addRunRequest); err != nil { + newError := sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("cannot add run result: %s", err)) + writeError(w, r, newError) + return } + writeJSON(w, nil, http.StatusOK) } diff --git a/engine/worker/internal/handler_run_result_test.go b/engine/worker/internal/handler_run_result_test.go new file mode 100644 index 0000000000..397f5e002d --- /dev/null +++ b/engine/worker/internal/handler_run_result_test.go @@ -0,0 +1,126 @@ +package internal + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/ovh/cds/engine/test" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/cdsclient/mock_cdsclient" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_addRunResultStaticFileHandler(t *testing.T) { + // Create test directory for current test + fs := afero.NewOsFs() + basedir := "test-" + test.GetTestName(t) + "-" + sdk.RandomString(10) + "-" + fmt.Sprintf("%d", time.Now().Unix()) + t.Logf("Creating worker basedir at %s", basedir) + require.NoError(t, fs.MkdirAll(basedir, os.FileMode(0755))) + + // Setup test worker + wk := &CurrentWorker{basedir: afero.NewBasePathFs(fs, basedir)} + wk.currentJob.wJob = &sdk.WorkflowNodeJobRun{ID: 1} + + // Prepare mock client for cds workers + ctrl := gomock.NewController(t) + t.Cleanup(func() { ctrl.Finish() }) + m := mock_cdsclient.NewMockWorkerInterface(ctrl) + wk.client = m + + m.EXPECT().QueueWorkflowRunResultCheck(gomock.Any(), int64(1), gomock.Any()).DoAndReturn( + func(ctx context.Context, jobID int64, check sdk.WorkflowRunResultCheck) (int, error) { + assert.Equal(t, sdk.WorkflowRunResultTypeStaticFile, check.ResultType) + return 200, nil + }, + ).Times(1) + m.EXPECT().QueueWorkflowRunResultsAdd(gomock.Any(), int64(1), gomock.Any()).DoAndReturn( + func(ctx context.Context, jobID int64, result sdk.WorkflowRunResult) (int, error) { + assert.Equal(t, sdk.WorkflowRunResultTypeStaticFile, result.Type) + var reqArgs sdk.WorkflowRunResultStaticFile + require.NoError(t, sdk.JSONUnmarshal(result.DataRaw, &reqArgs)) + assert.Equal(t, "http://locat.local/static/foo.html", reqArgs.RemoteURL) + return 200, nil + }, + ).Times(1) + + v := sdk.WorkflowRunResultStaticFile{ + Name: "foo", + RemoteURL: "http://locat.local/static/foo.html", + } + buf, err := json.Marshal(v) + require.NoError(t, err) + + req, err := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(buf)) + require.NoError(t, err) + w := httptest.NewRecorder() + addRunResultStaticFileHandler(context.Background(), wk)(w, req) + assert.Equal(t, http.StatusOK, w.Code) + if w.Code != 200 { + cdsError := sdk.DecodeError(w.Body.Bytes()) + t.Logf("add run result return an error: %v", cdsError.Error()) + t.FailNow() + } +} + +func Test_addRunResultArtifactManagerHandler(t *testing.T) { + // Create test directory for current test + fs := afero.NewOsFs() + basedir := "test-" + test.GetTestName(t) + "-" + sdk.RandomString(10) + "-" + fmt.Sprintf("%d", time.Now().Unix()) + t.Logf("Creating worker basedir at %s", basedir) + require.NoError(t, fs.MkdirAll(basedir, os.FileMode(0755))) + + // Setup test worker + wk := &CurrentWorker{basedir: afero.NewBasePathFs(fs, basedir)} + wk.currentJob.wJob = &sdk.WorkflowNodeJobRun{ID: 1} + + // Prepare mock client for cds workers + ctrl := gomock.NewController(t) + t.Cleanup(func() { ctrl.Finish() }) + m := mock_cdsclient.NewMockWorkerInterface(ctrl) + wk.client = m + + m.EXPECT().QueueWorkflowRunResultCheck(gomock.Any(), int64(1), gomock.Any()).DoAndReturn( + func(ctx context.Context, jobID int64, check sdk.WorkflowRunResultCheck) (int, error) { + assert.Equal(t, sdk.WorkflowRunResultTypeArtifactManager, check.ResultType) + return 200, nil + }, + ).Times(1) + m.EXPECT().QueueWorkflowRunResultsAdd(gomock.Any(), int64(1), gomock.Any()).DoAndReturn( + func(ctx context.Context, jobID int64, result sdk.WorkflowRunResult) (int, error) { + assert.Equal(t, sdk.WorkflowRunResultTypeArtifactManager, result.Type) + var reqArgs sdk.WorkflowRunResultArtifactManager + require.NoError(t, sdk.JSONUnmarshal(result.DataRaw, &reqArgs)) + assert.Equal(t, "foo", reqArgs.Name) + assert.Equal(t, "my-repo", reqArgs.RepoName) + return 200, nil + }, + ).Times(1) + + v := sdk.WorkflowRunResultArtifactManager{ + Name: "foo", + RepoName: "my-repo", + } + buf, err := json.Marshal(v) + require.NoError(t, err) + + req, err := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(buf)) + require.NoError(t, err) + w := httptest.NewRecorder() + addRunResultArtifactManagerHandler(context.Background(), wk)(w, req) + assert.Equal(t, http.StatusOK, w.Code) + if w.Code != 200 { + cdsError := sdk.DecodeError(w.Body.Bytes()) + t.Logf("add run result return an error: %v", cdsError.Error()) + t.FailNow() + } +} diff --git a/engine/worker/internal/http_server.go b/engine/worker/internal/http_server.go index be9b7c8a8a..7f9d6134b4 100644 --- a/engine/worker/internal/http_server.go +++ b/engine/worker/internal/http_server.go @@ -58,7 +58,8 @@ func (w *CurrentWorker) Serve(c context.Context) error { r.HandleFunc("/checksecret", LogMiddleware(checkSecretHandler(c, w))) r.HandleFunc("/var", LogMiddleware(addBuildVarHandler(c, w))) r.HandleFunc("/run-result", LogMiddleware(getRunResultHandler(c, w))) - r.HandleFunc("/run-result/add", LogMiddleware(addRunResulthandler(c, w))) + r.HandleFunc("/run-result/add/artifact-manager", LogMiddleware(addRunResultArtifactManagerHandler(c, w))) + r.HandleFunc("/run-result/add/static-file", LogMiddleware(addRunResultStaticFileHandler(c, w))) r.HandleFunc("/vulnerability", LogMiddleware(vulnerabilityHandler(c, w))) r.HandleFunc("/version", LogMiddleware(setVersionHandler(c, w))) diff --git a/sdk/workflow_run_result.go b/sdk/workflow_run_result.go index f7ef4d1bd9..a085c01263 100644 --- a/sdk/workflow_run_result.go +++ b/sdk/workflow_run_result.go @@ -9,6 +9,7 @@ const ( WorkflowRunResultTypeArtifact WorkflowRunResultType = "artifact" WorkflowRunResultTypeCoverage WorkflowRunResultType = "coverage" WorkflowRunResultTypeArtifactManager WorkflowRunResultType = "artifact-manager" + WorkflowRunResultTypeStaticFile WorkflowRunResultType = "static-file" ) type WorkflowRunResultType string @@ -49,6 +50,14 @@ func (r *WorkflowRunResult) GetArtifactManager() (WorkflowRunResultArtifactManag return data, nil } +func (r *WorkflowRunResult) GetStaticFile() (WorkflowRunResultStaticFile, error) { + var data WorkflowRunResultStaticFile + if err := JSONUnmarshal(r.DataRaw, &data); err != nil { + return data, WithStack(err) + } + return data, nil +} + type WorkflowRunResultCheck struct { Name string `json:"name"` RunID int64 `json:"run_id"` @@ -83,6 +92,21 @@ func (a *WorkflowRunResultArtifactManager) IsValid() error { return nil } +type WorkflowRunResultStaticFile struct { + Name string `json:"name"` + RemoteURL string `json:"remote_url"` +} + +func (a *WorkflowRunResultStaticFile) IsValid() error { + if a.Name == "" { + return WrapError(ErrInvalidData, "missing static-file name") + } + if a.RemoteURL == "" { + return WrapError(ErrInvalidData, "missing remote url") + } + return nil +} + type WorkflowRunResultArtifact struct { Name string `json:"name"` Size int64 `json:"size"` diff --git a/ui/src/app/model/workflow.run.model.ts b/ui/src/app/model/workflow.run.model.ts index 6b5fe8cfc1..d5cd12d744 100644 --- a/ui/src/app/model/workflow.run.model.ts +++ b/ui/src/app/model/workflow.run.model.ts @@ -171,6 +171,11 @@ export class WorkflowRunResultArtifactManager { repository_type: string; } +export class WorkflowRunResultStaticFile { + name: string; + remote_url: string; +} + export class WorkflowNodeOutgoingHookRunCallback { workflow_node_outgoing_hook_id: number; start: Date; diff --git a/ui/src/app/views/workflow/run/node/artifact/artifact.list.component.ts b/ui/src/app/views/workflow/run/node/artifact/artifact.list.component.ts index 04840e50a2..41d9f1e940 100644 --- a/ui/src/app/views/workflow/run/node/artifact/artifact.list.component.ts +++ b/ui/src/app/views/workflow/run/node/artifact/artifact.list.component.ts @@ -5,7 +5,7 @@ import { WorkflowNodeRun, WorkflowNodeRunArtifact, WorkflowNodeRunStaticFiles, WorkflowRunResult, - WorkflowRunResultArtifact, WorkflowRunResultArtifactManager + WorkflowRunResultArtifact, WorkflowRunResultArtifactManager, WorkflowRunResultStaticFile } from 'app/model/workflow.run.model'; import { AutoUnsubscribe } from 'app/shared/decorator/autoUnsubscribe'; @@ -45,11 +45,13 @@ export class WorkflowRunArtifactListComponent implements OnInit, OnDestroy { type: ColumnType.LINK, name: 'artifact_name', selector: (a: UIArtifact) => { - let size = this.getHumainFileSize(a.size); let link = a.link; let value = a.name; - if (size) { - value += ` (${size})`; + if (a.size) { + let size = this.getHumainFileSize(a.size); + if (size) { + value += ` (${size})`; + } } return { link, @@ -161,6 +163,13 @@ export class WorkflowRunArtifactListComponent implements OnInit, OnDestroy { uiArtifactAM.size = dataAM.size; uiArtifactAM.type = dataAM.repository_type; return uiArtifactAM; + case 'static-file': + let dataSF = r.data; + let uiArtifactSF = new UIArtifact(); + uiArtifactSF.link = dataSF.remote_url; + uiArtifactSF.name = dataSF.name; + uiArtifactSF.type = 'static file'; + return uiArtifactSF; } }) }