From b3c57c040c203c4b98fa0f8f0c6a2f621ebda19a Mon Sep 17 00:00:00 2001 From: Guiheux Steven Date: Tue, 5 Sep 2017 19:42:08 +0200 Subject: [PATCH] feat (api): add release builtin action (#1019) --- engine/api/action/builtin.go | 31 +++++ engine/api/main_routes.go | 1 + .../repositoriesmanager/repogithub/client.go | 72 ++++++++++++ .../repogithub/client_event.go | 2 +- .../repositoriesmanager/repogithub/http.go | 5 +- .../repositoriesmanager/repostash/client.go | 11 ++ engine/api/workflow/dao_node.go | 2 +- .../api/workflow/execute_node_job_run_log.go | 4 + engine/api/workflow/process.go | 13 ++- engine/api/workflow/process_parameters.go | 76 ++++++++---- engine/api/workflow_application.go | 108 ++++++++++++++++++ engine/api/workflow_queue.go | 4 +- engine/api/workflow_run.go | 26 +++-- engine/sql/036_workflow_name_constraint.sql | 5 + engine/worker/builtin.go | 2 + engine/worker/builtin_artifact.go | 3 +- engine/worker/builtin_release.go | 106 +++++++++++++++++ engine/worker/cmd_export.go | 18 ++- engine/worker/run.go | 6 +- sdk/action.go | 1 + sdk/action_script.go | 16 +++ sdk/cdsclient/client_queue.go | 3 - sdk/cdsclient/client_workflow.go | 12 ++ sdk/cdsclient/interface.go | 1 + sdk/repositories_manager.go | 11 ++ sdk/workflow_run.go | 8 ++ 26 files changed, 498 insertions(+), 49 deletions(-) create mode 100644 engine/api/workflow_application.go create mode 100644 engine/sql/036_workflow_name_constraint.sql create mode 100644 engine/worker/builtin_release.go diff --git a/engine/api/action/builtin.go b/engine/api/action/builtin.go index ba636c2794..95c6f77de2 100644 --- a/engine/api/action/builtin.go +++ b/engine/api/action/builtin.go @@ -151,6 +151,37 @@ Tag the current branch and push it.` return err } + // ----------------------------------- Git Release ----------------------- + gitrelease := sdk.NewAction(sdk.ReleaseAction) + gitrelease.Type = sdk.BuiltinAction + gitrelease.Description = `CDS Builtin Action. Make a release using repository manager.` + + gitrelease.Parameter(sdk.Parameter{ + Name: "tag", + Description: "Tag name.", + Value: "{{.cds.release.version}}", + Type: sdk.StringParameter, + }) + gitrelease.Parameter(sdk.Parameter{ + Name: "title", + Value: "", + Description: "Set a title for the release", + Type: sdk.StringParameter, + }) + gitrelease.Parameter(sdk.Parameter{ + Name: "releaseNote", + Description: "Set a release note for the release", + Type: sdk.TextParameter, + }) + gitrelease.Parameter(sdk.Parameter{ + Name: "artifacts", + Description: "Set a list of artifacts, separate by , . You can also use regexp.", + Type: sdk.StringParameter, + }) + if err := checkBuiltinAction(db, gitrelease); err != nil { + return err + } + return nil } diff --git a/engine/api/main_routes.go b/engine/api/main_routes.go index 3002633df1..b400bbc66b 100644 --- a/engine/api/main_routes.go +++ b/engine/api/main_routes.go @@ -146,6 +146,7 @@ func (router *Router) init() { router.Handle("/project/{permProjectKey}/workflows/{workflowName}/artifact/{artifactId}", GET(getDownloadArtifactHandler)) router.Handle("/project/{permProjectKey}/workflows/{workflowName}/node/{nodeID}/triggers/condition", GET(getWorkflowTriggerConditionHandler)) router.Handle("/project/{permProjectKey}/workflows/{workflowName}/join/{joinID}/triggers/condition", GET(getWorkflowTriggerJoinConditionHandler)) + router.Handle("/project/{permProjectKey}/workflows/{workflowName}/runs/{number}/nodes/{id}/release", POST(releaseApplicationWorkflowHandler)) // DEPRECATED router.Handle("/project/{key}/pipeline/{permPipelineKey}/action/{jobID}", PUT(updatePipelineActionHandler, DEPRECATED), DELETE(deleteJobHandler)) diff --git a/engine/api/repositoriesmanager/repogithub/client.go b/engine/api/repositoriesmanager/repogithub/client.go index 91ff465ad9..3d53c9aa89 100644 --- a/engine/api/repositoriesmanager/repogithub/client.go +++ b/engine/api/repositoriesmanager/repogithub/client.go @@ -1,8 +1,10 @@ package repogithub import ( + "bytes" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" "strconv" @@ -22,6 +24,76 @@ type GithubClient struct { DisableStatusURL bool } +// ReleaseRequest Request sent to Github to create a release +type ReleaseRequest struct { + TagName string `json:"tag_name"` + Name string `json:"name"` + Body string `json:"body"` +} + +// ReleaseResponse Response return by Github after release creation +type ReleaseResponse struct { + ID int64 `json:"id"` + UploadURL string `json:"upload_url"` +} + +// Release Create a release Github +func (g *GithubClient) Release(fullname string, tagName string, title string, releaseNote string) (*sdk.VCSRelease, error) { + var url = "/repos/" + fullname + "/releases" + + req := ReleaseRequest{ + TagName: tagName, + Name: title, + Body: releaseNote, + } + b, err := json.Marshal(req) + if err != nil { + return nil, sdk.WrapError(err, "github.Release > Cannot marshal body %+v", req) + } + + res, err := g.post(url, "application/json", bytes.NewBuffer(b), false) + if err != nil { + return nil, sdk.WrapError(err, "github.Release > Cannot create release on github") + } + + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, sdk.WrapError(err, "github.Release > Cannot read release response") + } + + if res.StatusCode != 201 { + return nil, sdk.WrapError(fmt.Errorf("github.Release >Unable to create release on github. Url : %s Status code : %d - Body: %s", url, res.StatusCode, body), "") + } + + var response ReleaseResponse + if err := json.Unmarshal(body, &response); err != nil { + return nil, sdk.WrapError(err, "github.Release> Cannot unmarshal response: %s", string(body)) + } + + release := &sdk.VCSRelease{ + ID: response.ID, + UploadURL: response.UploadURL, + } + + return release, nil +} + +// UploadReleaseFile Attach a file into the release +func (g *GithubClient) UploadReleaseFile(repo string, release *sdk.VCSRelease, runArtifact sdk.WorkflowNodeRunArtifact, buf *bytes.Buffer) error { + var url = strings.Split(release.UploadURL, "{")[0] + "?name=" + runArtifact.Name + res, err := g.post(url, "application/octet-stream", buf, true) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 201 { + return sdk.WrapError(fmt.Errorf("github.Release >Unable to upload file on release. Url : %s - Status code : %d", url, res.StatusCode), "") + } + return nil +} + // Repos list repositories that are accessible to the authenticated user // https://developer.github.com/v3/repos/#list-your-repositories func (g *GithubClient) Repos() ([]sdk.VCSRepo, error) { diff --git a/engine/api/repositoriesmanager/repogithub/client_event.go b/engine/api/repositoriesmanager/repogithub/client_event.go index 6059f821ac..9b13b80030 100644 --- a/engine/api/repositoriesmanager/repogithub/client_event.go +++ b/engine/api/repositoriesmanager/repogithub/client_event.go @@ -102,7 +102,7 @@ func (g *GithubClient) SetStatus(event sdk.Event) error { } buf := bytes.NewBuffer(b) - res, err := g.post(path, "application/json", buf) + res, err := g.post(path, "application/json", buf, false) if err != nil { return err } diff --git a/engine/api/repositoriesmanager/repogithub/http.go b/engine/api/repositoriesmanager/repogithub/http.go index 0520dcd786..dc929c20e5 100644 --- a/engine/api/repositoriesmanager/repogithub/http.go +++ b/engine/api/repositoriesmanager/repogithub/http.go @@ -117,8 +117,8 @@ func withETag(c *GithubClient, req *http.Request, path string) { } func withoutETag(c *GithubClient, req *http.Request, path string) {} -func (c *GithubClient) post(path string, bodyType string, body io.Reader) (*http.Response, error) { - if !strings.HasPrefix(path, APIURL) { +func (c *GithubClient) post(path string, bodyType string, body io.Reader, skipDefaultBaseURL bool) (*http.Response, error) { + if !skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) { path = APIURL + path } @@ -127,6 +127,7 @@ func (c *GithubClient) post(path string, bodyType string, body io.Reader) (*http return nil, err } + req.Header.Set("Content-Type", bodyType) req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID) req.Header.Add("Accept", "application/json") req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken)) diff --git a/engine/api/repositoriesmanager/repostash/client.go b/engine/api/repositoriesmanager/repostash/client.go index 2f8277a25b..541fba41b8 100644 --- a/engine/api/repositoriesmanager/repostash/client.go +++ b/engine/api/repositoriesmanager/repostash/client.go @@ -1,6 +1,7 @@ package repostash import ( + "bytes" "fmt" "net/http" "net/url" @@ -34,6 +35,16 @@ type StashClient struct { disableSetStatus bool } +// Release not implemented on bitbucket +func (s *StashClient) Release(repo string, tagName string, title string, releaseNote string) (*sdk.VCSRelease, error) { + return nil, fmt.Errorf("Stash do not provide release system") +} + +// UploadReleaseFile not implemented on bitbucket +func (s *StashClient) UploadReleaseFile(repo string, release *sdk.VCSRelease, runArtifact sdk.WorkflowNodeRunArtifact, buf *bytes.Buffer) error { + return fmt.Errorf("Stash do not provide an upload file system") +} + //Repos returns the list of accessible repositories func (s *StashClient) Repos() ([]sdk.VCSRepo, error) { repos := []sdk.VCSRepo{} diff --git a/engine/api/workflow/dao_node.go b/engine/api/workflow/dao_node.go index 7e18a7b750..fcfbea19d4 100644 --- a/engine/api/workflow/dao_node.go +++ b/engine/api/workflow/dao_node.go @@ -233,7 +233,7 @@ func loadNodeContext(db gorp.SqlExecutor, wn *sdk.WorkflowNode, u *sdk.User) (*s //Load the application in the context if ctx.ApplicationID != 0 { - app, err := application.LoadByID(db, ctx.ApplicationID, u) + app, err := application.LoadByID(db, ctx.ApplicationID, u, application.LoadOptions.WithRepositoryManager) if err != nil { return nil, sdk.WrapError(err, "loadNodeContext> Unable to load application %d", ctx.ApplicationID) } diff --git a/engine/api/workflow/execute_node_job_run_log.go b/engine/api/workflow/execute_node_job_run_log.go index d4a04eec30..8cfed4df38 100644 --- a/engine/api/workflow/execute_node_job_run_log.go +++ b/engine/api/workflow/execute_node_job_run_log.go @@ -1,6 +1,7 @@ package workflow import ( + "database/sql" "time" "github.com/go-gorp/gorp" @@ -17,6 +18,9 @@ func LoadStepLogs(db gorp.SqlExecutor, id int64, order int64) (*sdk.Log, error) logs := &sdk.Log{} var s, m, d time.Time if err := db.QueryRow(query, id, order).Scan(&logs.Id, &logs.PipelineBuildJobID, &logs.PipelineBuildID, &s, &m, &d, &logs.StepOrder, &logs.Val); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } return nil, err } var err error diff --git a/engine/api/workflow/process.go b/engine/api/workflow/process.go index 1523c35469..b3cc4d821b 100644 --- a/engine/api/workflow/process.go +++ b/engine/api/workflow/process.go @@ -69,7 +69,7 @@ func processWorkflowRun(db gorp.SqlExecutor, w *sdk.WorkflowRun, hookEvent *sdk. //Check conditions var params = nodeRun.BuildParameters - //Define specific desitination parameters + //Define specific destination parameters sdk.AddParameter(¶ms, "cds.dest.pipeline", sdk.StringParameter, t.WorkflowDestNode.Pipeline.Name) if t.WorkflowDestNode.Context.Application != nil { sdk.AddParameter(¶ms, "cds.dest.application", sdk.StringParameter, t.WorkflowDestNode.Context.Application.Name) @@ -298,8 +298,7 @@ func processWorkflowNodeRun(db gorp.SqlExecutor, w *sdk.WorkflowRun, n *sdk.Work run.PipelineParameters = m.PipelineParameters } - //Process parameters for the jobs - //TODO inherit parameter from parent job + // Process parameters for the jobs jobParams, errParam := getNodeRunBuildParameters(db, run) if errParam != nil { AddWorkflowRunInfo(w, sdk.SpawnMsg{ @@ -310,6 +309,14 @@ func processWorkflowNodeRun(db gorp.SqlExecutor, w *sdk.WorkflowRun, n *sdk.Work } run.BuildParameters = jobParams + // Inherit parameter from parent job + if len(sourceNodeRuns) > 0 { + parentsParams, errPP := getParentParameters(db, run, sourceNodeRuns) + if errPP != nil { + return sdk.WrapError(errPP, "processWorkflowNodeRun> getParentParameters failed") + } + run.BuildParameters = append(run.BuildParameters, parentsParams...) + } for _, p := range jobParams { switch p.Name { case "git.hash", "git.branch", "git.tag", "git.author": diff --git a/engine/api/workflow/process_parameters.go b/engine/api/workflow/process_parameters.go index 1ff008b2f8..c431055aa4 100644 --- a/engine/api/workflow/process_parameters.go +++ b/engine/api/workflow/process_parameters.go @@ -2,6 +2,7 @@ package workflow import ( "fmt" + "strings" "github.com/fsamin/go-dump" "github.com/go-gorp/gorp" @@ -12,11 +13,7 @@ import ( ) func getNodeJobRunParameters(db gorp.SqlExecutor, j sdk.Job, run *sdk.WorkflowNodeRun, stage *sdk.Stage) ([]sdk.Parameter, error) { - params, err := getNodeRunBuildParameters(db, run) - if err != nil { - return nil, err - } - + params := run.BuildParameters tmp := map[string]string{} tmp["cds.stage"] = stage.Name @@ -35,15 +32,14 @@ func getNodeJobRunParameters(db gorp.SqlExecutor, j sdk.Job, run *sdk.WorkflowNo if errm.IsEmpty() { return params, nil } - return params, errm } // GetNodeBuildParameters returns build parameters with default values for cds.version, cds.run, cds.run.number, cds.run.subnumber func GetNodeBuildParameters(proj *sdk.Project, w *sdk.Workflow, n *sdk.WorkflowNode, pipelineParameters []sdk.Parameter, payload interface{}) ([]sdk.Parameter, error) { vars := map[string]string{} - tmp := sdk.ParametersFromProjectVariables(proj) - for k, v := range tmp { + tmpProj := sdk.ParametersFromProjectVariables(proj) + for k, v := range tmpProj { vars[k] = v } @@ -66,8 +62,8 @@ func GetNodeBuildParameters(proj *sdk.Project, w *sdk.Workflow, n *sdk.WorkflowN } // compute pipeline parameters - tmp = sdk.ParametersFromPipelineParameters(pipelineParameters) - for k, v := range tmp { + tmpPip := sdk.ParametersFromPipelineParameters(pipelineParameters) + for k, v := range tmpPip { vars[k] = v } @@ -80,20 +76,22 @@ func GetNodeBuildParameters(proj *sdk.Project, w *sdk.Workflow, n *sdk.WorkflowN errm.Append(errdump) } for k, v := range payloadMap { - tmp[k] = v + vars[k] = v } - tmp["cds.project"] = w.ProjectKey - tmp["cds.workflow"] = w.Name - tmp["cds.pipeline"] = n.Pipeline.Name - tmp["cds.version"] = fmt.Sprintf("%d.%d", 1, 0) - tmp["cds.run"] = fmt.Sprintf("%d.%d", 1, 0) - tmp["cds.run.number"] = fmt.Sprintf("%d", 1) - tmp["cds.run.subnumber"] = fmt.Sprintf("%d", 0) + // TODO Update suggest.go with new variable + + vars["cds.project"] = w.ProjectKey + vars["cds.workflow"] = w.Name + vars["cds.pipeline"] = n.Pipeline.Name + vars["cds.version"] = fmt.Sprintf("%d.%d", 1, 0) + vars["cds.run"] = fmt.Sprintf("%d.%d", 1, 0) + vars["cds.run.number"] = fmt.Sprintf("%d", 1) + vars["cds.run.subnumber"] = fmt.Sprintf("%d", 0) params := []sdk.Parameter{} - for k, v := range tmp { - s, err := sdk.Interpolate(v, tmp) + for k, v := range vars { + s, err := sdk.Interpolate(v, vars) if err != nil { errm.Append(err) continue @@ -108,6 +106,44 @@ func GetNodeBuildParameters(proj *sdk.Project, w *sdk.Workflow, n *sdk.WorkflowN return params, errm } +func getParentParameters(db gorp.SqlExecutor, run *sdk.WorkflowNodeRun, nodeRunIds []int64) ([]sdk.Parameter, error) { + //Load workflow run + w, err := LoadRunByID(db, run.WorkflowRunID) + if err != nil { + return nil, sdk.WrapError(err, "getParentParameters> Unable to load workflow run") + } + + params := []sdk.Parameter{} + for _, nodeRunID := range nodeRunIds { + parentNodeRun, errNR := LoadNodeRunByID(db, nodeRunID) + if errNR != nil { + return nil, sdk.WrapError(errNR, "getParentParameters> Cannot get parent node run") + } + + node := w.Workflow.GetNode(parentNodeRun.WorkflowNodeID) + if node == nil { + return nil, sdk.WrapError(fmt.Errorf("Unable to find node %d in workflow", parentNodeRun.WorkflowNodeID), "getParentParameters>") + } + + for i := range parentNodeRun.BuildParameters { + p := &parentNodeRun.BuildParameters[i] + + if p.Name == "cds.semver" || p.Name == "cds.release.version" || strings.HasPrefix(p.Name, "cds.proj") || strings.HasPrefix(p.Name, "workflow.") { + continue + } + + prefix := "workflow." + node.Name + "." + if strings.HasPrefix(p.Name, "cds.") { + p.Name = strings.Replace(p.Name, "cds.", prefix, 1) + } else { + p.Name = prefix + p.Name + } + } + params = append(params, parentNodeRun.BuildParameters...) + } + return params, nil +} + func getNodeRunBuildParameters(db gorp.SqlExecutor, run *sdk.WorkflowNodeRun) ([]sdk.Parameter, error) { //Load workflow run w, err := LoadRunByID(db, run.WorkflowRunID) diff --git a/engine/api/workflow_application.go b/engine/api/workflow_application.go new file mode 100644 index 0000000000..c80bf6172b --- /dev/null +++ b/engine/api/workflow_application.go @@ -0,0 +1,108 @@ +package main + +import ( + "bytes" + "net/http" + "regexp" + "sort" + + "github.com/go-gorp/gorp" + "github.com/gorilla/mux" + + "github.com/ovh/cds/engine/api/artifact" + "github.com/ovh/cds/engine/api/businesscontext" + "github.com/ovh/cds/engine/api/repositoriesmanager" + "github.com/ovh/cds/engine/api/workflow" + "github.com/ovh/cds/sdk" +) + +func releaseApplicationWorkflowHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbMap, c *businesscontext.Ctx) error { + vars := mux.Vars(r) + key := vars["permProjectKey"] + name := vars["workflowName"] + nodeRunID, errN := requestVarInt(r, "id") + if errN != nil { + return errN + } + + number, errNRI := requestVarInt(r, "number") + if errNRI != nil { + return errNRI + } + + var req sdk.WorkflowNodeRunRelease + if errU := UnmarshalBody(r, &req); errU != nil { + return errU + } + + wNodeRun, errWNR := workflow.LoadNodeRun(db, key, name, number, nodeRunID) + if errWNR != nil { + return sdk.WrapError(errWNR, "releaseApplicationWorkflowHandler") + } + + workflowRun, errWR := workflow.LoadRunByIDAndProjectKey(db, key, wNodeRun.WorkflowRunID) + if errWR != nil { + return sdk.WrapError(errWR, "releaseApplicationWorkflowHandler") + } + + workflowArtifacts := []sdk.WorkflowNodeRunArtifact{} + for _, runs := range workflowRun.WorkflowNodeRuns { + if len(runs) == 0 { + continue + } + sort.Slice(runs, func(i, j int) bool { + return runs[i].SubNumber > runs[j].SubNumber + }) + workflowArtifacts = append(workflowArtifacts, runs[0].Artifacts...) + } + + workflowNode := workflowRun.Workflow.GetNode(wNodeRun.WorkflowNodeID) + if workflowNode == nil { + return sdk.WrapError(sdk.ErrWorkflowNodeNotFound, "releaseApplicationWorkflowHandler") + } + + if workflowNode.Context == nil || workflowNode.Context.Application == nil { + return sdk.WrapError(sdk.ErrApplicationNotFound, "releaseApplicationWorkflowHandler") + } + + if workflowNode.Context.Application.RepositoriesManager == nil { + return sdk.WrapError(sdk.ErrNoReposManager, "releaseApplicationWorkflowHandler") + } + + client, err := repositoriesmanager.AuthorizedClient(db, key, workflowNode.Context.Application.RepositoriesManager.Name) + if err != nil { + return sdk.WrapError(err, "releaseApplicationWorkflowHandler> Cannot get client got %s %s", key, workflowNode.Context.Application.RepositoriesManager.Name) + } + + release, errRelease := client.Release(workflowNode.Context.Application.RepositoryFullname, req.TagName, req.ReleaseTitle, req.ReleaseContent) + if errRelease != nil { + return sdk.WrapError(errRelease, "releaseApplicationWorkflowHandler") + } + + // Get artifacts to upload + var artifactToUpload []sdk.WorkflowNodeRunArtifact + for _, a := range workflowArtifacts { + for _, aToUp := range req.Artifacts { + ok, errRX := regexp.Match(aToUp, []byte(a.Name)) + if errRX != nil { + return sdk.WrapError(errRX, "releaseApplicationWorkflowHandler> %s is not a valid regular expression", aToUp) + } + if ok { + artifactToUpload = append(artifactToUpload, a) + break + } + } + } + + for _, a := range artifactToUpload { + b := &bytes.Buffer{} + if err := artifact.StreamFile(b, &a); err != nil { + return sdk.WrapError(err, "Cannot get artifact") + } + if err := client.UploadReleaseFile(workflowNode.Context.Application.RepositoryFullname, release, a, b); err != nil { + return sdk.WrapError(err, "releaseApplicationWorkflowHandler") + } + } + + return nil +} diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index b57fb8929f..20057ba888 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -425,7 +425,7 @@ func postWorkflowJobVariableHandler(w http.ResponseWriter, r *http.Request, db * return sdk.WrapError(errj, "postWorkflowJobVariableHandler> Unable to load job") } - sdk.AddParameter(&job.Parameters, "cds.build."+v.Name, sdk.StringParameter, v.Value) + sdk.AddParameter(&job.Parameters, v.Name, sdk.StringParameter, v.Value) if err := workflow.UpdateNodeJobRun(tx, job); err != nil { return sdk.WrapError(err, "postWorkflowJobVariableHandler> Unable to update node job run") @@ -436,7 +436,7 @@ func postWorkflowJobVariableHandler(w http.ResponseWriter, r *http.Request, db * return sdk.WrapError(errn, "postWorkflowJobVariableHandler> Unable to load node") } - sdk.AddParameter(&node.BuildParameters, "cds.build."+v.Name, sdk.StringParameter, v.Value) + sdk.AddParameter(&node.BuildParameters, v.Name, sdk.StringParameter, v.Value) if err := workflow.UpdateNodeRun(tx, node); err != nil { return sdk.WrapError(err, "postWorkflowJobVariableHandler> Unable to update node run") diff --git a/engine/api/workflow_run.go b/engine/api/workflow_run.go index e2a0821713..4a3044a9d2 100644 --- a/engine/api/workflow_run.go +++ b/engine/api/workflow_run.go @@ -62,8 +62,8 @@ func getWorkflowRunsHandler(w http.ResponseWriter, r *http.Request, db *gorp.DbM return sdk.WrapError(err, "getWorkflowRunsHandler> Unable to load workflow runs") } - if limit-offset > count { - return sdk.WrapError(sdk.ErrWrongRequest, "getWorkflowRunsHandler> Requested range %d not allowed", (limit - offset)) + if offset > count { + return sdk.WrapError(sdk.ErrWrongRequest, "getWorkflowRunsHandler> Requested range %d not allowed", offset) } code := http.StatusOK @@ -374,30 +374,30 @@ func getWorkflowNodeRunJobStepHandler(w http.ResponseWriter, r *http.Request, db workflowName := vars["workflowName"] number, errN := requestVarInt(r, "number") if errN != nil { - return sdk.WrapError(errN, "getWorkflowNodeRunJobBuildLogsHandler> Number: invalid number") + return sdk.WrapError(errN, "getWorkflowNodeRunJobStepHandler> Number: invalid number") } nodeRunID, errNI := requestVarInt(r, "id") if errNI != nil { - return sdk.WrapError(errNI, "getWorkflowNodeRunJobBuildLogsHandler> id: invalid number") + return sdk.WrapError(errNI, "getWorkflowNodeRunJobStepHandler> id: invalid number") } runJobID, errJ := requestVarInt(r, "runJobId") if errJ != nil { - return sdk.WrapError(errJ, "getWorkflowNodeRunJobBuildLogsHandler> runJobId: invalid number") + return sdk.WrapError(errJ, "getWorkflowNodeRunJobStepHandler> runJobId: invalid number") } stepOrder, errS := requestVarInt(r, "stepOrder") if errS != nil { - return sdk.WrapError(errS, "getWorkflowNodeRunJobBuildLogsHandler> stepOrder: invalid number") + return sdk.WrapError(errS, "getWorkflowNodeRunJobStepHandler> stepOrder: invalid number") } // Check workflow is in project if _, errW := workflow.Load(db, projectKey, workflowName, c.User); errW != nil { - return sdk.WrapError(errW, "getWorkflowNodeRunJobBuildLogsHandler> Cannot find workflow %s in project %s", workflowName, projectKey) + return sdk.WrapError(errW, "getWorkflowNodeRunJobStepHandler> Cannot find workflow %s in project %s", workflowName, projectKey) } // Check nodeRunID is link to workflow nodeRun, errNR := workflow.LoadNodeRun(db, projectKey, workflowName, number, nodeRunID) if errNR != nil { - return sdk.WrapError(errNR, "getWorkflowNodeRunJobBuildLogsHandler> Cannot find nodeRun %d/%d for workflow %s in project %s", nodeRunID, number, workflowName, projectKey) + return sdk.WrapError(errNR, "getWorkflowNodeRunJobStepHandler> Cannot find nodeRun %d/%d for workflow %s in project %s", nodeRunID, number, workflowName, projectKey) } var stepStatus string @@ -420,18 +420,22 @@ stageLoop: } if stepStatus == "" { - return sdk.WrapError(fmt.Errorf("getWorkflowNodeRunJobBuildLogsHandler> Cannot find step %d on job %d in nodeRun %d/%d for workflow %s in project %s", + return sdk.WrapError(fmt.Errorf("getWorkflowNodeRunJobStepHandler> Cannot find step %d on job %d in nodeRun %d/%d for workflow %s in project %s", stepOrder, runJobID, nodeRunID, number, workflowName, projectKey), "") } logs, errL := workflow.LoadStepLogs(db, runJobID, stepOrder) if errL != nil { - return sdk.WrapError(errL, "getWorkflowNodeRunJobBuildLogsHandler> Cannot load log for runJob %d on step %d", runJobID, stepOrder) + return sdk.WrapError(errL, "getWorkflowNodeRunJobStepHandler> Cannot load log for runJob %d on step %d", runJobID, stepOrder) } + ls := &sdk.Log{} + if logs != nil { + ls = logs + } result := &sdk.BuildState{ Status: sdk.StatusFromString(stepStatus), - StepLogs: *logs, + StepLogs: *ls, } return WriteJSON(w, r, result, http.StatusOK) diff --git a/engine/sql/036_workflow_name_constraint.sql b/engine/sql/036_workflow_name_constraint.sql new file mode 100644 index 0000000000..41ba310788 --- /dev/null +++ b/engine/sql/036_workflow_name_constraint.sql @@ -0,0 +1,5 @@ +-- +migrate Up +ALTER TABLE workflow_node ADD CONSTRAINT "UNIQ_WORKFLOW_NODE_NAME" UNIQUE (id, name); + +-- +migrate Down +ALTER TABLE workflow_node DROP CONSTRAINT "UNIQ_WORKFLOW_NODE_NAME"; diff --git a/engine/worker/builtin.go b/engine/worker/builtin.go index b8f98dc818..1df631c6eb 100644 --- a/engine/worker/builtin.go +++ b/engine/worker/builtin.go @@ -21,6 +21,8 @@ func init() { mapBuiltinActions[sdk.JUnitAction] = runParseJunitTestResultAction mapBuiltinActions[sdk.GitCloneAction] = runGitClone mapBuiltinActions[sdk.GitTagAction] = runGitTag + mapBuiltinActions[sdk.ReleaseAction] = runRelease + } // BuiltInAction defines builtin action signature diff --git a/engine/worker/builtin_artifact.go b/engine/worker/builtin_artifact.go index 7ae499f30c..f451024ec8 100644 --- a/engine/worker/builtin_artifact.go +++ b/engine/worker/builtin_artifact.go @@ -115,7 +115,8 @@ func runArtifactUpload(w *currentWorker) BuiltInAction { for _, filePath := range filesPath { filename := filepath.Base(filePath) sendLog(fmt.Sprintf("Uploading '%s'\n", filename)) - if err := w.client.QueueArtifactUpload(buildID, tag.Value, filename); err != nil { + if err := w.client.QueueArtifactUpload(buildID, tag.Value, filePath); err != nil { + res.Status = sdk.StatusFail.String() res.Reason = fmt.Sprintf("Error while uploading artefact: %s\n", err) sendLog(res.Reason) diff --git a/engine/worker/builtin_release.go b/engine/worker/builtin_release.go new file mode 100644 index 0000000000..92705665fe --- /dev/null +++ b/engine/worker/builtin_release.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/ovh/cds/sdk" +) + +func runRelease(w *currentWorker) BuiltInAction { + return func(ctx context.Context, a *sdk.Action, buildID int64, params *[]sdk.Parameter, sendLog LoggerFunc) sdk.Result { + artifactList := sdk.ParameterFind(a.Parameters, "artifacts") + tag := sdk.ParameterFind(a.Parameters, "tag") + title := sdk.ParameterFind(a.Parameters, "title") + releaseNote := sdk.ParameterFind(a.Parameters, "releaseNote") + + pkey := sdk.ParameterFind(*params, "cds.project") + wName := sdk.ParameterFind(*params, "cds.workflow") + workflowNum := sdk.ParameterFind(*params, "cds.run.number") + + if pkey == nil { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: "cds.project variable not found.", + } + sendLog(res.Reason) + return res + } + + if wName == nil { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: "cds.workflow variable not found.", + } + sendLog(res.Reason) + return res + } + + if workflowNum == nil { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: "cds.run.number variable not found.", + } + sendLog(res.Reason) + return res + } + + if tag == nil || tag.Value == "" { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: "Tag name is not set. Nothing to perform.", + } + sendLog(res.Reason) + return res + } + + if title == nil || title.Value == "" { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: "Release title is not set.", + } + sendLog(res.Reason) + return res + } + + if releaseNote == nil || releaseNote.Value == "" { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: "Release not is not set.", + } + sendLog(res.Reason) + return res + } + + wRunNumber, errI := strconv.ParseInt(workflowNum.Value, 10, 64) + if errI != nil { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: fmt.Sprintf("Workflow number is not a number. Got %s: %s", workflowNum.Value, errI), + } + sendLog(res.Reason) + return res + } + + artSplitted := strings.Split(artifactList.Value, ",") + req := sdk.WorkflowNodeRunRelease{ + ReleaseContent: releaseNote.Value, + ReleaseTitle: title.Value, + TagName: tag.Value, + Artifacts: artSplitted, + } + + if err := w.client.WorkflowNodeRunRelease(pkey.Value, wName.Value, wRunNumber, w.currentJob.wJob.WorkflowNodeRunID, req); err != nil { + res := sdk.Result{ + Status: sdk.StatusFail.String(), + Reason: fmt.Sprintf("Cannot make workflow node run release: %s", err), + } + sendLog(res.Reason) + return res + } + + return sdk.Result{Status: sdk.StatusSuccess.String()} + } +} diff --git a/engine/worker/cmd_export.go b/engine/worker/cmd_export.go index 9815c87768..b384834370 100644 --- a/engine/worker/cmd_export.go +++ b/engine/worker/cmd_export.go @@ -107,8 +107,14 @@ func (wk *currentWorker) addVariableInPipelineBuild(v sdk.Variable, params *[]sd return http.StatusBadRequest, fmt.Errorf("addBuildVarHandler> Cannot Marshal err: %s", errm) } // Retrieve build info + var currentParam []sdk.Parameter + if wk.currentJob.wJob != nil { + currentParam = wk.currentJob.wJob.Parameters + } else { + currentParam = wk.currentJob.pbJob.Parameters + } var proj, app, pip, bnS, env string - for _, p := range wk.currentJob.pbJob.Parameters { + for _, p := range currentParam { switch p.Name { case "cds.pipeline": pip = p.Value @@ -123,13 +129,19 @@ func (wk *currentWorker) addVariableInPipelineBuild(v sdk.Variable, params *[]sd } } - uri := fmt.Sprintf("/project/%s/application/%s/pipeline/%s/build/%s/variable?envName=%s", proj, app, pip, bnS, url.QueryEscape(env)) + var uri string + if wk.currentJob.wJob != nil { + uri = fmt.Sprintf("/queue/workflows/%d/variable", wk.currentJob.wJob.ID) + } else { + uri = fmt.Sprintf("/project/%s/application/%s/pipeline/%s/build/%s/variable?envName=%s", proj, app, pip, bnS, url.QueryEscape(env)) + } + _, code, err := sdk.Request("POST", uri, data) if err == nil && code > 300 { err = fmt.Errorf("HTTP %d", code) } if err != nil { - log.Error("addBuildVarHandler> Cannot export variable: %s", err) + log.Error("addBuildVarHandler> Cannot export variable. %s: %s", uri, err) return http.StatusServiceUnavailable, fmt.Errorf("addBuildVarHandler> Cannot export variable: %s", err) } return http.StatusOK, nil diff --git a/engine/worker/run.go b/engine/worker/run.go index 095fdeb9e8..3d3e93a584 100644 --- a/engine/worker/run.go +++ b/engine/worker/run.go @@ -316,7 +316,8 @@ func (w *currentWorker) processJob(ctx context.Context, jobInfo *worker.Workflow // Setup working directory pbJobPath := path.Join(fmt.Sprintf("%d", jobInfo.Number), fmt.Sprintf("%d", jobInfo.SubNumber), - fmt.Sprintf("%s", jobInfo.NodeJobRun.Job.Action.Name)) + fmt.Sprintf("%d", jobInfo.NodeJobRun.ID), + fmt.Sprintf("%d", jobInfo.NodeJobRun.Job.PipelineActionID)) log.Debug("processJob> init workingDirectory basedir:%s pbJobPath:%s", w.basedir, pbJobPath) wd := workingDirectory(w.basedir, pbJobPath) @@ -432,7 +433,7 @@ func (w *currentWorker) run(ctx context.Context, pbji *worker.PipelineBuildJobIn fmt.Sprintf("%d", pbji.BuildNumber)) wd := workingDirectory(w.basedir, pbJobPath) - log.Debug("run> setupBuildDirectory %s", setupBuildDirectory) + log.Debug("run> init workingDirectory basedir:%s pbJobPath:%s", w.basedir, pbJobPath) if err := setupBuildDirectory(wd); err != nil { log.Debug("run> setupBuildDirectory error %s", err) return sdk.Result{ @@ -517,6 +518,7 @@ func (w *currentWorker) run(ctx context.Context, pbji *worker.PipelineBuildJobIn res := w.startAction(ctx, &pbji.PipelineBuildJob.Job.Action, pbji.PipelineBuildJob.ID, &pbji.PipelineBuildJob.Parameters, -1, "") logsecrets = nil + log.Debug("processJob> call teardownBuildDirectory wd:%s", wd) if err := teardownBuildDirectory(wd); err != nil { log.Error("Cannot remove build directory: %s", err) } diff --git a/sdk/action.go b/sdk/action.go index 6369cbe9b9..4b4479fd09 100644 --- a/sdk/action.go +++ b/sdk/action.go @@ -70,6 +70,7 @@ const ( JUnitAction = "JUnit" GitCloneAction = "GitClone" GitTagAction = "GitTag" + ReleaseAction = "Release" ) const ( diff --git a/sdk/action_script.go b/sdk/action_script.go index fc0bc9aeb9..7456248b64 100644 --- a/sdk/action_script.go +++ b/sdk/action_script.go @@ -27,6 +27,7 @@ type ActionScript struct { Script string `json:"script,omitempty"` JUnitReport string `json:"jUnitReport,omitempty"` Plugin map[string]map[string]string `json:"plugin,omitempty"` + Release map[string]string `json:"release,omitempty"` } `json:"steps"` } @@ -82,6 +83,16 @@ func NewStepGitTag(v map[string]string) Action { return newAction } +// NewStepRelease returns an action (basically used as a step of a job) of Release type +func NewStepRelease(v map[string]string) Action { + newAction := Action{ + Name: ReleaseAction, + Type: BuiltinAction, + Parameters: ParametersFromMap(v), + } + return newAction +} + // NewStepArtifactUpload returns an action (basically used as a step of a job) of artifact upload type func NewStepArtifactUpload(v map[string]string) Action { newAction := Action{ @@ -204,10 +215,15 @@ func NewActionFromScript(btes []byte) (*Action, error) { goto next } + // Action builtin = GitTag if v.GitTag != nil { newAction = NewStepGitTag(v.GitTag) } + if v.Release != nil { + newAction = NewStepRelease(v.Release) + } + //Action builtin = Plugin if v.Plugin != nil { a, err := NewStepPlugin(v.Plugin) diff --git a/sdk/cdsclient/client_queue.go b/sdk/cdsclient/client_queue.go index e76371f563..f860e8692d 100644 --- a/sdk/cdsclient/client_queue.go +++ b/sdk/cdsclient/client_queue.go @@ -179,13 +179,11 @@ func (c *client) QueueArtifactUpload(id int64, tag, filePath string) error { if errop != nil { return errop } - //File stat stat, errst := fileForMD5.Stat() if errst != nil { return errst } - //Compute md5sum hash := md5.New() if _, errcopy := io.Copy(hash, fileForMD5); errcopy != nil { @@ -194,7 +192,6 @@ func (c *client) QueueArtifactUpload(id int64, tag, filePath string) error { hashInBytes := hash.Sum(nil)[:16] md5sumStr := hex.EncodeToString(hashInBytes) fileForMD5.Close() - //Reopen the file because we already read it for md5 fileReopen, erro := os.Open(filePath) if erro != nil { diff --git a/sdk/cdsclient/client_workflow.go b/sdk/cdsclient/client_workflow.go index 56f2015142..f3d2b3d7b0 100644 --- a/sdk/cdsclient/client_workflow.go +++ b/sdk/cdsclient/client_workflow.go @@ -74,3 +74,15 @@ func (c *client) WorkflowNodeRunArtifactDownload(projectKey string, name string, } return nil } + +func (c *client) WorkflowNodeRunRelease(projectKey string, workflowName string, runNumber int64, nodeRunID int64, release sdk.WorkflowNodeRunRelease) error { + url := fmt.Sprintf("/project/%s/workflows/%s/runs/%d/nodes/%d/release", projectKey, workflowName, runNumber, nodeRunID) + code, err := c.PostJSON(url, release, nil) + if err != nil { + return err + } + if code >= 300 { + return fmt.Errorf("Cannot create workflow node run release. Http code error : %d", code) + } + return nil +} diff --git a/sdk/cdsclient/interface.go b/sdk/cdsclient/interface.go index 120086e1d1..d351264d3e 100644 --- a/sdk/cdsclient/interface.go +++ b/sdk/cdsclient/interface.go @@ -66,4 +66,5 @@ type Interface interface { WorkflowNodeRun(projectKey string, name string, number int64, nodeRunID int64) (*sdk.WorkflowNodeRun, error) WorkflowNodeRunArtifacts(projectKey string, name string, number int64, nodeRunID int64) ([]sdk.Artifact, error) WorkflowNodeRunArtifactDownload(projectKey string, name string, artifactID int64, w io.Writer) error + WorkflowNodeRunRelease(projectKey string, workflowName string, runNumber int64, nodeRunID int64, release sdk.WorkflowNodeRunRelease) error } diff --git a/sdk/repositories_manager.go b/sdk/repositories_manager.go index 49fb9dbd4c..e43bd75ba9 100644 --- a/sdk/repositories_manager.go +++ b/sdk/repositories_manager.go @@ -1,6 +1,7 @@ package sdk import ( + "bytes" "encoding/json" "fmt" "net/url" @@ -308,6 +309,16 @@ type RepositoriesManagerClient interface { // Set build status on repository SetStatus(event Event) error + + // Release + Release(repo, tagName, releaseTitle, releaseDescription string) (*VCSRelease, error) + UploadReleaseFile(repo string, release *VCSRelease, runArtifact WorkflowNodeRunArtifact, file *bytes.Buffer) error +} + +// Release represents data about release on github, etc.. +type VCSRelease struct { + ID int64 `json:"id"` + UploadURL string `json:"upload_url"` } //VCSRepo represents data about repository even on stash, or github, etc... diff --git a/sdk/workflow_run.go b/sdk/workflow_run.go index 3e04bddf14..cadc116720 100644 --- a/sdk/workflow_run.go +++ b/sdk/workflow_run.go @@ -23,6 +23,14 @@ type WorkflowRun struct { Tags []WorkflowRunTag `json:"tags" db:"-"` } +// WorkflowNodeRunRelease represents the request struct use by release builtin action for workflow +type WorkflowNodeRunRelease struct { + TagName string `json:"tag_name"` + ReleaseTitle string `json:"release_title"` + ReleaseContent string `json:"release_content"` + Artifacts []string `json:"artifacts"` +} + // Translate translates messages in WorkflowNodeRun func (r *WorkflowRun) Translate(lang string) { for ki, info := range r.Infos {