feat(api,hooks): add variable {{.payload}} (#3748)

* feat(api,hooks): add variable {{.paylaod}} close #3612 Signed-off-by: Yvonnick Esnault <[email protected]>
ovh · Dec 20, 2018 · db2a59e · db2a59e
1 parent 6a2a57e
commit db2a59e
Show file tree

Hide file tree

Showing 13 changed files with 105 additions and 8 deletions.
diff --git a/engine/api/suggest.go b/engine/api/suggest.go
@@ -128,6 +128,7 @@ func (api *API) getVariablesHandler() service.Handler {
 			"{{.cds.worker}}",
 			"{{.cds.workflow}}",
 			"{{.cds.workspace}}",
+			"{{.payload}}",
 		}
 		allVariables = append(allVariables, cdsVar...)
 

diff --git a/engine/api/workflow/process_node.go b/engine/api/workflow/process_node.go
@@ -3,6 +3,7 @@ package workflow
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -231,6 +232,16 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr
 
 	run.Manual = manual
 	if manual != nil {
+		payloadStr, err := json.Marshal(manual.Payload)
+		if err != nil {
+			log.Error("processNode> Unable to marshal payload: %v", err)
+		}
+		run.BuildParameters = append(run.BuildParameters, sdk.Parameter{
+			Name:  "payload",
+			Type:  sdk.TextParameter,
+			Value: string(payloadStr),
+		})
+
 		e := dump.NewDefaultEncoder(new(bytes.Buffer))
 		e.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
 		e.ExtraFields.DetailedMap = false
@@ -239,7 +250,7 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr
 		e.ExtraFields.Type = false
 		m1, errm1 := e.ToStringMap(manual.Payload)
 		if errm1 != nil {
-			return report, false, sdk.WrapError(errm1, "r> Unable to compute payload")
+			return report, false, sdk.WrapError(errm1, "processNode> Unable to compute payload")
 		}
 		runPayload = sdk.ParametersMapMerge(runPayload, m1)
 		run.Payload = runPayload

diff --git a/engine/api/workflow/process_parameters.go b/engine/api/workflow/process_parameters.go
@@ -189,7 +189,10 @@ func getParentParameters(w *sdk.WorkflowRun, nodeRuns []*sdk.WorkflowNodeRun, pa
 			}
 
 			prefix := "workflow." + nodeName + "."
-			if strings.HasPrefix(p.Name, "cds.") {
+
+			if p.Name == "payload" {
+				// keep p.Name as is
+			} else if strings.HasPrefix(p.Name, "cds.") {
 				p.Name = strings.Replace(p.Name, "cds.", prefix, 1)
 			} else {
 				p.Name = prefix + p.Name

diff --git a/engine/hooks/kafka.go b/engine/hooks/kafka.go
@@ -159,6 +159,7 @@ func (s *Service) doKafkaTaskExecution(t *sdk.TaskExecution) (*sdk.WorkflowNodeR
 		return nil, sdk.WrapError(err, "Unable to dump body %s", t.WebHook.RequestBody)
 	}
 	h.Payload = m
+	h.Payload["payload"] = string(t.Kafka.Message)
 
 	return &h, nil
 }
diff --git a/engine/hooks/outgoing_hooks.go b/engine/hooks/outgoing_hooks.go
@@ -212,6 +212,7 @@ func (s *Service) doOutgoingWorkflowExecution(t *sdk.TaskExecution) error {
 		} else {
 			log.Error("Hooks> doOutgoingWorkflowExecution> Cannot unmarshall payload %s", err)
 		}
+		payloadValues["payload"] = string(payloadstr)
 	}
 
 	evt := sdk.WorkflowNodeRunHookEvent{

diff --git a/engine/hooks/poller.go b/engine/hooks/poller.go
@@ -25,6 +25,12 @@ func fillPayload(pushEvent sdk.VCSPushEvent) map[string]string {
 	payload["cds.triggered_by.email"] = pushEvent.Commit.Author.Email
 	payload["git.message"] = pushEvent.Commit.Message
 
+	payloadStr, err := json.Marshal(pushEvent)
+	if err != nil {
+		log.Error("Unable to marshal payload: %v", err)
+	}
+	payload["payload"] = string(payloadStr)
+
 	if strings.HasPrefix(pushEvent.Branch.DisplayID, "refs/tags/") {
 		payload["git.tag"] = strings.TrimPrefix(pushEvent.Branch.DisplayID, "refs/tags/")
 	}
@@ -78,6 +84,7 @@ func (s *Service) doPollerTaskExecution(task *sdk.Task, taskExec *sdk.TaskExecut
 		} else {
 			log.Error("Hooks> doPollerTaskExecution> Cannot unmarshall payload %s", err)
 		}
+		payloadValues["payload"] = string(payload.Value)
 	}
 
 	var hookEvents []sdk.WorkflowNodeRunHookEvent

diff --git a/engine/hooks/rabbitmq.go b/engine/hooks/rabbitmq.go
@@ -51,12 +51,12 @@ func (s *Service) startRabbitMQHook(t *sdk.Task) error {
 
 	deliveries, errConsume := consumer.channel.Consume(
 		t.Config[sdk.RabbitMQHookModelQueue].Value, // name
-		consumer.tag,                               // consumerTag,
-		false,                                      // noAck
-		false,                                      // exclusive
-		false,                                      // noLocal
-		false,                                      // noWait
-		nil,                                        // arguments
+		consumer.tag, // consumerTag,
+		false,        // noAck
+		false,        // exclusive
+		false,        // noLocal
+		false,        // noWait
+		nil,          // arguments
 	)
 	if errConsume != nil {
 		_ = s.stopTask(t)
@@ -128,6 +128,7 @@ func (s *Service) doRabbitMQTaskExecution(t *sdk.TaskExecution) (*sdk.WorkflowNo
 		return nil, sdk.WrapError(err, "Unable to dump body %s", t.WebHook.RequestBody)
 	}
 	h.Payload = m
+	h.Payload["payload"] = string(t.RabbitMQ.Message)
 
 	return &h, nil
 }

diff --git a/engine/hooks/scheduled_task.go b/engine/hooks/scheduled_task.go
@@ -19,6 +19,7 @@ func (s *Service) doScheduledTaskExecution(t *sdk.TaskExecution) (*sdk.WorkflowN
 
 	//Prepare the payload
 	//Anything can be pushed in the configuration, just avoid sending
+
 	payloadValues := map[string]string{}
 	if payload, ok := t.Config[sdk.Payload]; ok && payload.Value != "{}" {
 		var payloadInt interface{}
@@ -36,6 +37,7 @@ func (s *Service) doScheduledTaskExecution(t *sdk.TaskExecution) (*sdk.WorkflowN
 			} else {
 				payloadValues = m1
 			}
+			payloadValues["payload"] = payload.Value
 		} else {
 			log.Error("Hooks> doScheduledTaskExecution> Cannot unmarshall payload %s", err)
 		}

diff --git a/engine/hooks/webhook.go b/engine/hooks/webhook.go
@@ -68,6 +68,11 @@ func executeRepositoryWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEve
 		if len(pushEvent.Commits) > 0 {
 			payload["git.message"] = pushEvent.Commits[0].Message
 		}
+		payloadStr, err := json.Marshal(pushEvent)
+		if err != nil {
+			log.Error("Unable to marshal payload: %v", err)
+		}
+		payload["payload"] = string(payloadStr)
 	case GitlabHeader:
 		var pushEvent GitlabPushEvent
 		if err := json.Unmarshal(t.WebHook.RequestBody, &pushEvent); err != nil {
@@ -95,6 +100,11 @@ func executeRepositoryWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEve
 		if len(pushEvent.Commits) > 0 {
 			payload["git.message"] = pushEvent.Commits[0].Message
 		}
+		payloadStr, err := json.Marshal(pushEvent)
+		if err != nil {
+			log.Error("Unable to marshal payload: %v", err)
+		}
+		payload["payload"] = string(payloadStr)
 	case BitbucketHeader:
 		var pushEvent BitbucketPushEvent
 		if err := json.Unmarshal(t.WebHook.RequestBody, &pushEvent); err != nil {
@@ -119,6 +129,11 @@ func executeRepositoryWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEve
 		payload["cds.triggered_by.username"] = pushEvent.Actor.Name
 		payload["cds.triggered_by.fullname"] = pushEvent.Actor.DisplayName
 		payload["cds.triggered_by.email"] = pushEvent.Actor.EmailAddress
+		payloadStr, err := json.Marshal(pushEvent)
+		if err != nil {
+			log.Error("Unable to marshal payload: %v", err)
+		}
+		payload["payload"] = string(payloadStr)
 	default:
 		log.Warning("executeRepositoryWebHook> Repository manager not found. Cannot read %s", string(t.WebHook.RequestBody))
 		return nil, fmt.Errorf("Repository manager not found. Cannot read request body")
@@ -204,6 +219,7 @@ func executeWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error)
 			for k, v := range m {
 				values.Add(k, v)
 			}
+			h.Payload["payload"] = string(t.WebHook.RequestBody)
 		}
 	}
 

diff --git a/tests/fixtures/ITSCWRKFLW8/ITSCWRKFLW8.yml b/tests/fixtures/ITSCWRKFLW8/ITSCWRKFLW8.yml
@@ -0,0 +1,3 @@
+name: "ITSCWRKFLW8-WORKFLOW"
+version: v1.0
+pipeline: payload
diff --git a/tests/fixtures/ITSCWRKFLW8/payload.pip.yml b/tests/fixtures/ITSCWRKFLW8/payload.pip.yml
@@ -0,0 +1,10 @@
+version: v1.0
+name: payload
+jobs:
+- job: New Job
+  steps:
+  - name: "Payload"
+    script:
+    - mkdir -p website/sub
+    - env > thefile
+    - worker upload --tag={{.cds.version}} thefile
diff --git a/tests/sc_workflow_run_check_payload.yml b/tests/sc_workflow_run_check_payload.yml
@@ -0,0 +1,41 @@
+name: Create a simple workflow (ITSCWRKFLW8)
+testcases:
+- name: assert filepath, your current directory must be at the root of this project
+  steps:
+  - script: '[ -f ./tests/fixtures/ITSCWRKFLW8/payload.pip.yml ]'
+  - script: '[ -f ./tests/fixtures/ITSCWRKFLW8/ITSCWRKFLW8.yml ]'
+
+- name: prepare test
+  steps:
+  - script: "{{.cds.build.cdsctl}} project remove --force ITSCWRKFLW8"
+  - script: "{{.cds.build.cdsctl}} group remove --force ITSCWRKFLW8 "
+  - script: "{{.cds.build.cdsctl}} project add ITSCWRKFLW8 ITSCWRKFLW8"
+
+- name: push workflow
+  steps:
+  - script: {{.cds.build.cdsctl}} workflow push ITSCWRKFLW8 ./tests/fixtures/ITSCWRKFLW8/*.yml --skip-update-files
+    assertions:
+    - result.code ShouldEqual 0
+
+- name: run workflow
+  steps:
+  - script:  {{.cds.build.cdsctl}} workflow run ITSCWRKFLW8 ITSCWRKFLW8-WORKFLOW -d '{"aa":"bb","cc":["dd","ee"],"ff":{"gg":"hh"}}'
+    assertions:
+    - result.code ShouldEqual 0
+    - "result.systemout ShouldContainSubstring Workflow ITSCWRKFLW8-WORKFLOW #1 has been launched"
+
+- name: the workflow should be successful
+  steps:
+  - script:  {{.cds.build.cdsctl}} workflow status ITSCWRKFLW8 ITSCWRKFLW8-WORKFLOW 1 --format json
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemoutjson.status ShouldEqual Success
+    retry: 30
+    delay: 10
+
+- name: download artifact with cdsctl
+  steps:
+  - script: rm -f thefile
+  - script: {{.cds.build.cdsctl}} workflow artifact download ITSCWRKFLW8 ITSCWRKFLW8-WORKFLOW 1
+  - script: cat thefile|grep PAYLOAD
+  - script: cat thefile|grep PAYLOAD|grep '{"aa":"bb","cc":\["dd","ee"\],"ff":{"gg":"hh"}}'
diff --git a/tests/sc_workflow_run_simple_serve.yml → tests/sc_workflow_simple_serve.yml b/tests/sc_workflow_run_simple_serve.yml → tests/sc_workflow_simple_serve.yml