Skip to content

Commit

Permalink
fix(hooks): run multiple events on a single push event from bitbucket (…
Browse files Browse the repository at this point in the history 
…#3915)

Signed-off-by: Benjamin Coenen <[email protected]>

1. Related issues
close #3907
  • Loading branch information
@bnjjj
bnjjj authored Feb 6, 2019
1 parent 9b2fd52 commit 444cf46
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 63 deletions.
2 changes: 1 addition & 1 deletion engine/hooks/tasks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ func (s *Service) doTask(ctx context.Context, t *sdk.Task, e *sdk.TaskExecution)
case e.Type == TypeOutgoingWorkflow:
err = s.doOutgoingWorkflowExecution(e)
case e.WebHook != nil && (e.Type == TypeWebHook || e.Type == TypeRepoManagerWebHook):
h, err = s.doWebHookExecution(e)
hs, err = s.doWebHookExecution(e)
case e.ScheduledTask != nil && e.Type == TypeScheduler:
h, err = s.doScheduledTaskExecution(e)
doRestart = true
Expand Down
146 changes: 121 additions & 25 deletions engine/hooks/tasks_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,14 @@ func Test_doWebHookExecution(t *testing.T) {
RequestURL: "uid=42413e87905b813a375c7043ce9d4047b7e265ae3730b60180cad02ae81cc62385e5b05b9e7c758b15bb3872498a5e88963f3deac308f636baf345ed9cf1b259&project=IRTM&name=rtm-packaging&branch=master&hash=123456789&message=monmessage&author=sguiheux",
},
}
h, err := s.doWebHookExecution(task)
hs, err := s.doWebHookExecution(task)
test.NoError(t, err)

assert.Equal(t, "master", h.Payload["branch"])
assert.Equal(t, "sguiheux", h.Payload["author"])
assert.Equal(t, "monmessage", h.Payload["message"])
assert.Equal(t, "123456789", h.Payload["hash"])
assert.Equal(t, 1, len(hs))
assert.Equal(t, "master", hs[0].Payload["branch"])
assert.Equal(t, "sguiheux", hs[0].Payload["author"])
assert.Equal(t, "monmessage", hs[0].Payload["message"])
assert.Equal(t, "123456789", hs[0].Payload["hash"])
}
func Test_doWebHookExecutionGithub(t *testing.T) {
log.SetLogger(t)
Expand All @@ -47,13 +48,14 @@ func Test_doWebHookExecutionGithub(t *testing.T) {
RequestURL: "",
},
}
h, err := s.doWebHookExecution(task)
hs, err := s.doWebHookExecution(task)
test.NoError(t, err)

assert.Equal(t, "my-branch", h.Payload["git.branch"])
assert.Equal(t, "baxterthehacker", h.Payload["git.author"])
assert.Equal(t, "Update README.md", h.Payload["git.message"])
assert.Equal(t, "0d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c", h.Payload["git.hash"])
assert.Equal(t, 1, len(hs))
assert.Equal(t, "my-branch", hs[0].Payload["git.branch"])
assert.Equal(t, "baxterthehacker", hs[0].Payload["git.author"])
assert.Equal(t, "Update README.md", hs[0].Payload["git.message"])
assert.Equal(t, "0d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c", hs[0].Payload["git.hash"])
}

func Test_doWebHookExecutionTagGithub(t *testing.T) {
Expand All @@ -70,14 +72,15 @@ func Test_doWebHookExecutionTagGithub(t *testing.T) {
RequestURL: "",
},
}
h, err := s.doWebHookExecution(task)
hs, err := s.doWebHookExecution(task)
test.NoError(t, err)

assert.Equal(t, "", h.Payload["git.branch"])
assert.Equal(t, "my-branch", h.Payload["git.tag"])
assert.Equal(t, "baxterthehacker", h.Payload["git.author"])
assert.Equal(t, "Update README.md", h.Payload["git.message"])
assert.Equal(t, "0d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c", h.Payload["git.hash"])
assert.Equal(t, 1, len(hs))
assert.Equal(t, "", hs[0].Payload["git.branch"])
assert.Equal(t, "my-branch", hs[0].Payload["git.tag"])
assert.Equal(t, "baxterthehacker", hs[0].Payload["git.author"])
assert.Equal(t, "Update README.md", hs[0].Payload["git.message"])
assert.Equal(t, "0d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c", hs[0].Payload["git.hash"])
}

func Test_doWebHookExecutionGitlab(t *testing.T) {
Expand All @@ -94,13 +97,14 @@ func Test_doWebHookExecutionGitlab(t *testing.T) {
RequestURL: "",
},
}
h, err := s.doWebHookExecution(task)
hs, err := s.doWebHookExecution(task)
test.NoError(t, err)

assert.Equal(t, "master", h.Payload["git.branch"])
assert.Equal(t, "jsmith", h.Payload["git.author"])
assert.Equal(t, "Update Catalan translation to e38cb41.", h.Payload["git.message"])
assert.Equal(t, "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", h.Payload["git.hash"])
assert.Equal(t, 1, len(hs))
assert.Equal(t, "master", hs[0].Payload["git.branch"])
assert.Equal(t, "jsmith", hs[0].Payload["git.author"])
assert.Equal(t, "Update Catalan translation to e38cb41.", hs[0].Payload["git.message"])
assert.Equal(t, "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", hs[0].Payload["git.hash"])
}

func Test_doWebHookExecutionBitbucket(t *testing.T) {
Expand All @@ -117,12 +121,39 @@ func Test_doWebHookExecutionBitbucket(t *testing.T) {
RequestURL: "",
},
}
h, err := s.doWebHookExecution(task)
hs, err := s.doWebHookExecution(task)
test.NoError(t, err)

assert.Equal(t, "name-of-branch", h.Payload["git.branch"])
assert.Equal(t, "steven.guiheux", h.Payload["git.author"])
assert.Equal(t, "9f4fac7ec5642099982a86f584f2c4a362adb670", h.Payload["git.hash"])
assert.Equal(t, 1, len(hs))
assert.Equal(t, "name-of-branch", hs[0].Payload["git.branch"])
assert.Equal(t, "steven.guiheux", hs[0].Payload["git.author"])
assert.Equal(t, "9f4fac7ec5642099982a86f584f2c4a362adb670", hs[0].Payload["git.hash"])
}

func Test_doWebHookExecutionBitbucketMultiple(t *testing.T) {
log.SetLogger(t)
s := Service{}
task := &sdk.TaskExecution{
UUID: sdk.RandomString(10),
Type: TypeRepoManagerWebHook,
WebHook: &sdk.WebHookExecution{
RequestBody: []byte(bitbucketMultiplePushEvent),
RequestHeader: map[string][]string{
BitbucketHeader: {"repo:refs_changed"},
},
RequestURL: "",
},
}
hs, err := s.doWebHookExecution(task)
test.NoError(t, err)

assert.Equal(t, 2, len(hs))
assert.Equal(t, "name-of-branch", hs[0].Payload["git.branch"])
assert.Equal(t, "steven.guiheux", hs[0].Payload["git.author"])
assert.Equal(t, "9f4fac7ec5642099982a86f584f2c4a362adb670", hs[0].Payload["git.hash"])
assert.Equal(t, "name-of-branch-bis", hs[1].Payload["git.branch"])
assert.Equal(t, "steven.guiheux", hs[1].Payload["git.author"])
assert.Equal(t, "9f4fac7ec5642099982a86f584f2c4a362adb670", hs[0].Payload["git.hash"])
}

var bitbucketPushEvent = `
Expand Down Expand Up @@ -179,6 +210,71 @@ var bitbucketPushEvent = `
}
`

var bitbucketMultiplePushEvent = `
{
"eventKey": "repo:refs_changed",
"date": "2017-11-30T15:24:01+0100",
"actor": {
"name": "steven.guiheux",
"emailAddress": "[email protected]",
"id": 1363,
"displayName": "Steven Guiheux",
"active": true,
"slug": "steven.guiheux",
"type": "NORMAL"
},
"repository": {
"slug": "sseclient",
"id": 6096,
"name": "sseclient",
"scmId": "git",
"state": "AVAILABLE",
"statusMessage": "Available",
"forkable": true,
"project": {
"key": "~STEVEN.GUIHEUX",
"id": 112,
"name": "Steven Guiheux",
"type": "PERSONAL",
"owner": {
"name": "steven.guiheux",
"emailAddress": "[email protected]",
"id": 1363,
"displayName": "Steven Guiheux",
"active": true,
"slug": "steven.guiheux",
"type": "NORMAL"
}
},
"public": true
},
"changes": [
{
"ref": {
"id": "refs/heads/name-of-branch",
"displayId": "name-of-branch",
"type": "BRANCH"
},
"refId": "refs/heads/name-of-branch",
"fromHash": "0000000000000000000000000000000000000000",
"toHash": "9f4fac7ec5642099982a86f584f2c4a362adb670",
"type": "ADD"
},
{
"ref": {
"id": "refs/heads/name-of-branch-bis",
"displayId": "name-of-branch-bis",
"type": "BRANCH"
},
"refId": "refs/heads/name-of-branch-bis",
"fromHash": "0000000000000000000000000000000000000000",
"toHash": "9f4fac7ec5642099982a86f584f2c4a362adb670",
"type": "ADD"
}
]
}
`

var gitlabPushEvent = `
{
"object_kind": "push",
Expand Down
95 changes: 58 additions & 37 deletions engine/hooks/webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,17 @@ import (
"github.com/ovh/cds/sdk/log"
)

func (s *Service) doWebHookExecution(e *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error) {
func (s *Service) doWebHookExecution(e *sdk.TaskExecution) ([]sdk.WorkflowNodeRunHookEvent, error) {
log.Debug("Hooks> Processing webhook %s %s", e.UUID, e.Type)

if e.Type == TypeRepoManagerWebHook {
return executeRepositoryWebHook(e)
}
return executeWebHook(e)
event, err := executeWebHook(e)
if err != nil {
return nil, err
}
return []sdk.WorkflowNodeRunHookEvent{*event}, nil
}

func getRepositoryHeader(whe *sdk.WebHookExecution) string {
Expand All @@ -34,15 +38,13 @@ func getRepositoryHeader(whe *sdk.WebHookExecution) string {
return ""
}

func executeRepositoryWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error) {
func executeRepositoryWebHook(t *sdk.TaskExecution) ([]sdk.WorkflowNodeRunHookEvent, error) {
// Prepare a struct to send to CDS API
h := sdk.WorkflowNodeRunHookEvent{
WorkflowNodeHookUUID: t.UUID,
}
payloads := []map[string]interface{}{}

payload := make(map[string]interface{})
switch getRepositoryHeader(t.WebHook) {
case GithubHeader:
payload := make(map[string]interface{})
var pushEvent GithubPushEvent
if err := json.Unmarshal(t.WebHook.RequestBody, &pushEvent); err != nil {
return nil, sdk.WrapError(err, "unable ro read github request: %s", string(t.WebHook.RequestBody))
Expand Down Expand Up @@ -73,7 +75,9 @@ func executeRepositoryWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEve
log.Error("Unable to marshal payload: %v", err)
}
payload["payload"] = string(payloadStr)
payloads = append(payloads, payload)
case GitlabHeader:
payload := make(map[string]interface{})
var pushEvent GitlabPushEvent
if err := json.Unmarshal(t.WebHook.RequestBody, &pushEvent); err != nil {
return nil, sdk.WrapError(err, "unable ro read gitlab request: %s", string(t.WebHook.RequestBody))
Expand Down Expand Up @@ -105,52 +109,69 @@ func executeRepositoryWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEve
log.Error("Unable to marshal payload: %v", err)
}
payload["payload"] = string(payloadStr)
payloads = append(payloads, payload)
case BitbucketHeader:
var pushEvent BitbucketPushEvent
if err := json.Unmarshal(t.WebHook.RequestBody, &pushEvent); err != nil {
return nil, sdk.WrapError(err, "unable ro read bitbucket request: %s", string(t.WebHook.RequestBody))
}
payload["git.author"] = pushEvent.Actor.Name
payload["git.author.email"] = pushEvent.Actor.EmailAddress

if len(pushEvent.Changes) == 0 || pushEvent.Changes[0].Type == "DELETE" {
if len(pushEvent.Changes) == 0 {
return nil, nil
}

if !strings.HasPrefix(pushEvent.Changes[0].RefID, "refs/tags/") {
payload["git.branch"] = strings.TrimPrefix(pushEvent.Changes[0].RefID, "refs/heads/")
} else {
payload["git.tag"] = strings.TrimPrefix(pushEvent.Changes[0].RefID, "refs/tags/")
}
payload["git.hash.before"] = pushEvent.Changes[0].FromHash
payload["git.hash"] = pushEvent.Changes[0].ToHash
payload["git.repository"] = fmt.Sprintf("%s/%s", pushEvent.Repository.Project.Key, pushEvent.Repository.Slug)
for _, pushChange := range pushEvent.Changes {
if pushChange.Type == "DELETE" {
// TODO: delelete all runs for this branch
continue
}
payload := make(map[string]interface{})
payload["git.author"] = pushEvent.Actor.Name
payload["git.author.email"] = pushEvent.Actor.EmailAddress

payload["cds.triggered_by.username"] = pushEvent.Actor.Name
payload["cds.triggered_by.fullname"] = pushEvent.Actor.DisplayName
payload["cds.triggered_by.email"] = pushEvent.Actor.EmailAddress
payloadStr, err := json.Marshal(pushEvent)
if err != nil {
log.Error("Unable to marshal payload: %v", err)
if !strings.HasPrefix(pushChange.RefID, "refs/tags/") {
payload["git.branch"] = strings.TrimPrefix(pushChange.RefID, "refs/heads/")
} else {
payload["git.tag"] = strings.TrimPrefix(pushChange.RefID, "refs/tags/")
}
payload["git.hash.before"] = pushChange.FromHash
payload["git.hash"] = pushChange.ToHash
payload["git.repository"] = fmt.Sprintf("%s/%s", pushEvent.Repository.Project.Key, pushEvent.Repository.Slug)

payload["cds.triggered_by.username"] = pushEvent.Actor.Name
payload["cds.triggered_by.fullname"] = pushEvent.Actor.DisplayName
payload["cds.triggered_by.email"] = pushEvent.Actor.EmailAddress
payloadStr, err := json.Marshal(pushEvent)
if err != nil {
log.Error("Unable to marshal payload: %v", err)
}
payload["payload"] = string(payloadStr)
payloads = append(payloads, payload)
}
payload["payload"] = string(payloadStr)
default:
log.Warning("executeRepositoryWebHook> Repository manager not found. Cannot read %s", string(t.WebHook.RequestBody))
return nil, fmt.Errorf("Repository manager not found. Cannot read request body")
}

d := dump.NewDefaultEncoder(&bytes.Buffer{})
d.ExtraFields.Type = false
d.ExtraFields.Len = false
d.ExtraFields.DetailedMap = false
d.ExtraFields.DetailedStruct = false
d.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
payloadValues, errDump := d.ToStringMap(payload)
if errDump != nil {
return nil, sdk.WrapError(errDump, "executeRepositoryWebHook> Cannot dump payload %+v ", payload)
hs := make([]sdk.WorkflowNodeRunHookEvent, 0, len(payloads))
for _, payload := range payloads {
h := sdk.WorkflowNodeRunHookEvent{
WorkflowNodeHookUUID: t.UUID,
}
d := dump.NewDefaultEncoder(&bytes.Buffer{})
d.ExtraFields.Type = false
d.ExtraFields.Len = false
d.ExtraFields.DetailedMap = false
d.ExtraFields.DetailedStruct = false
d.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
payloadValues, errDump := d.ToStringMap(payload)
if errDump != nil {
return nil, sdk.WrapError(errDump, "executeRepositoryWebHook> Cannot dump payload %+v ", payload)
}
h.Payload = payloadValues
hs = append(hs, h)
}
h.Payload = payloadValues
return &h, nil

return hs, nil
}

func executeWebHook(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error) {
Expand Down

0 comments on commit 444cf46

Please sign in to comment.