feat: Initial work on Travis webhook

tests/zeus/providers/travis/test_webhook.py

@@ -0,0 +1,218 @@
+from zeus import factories
+from zeus.constants import Result, Status
+from zeus.models import Build, Job
+
+EXAMPLE = b"""{
+  "id": 288639281,
+  "number": "2843",
+  "config": {
+    "sudo": false,
+    "dist": "trusty",
+    "language": "python",
+    "python": [
+      "3.5.2"
+    ],
+    "branches": {
+      "only": [
+        "master"
+      ]
+    },
+    "cache": {
+      "pip": true,
+      "directories": [
+        "vendor/bundle",
+        "node_modules"
+      ]
+    },
+    "deploy": {
+      "provider": "heroku",
+      "api_key": {
+        "secure": "hylw2GIHMvZKOKX3uPSaLEzVrUGEA9mzGEA0s4zK37W9HJCTnvAcmgRCwOkRuC4L7R4Zshdh/CGORNnBBgh1xx5JGYwkdnqtjHuUQmWEXCusrIURu/iEBNSsZZEPK7zBuwqMHj2yRm64JfbTDJsku3xdoA5Z8XJG5AMJGKLFgUQ="
+      },
+      "app": "docs-travis-ci-com",
+      "skip_cleanup": true,
+      "true": {
+        "branch": [
+          "master"
+        ]
+      }
+    },
+    "notifications": {
+      "slack": {
+        "rooms": {
+          "secure": "LPNgf0Ra6Vu6I7XuK7tcnyFWJg+becx1RfAR35feWK81sru8TyuldQIt7uAKMA8tqFTP8j1Af7iz7UDokbCCfDNCX1GxdAWgXs+UKpwhO89nsidHAsCkW2lWSEM0E3xtOJDyNFoauiHxBKGKUsApJTnf39H+EW9tWrqN5W2sZg8="
+        },
+        "on_success": "never"
+      },
+      "webhooks": "https://docs.travis-ci.com/update_webhook_payload_doc"
+    },
+    "install": [
+      "rvm use 2.3.1 --install",
+      "bundle install --deployment"
+    ],
+    "script": [
+      "bundle exec rake test"
+    ],
+    ".result": "configured",
+    "global_env": [
+      "PATH=$HOME/.local/user/bin:$PATH"
+    ],
+    "group": "stable"
+  },
+  "type": "cron",
+  "state": "passed",
+  "status": 0,
+  "result": 0,
+  "status_message": "Passed",
+  "result_message": "Passed",
+  "started_at": "2017-10-16T16:08:56Z",
+  "finished_at": "2017-10-16T16:12:35Z",
+  "duration": 219,
+  "build_url": "https://travis-ci.org/travis-ci/docs-travis-ci-com/builds/288639281",
+  "commit_id": 84531696,
+  "commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+  "base_commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+  "head_commit": null,
+  "branch": "master",
+  "message": "Merge pull request #1389 from christopher-dG/patch-1\\n\\nJulia: Refer to PkgDev's generate instead of Pkg's",
+  "compare_url": "https://github.com/travis-ci/docs-travis-ci-com/compare/eb58bd2fac2e339d0339689d9eb7290246805a1d...d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+  "committed_at": "2017-10-16T14:56:34Z",
+  "author_name": "Plaindocs",
+  "author_email": "[email protected]",
+  "committer_name": "GitHub",
+  "committer_email": "[email protected]",
+  "pull_request": false,
+  "pull_request_number": null,
+  "pull_request_title": null,
+  "tag": null,
+  "repository": {
+    "id": 1771959,
+    "name": "docs-travis-ci-com",
+    "owner_name": "travis-ci",
+    "url": "http://docs.travis-ci.com"
+  },
+  "matrix": [
+    {
+      "id": 288639284,
+      "repository_id": 1771959,
+      "parent_id": 288639281,
+      "number": "2843.1",
+      "state": "passed",
+      "config": {
+        "sudo": false,
+        "dist": "trusty",
+        "language": "python",
+        "python": "3.5.2",
+        "branches": {
+          "only": [
+            "master"
+          ]
+        },
+        "cache": {
+          "pip": true,
+          "directories": [
+            "vendor/bundle",
+            "node_modules"
+          ]
+        },
+        "notifications": {
+          "slack": {
+            "rooms": {
+              "secure": "LPNgf0Ra6Vu6I7XuK7tcnyFWJg+becx1RfAR35feWK81sru8TyuldQIt7uAKMA8tqFTP8j1Af7iz7UDokbCCfDNCX1GxdAWgXs+UKpwhO89nsidHAsCkW2lWSEM0E3xtOJDyNFoauiHxBKGKUsApJTnf39H+EW9tWrqN5W2sZg8="
+            },
+            "on_success": "never"
+          },
+          "webhooks": "https://docs.travis-ci.com/update_webhook_payload_doc"
+        },
+        "install": [
+          "rvm use 2.3.1 --install",
+          "bundle install --deployment"
+        ],
+        "script": [
+          "bundle exec rake test"
+        ],
+        ".result": "configured",
+        "global_env": [
+          "PATH=$HOME/.local/user/bin:$PATH"
+        ],
+        "group": "stable",
+        "os": "linux",
+        "addons": {
+          "deploy": {
+            "provider": "heroku",
+            "api_key": {
+              "secure": "hylw2GIHMvZKOKX3uPSaLEzVrUGEA9mzGEA0s4zK37W9HJCTnvAcmgRCwOkRuC4L7R4Zshdh/CGORNnBBgh1xx5JGYwkdnqtjHuUQmWEXCusrIURu/iEBNSsZZEPK7zBuwqMHj2yRm64JfbTDJsku3xdoA5Z8XJG5AMJGKLFgUQ="
+            },
+            "app": "docs-travis-ci-com",
+            "skip_cleanup": true,
+            "true": {
+              "branch": [
+                "master"
+              ]
+            }
+          }
+        }
+      },
+      "status": 0,
+      "result": 0,
+      "commit": "d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+      "branch": "master",
+      "message": "Merge pull request #1389 from christopher-dG/patch-1\\n\\nJulia: Refer to PkgDev's generate instead of Pkg's",
+      "compare_url": "https://github.com/travis-ci/docs-travis-ci-com/compare/eb58bd2fac2e339d0339689d9eb7290246805a1d...d79e3a6ff0cada29d731ed93de203f76a81d02c0",
+      "started_at": null,
+      "finished_at": null,
+      "committed_at": "2017-10-16T14:56:34Z",
+      "author_name": "Plaindocs",
+      "author_email": "[email protected]",
+      "committer_name": "GitHub",
+      "committer_email": "[email protected]",
+      "allow_failure": false
+    }
+  ]
+}"""
+
+
+def test_missing_payload(client, default_repo, default_hook):
+    path = '/hooks/{}/{}/provider/travis/webhook/'.format(
+        default_hook.id, default_hook.get_signature(),
+    )
+
+    resp = client.post(path)
+    assert resp.status_code == 400, repr(resp.data)
+
+
+def test_queued_build(client, default_repo, default_hook, default_revision, mocker):
+    source = factories.SourceFactory.create(revision=default_revision)
+
+    mock_identify_revision = mocker.patch(
+        'zeus.api.resources.repository_builds.identify_revision')
+
+    mock_identify_revision.return_value = default_revision
+
+    path = '/hooks/{}/{}/provider/travis/webhook/'.format(
+        default_hook.id, default_hook.get_signature(),
+    )
+
+    resp = client.post(path, data={
+        'payload': EXAMPLE,
+    })
+    assert resp.status_code == 200, repr(resp.data)
+
+    build = Build.query.unrestricted_unsafe().filter(
+        Build.provider == 'travis',
+        Build.external_id == '288639281',
+    ).first()
+    assert build
+    assert build.repository_id == default_repo.id
+    assert build.source_id == source.id
+    assert build.label == default_revision.subject
+
+    job = Job.query.unrestricted_unsafe().filter(
+        Job.provider == 'travis',
+        Job.external_id == '288639284',
+    ).first()
+    assert job
+    assert job.build_id == build.id
+    assert job.repository_id == default_repo.id
+    assert job.status == Status.finished
+    assert job.result == Result.passed

zeus/api/resources/repository_builds.py

@@ -127,6 +127,9 @@ def post(self, repo: Repository):
             if not build.label:
                 build.label = source.revision.message.split('\n')[0]
 
+        if not build.label:
+            return self.error('missing build label')
+
         db.session.add(build)
 
         try:

zeus/api/utils/upserts.py

@@ -0,0 +1,74 @@
+from flask import Response
+
+from zeus.config import redis
+from zeus.models import Repository, Build, Job
+from zeus.api import client
+
+
+def upsert_job(build: Build, provider: str, external_id: str, data: dict=None) -> Response:
+    lock_key = 'upsert:job:{build_id}:{provider}:{job_xid}'.format(
+        build_id=build.id,
+        provider=provider,
+        job_xid=external_id,
+    )
+    with redis.lock(lock_key):
+        json = data.copy() if data else {}
+        json['external_id'] = external_id
+        json['provider'] = provider
+
+        job = Job.query.filter(
+            Job.provider == provider,
+            Job.external_id == external_id,
+            Job.build_id == build.id,
+        ).first()
+
+        if job:
+            return client.put(
+                '/repos/{}/builds/{}/jobs/{}'.format(
+                    build.repository.get_full_name(),
+                    job.build.number,
+                    job.number,
+                ),
+                json=json
+            )
+
+        return client.post(
+            '/repos/{}/builds/{}/jobs'.format(
+                build.repository.get_full_name(),
+                build.number,
+            ),
+            json=json
+        )
+
+
+def upsert_build(repository: Repository, provider: str,
+                 external_id: str, data: dict=None) -> Response:
+    lock_key = 'hook:build:{repo_id}:{provider}:{build_xid}'.format(
+        repo_id=repository.id,
+        provider=provider,
+        build_xid=external_id,
+    )
+    with redis.lock(lock_key):
+        json = data.copy() if data else {}
+        json['external_id'] = external_id
+        json['provider'] = provider
+
+        build = Build.query.filter(
+            Build.provider == provider,
+            Build.external_id == external_id,
+        ).first()
+
+        if build:
+            return client.put(
+                '/repos/{}/builds/{}'.format(
+                    repository.get_full_name(),
+                    build.number,
+                ),
+                json=json
+            )
+        return client.post(
+            '/repos/{}/builds'.format(
+                repository.get_full_name(),
+            ),
+            json=json
+        )

zeus/providers/travis/webhook.py

@@ -0,0 +1,61 @@
+import json
+
+from flask import request, Response
+
+from zeus.api.utils.upserts import upsert_build, upsert_job
+from zeus.models import Build
+from zeus.web.hooks.base import BaseHook
+
+
+def get_result(state):
+    return {
+        'pending': 'in_progress',
+        'passed': 'passed',
+        'fixed': 'passed',
+        'failed': 'failed',
+        'broken': 'failed',
+        'failing': 'failed',
+        'errored': 'failed',
+        'canceled': 'aborted',
+    }.get(state, 'unknown')
+
+
+class TravisWebhookView(BaseHook):
+    def post(self, hook):
+        # TODO(dcramer): We should add verification using travis signatures.. however
+        # this API is already considered semi secure (and insecure) by using hook tokens
+        payload = request.form.get('payload')
+        if not payload:
+            return Response(status=400)
+
+        try:
+            payload = json.loads(payload)
+        except (TypeError, ValueError):
+            return Response(status=400)
+
+        data = {
+            'ref': payload['commit'],
+        }
+
+        if payload['pull_request']:
+            data['label'] = 'PR #{}'.format(payload['pull_request_number'])
+
+        response = upsert_build(
+            repository=hook.repository,
+            provider=hook.provider,
+            external_id=str(payload['id']),
+            data=data,
+        )
+        build = Build.query.get(response.json()['id'])
+        for job_payload in payload['matrix']:
+            upsert_job(
+                build=build,
+                provider=hook.provider,
+                external_id=str(job_payload['id']),
+                data={
+                    'status': 'finished' if job_payload['status'] is not None else 'in_progress',
+                    'result': get_result(job_payload['state']),
+                }
+            )
+
+        return Response(status=200)

zeus/web/__init__.py

@@ -3,6 +3,8 @@
 from . import hooks as h
 from . import views as v
 
+from ..providers.travis.webhook import TravisWebhookView
+
 app = Blueprint('web', __name__)
 app.add_url_rule(
     '/auth/github',
@@ -33,5 +35,9 @@
     '/hooks/<hook_id>/<signature>/builds/<build_xid>/jobs/<job_xid>/artifacts',
     view_func=h.JobArtifactsHook.as_view('job-artifacts-hook')
 )
+app.add_url_rule(
+    '/hooks/<hook_id>/<signature>/provider/travis/webhook/',
+    view_func=TravisWebhookView.as_view('travis-webhook')
+)
 app.add_url_rule('/<path:path>', view_func=v.index)
 app.add_url_rule('/', 'index', view_func=v.index)