Merge pull request #780 from webcompat/dbdump

add incoming issues to local db, fixes #788

webcompat/db/__init__.py

@@ -6,8 +6,15 @@
 
 import os
 from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column
+from sqlalchemy import Integer
+from sqlalchemy import String
 from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import sessionmaker
+from hashlib import sha512
+from uuid import uuid4
 
 from webcompat import app
 
@@ -17,3 +24,47 @@
 session_db = scoped_session(sessionmaker(autocommit=False,
                                          autoflush=False,
                                          bind=session_engine))
+issue_engine = create_engine('sqlite:///' + os.path.join(app.config['BASE_DIR'],
+                                                   'issues.db'))
+
+issue_db = scoped_session(sessionmaker(autocommit=False,
+                                         autoflush=False,
+                                         bind=issue_engine))
+IssueBase = declarative_base()
+IssueBase.query = issue_db.query_property()
+
+class WCIssue(IssueBase):
+    __tablename__ = 'webcompat_issues'
+
+    issue_id = Column(String(128), unique=True, primary_key=True)
+    summary = Column(String(256))
+    url = Column(String(1024))
+    body = Column(String(2048))
+
+    def __init__(self, issue_id, summary, url, body):
+        self.issue_id = issue_id
+        self.summary = summary
+        self.url = url
+        self.body = body
+
+IssueBase.metadata.create_all(bind=issue_engine)
+
+UsersBase = declarative_base()
+UsersBase.query = session_db.query_property()
+
+
+class User(UsersBase):
+    __tablename__ = 'users'
+
+    user_id = Column(String(128), unique=True, primary_key=True)
+    access_token = Column(String(128), unique=True)
+
+    def __init__(self, access_token):
+        self.access_token = access_token
+        # We use the user_id in the session cookie to identify auth'd users.
+        # Here we salt and hash the GitHub access token so you can't get
+        # back to the auth token if the session cookie was ever compromised.
+        self.user_id = sha512(access_token + uuid4().hex).hexdigest()[0:128]
+
+
+UsersBase.metadata.create_all(bind=session_engine)

webcompat/helpers.py

@@ -5,10 +5,11 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import datetime
-import json
 import hashlib
+import json
 import math
 import os
+import re
 import urlparse
 
 from babel.dates import format_timedelta
@@ -43,7 +44,8 @@ def bust_cache(file_path):
 
     Uses a simple cache_dict to we don't have to hash each file for every
     request. This is kept in-memory so it will be blown away when the app
-    is restarted (which is when file changes would have been deployed).'''
+    is restarted (which is when file changes would have been deployed).
+    '''
     def get_checksum(file_path):
         try:
             checksum = cache_dict[file_path]
@@ -349,3 +351,20 @@ def wrapped_func(*args, **kwargs):
                 return (data, 200, get_fixture_headers(data))
         return func(*args, **kwargs)
     return wrapped_func
+
+
+def extract_url(issue_body):
+    '''Extract the URL for an issue from WebCompat.
+
+    URL in webcompat.com bugs follow this pattern:
+    **URL**: https://example.com/foobar
+    '''
+    url_pattern = re.compile('\*\*URL\*\*\: (.*)\n')
+    url_match = re.search(url_pattern, issue_body)
+    if url_match:
+        url = url_match.group(1).strip()
+        if not url.startswith(('http://', 'https://')):
+            url = "http://%s" % url
+    else:
+        url = ""
+    return url

webcompat/views.py

@@ -27,8 +27,8 @@
 from helpers import get_user_info
 from helpers import set_referer
 from issues import report_issue
-from models import session_db
-from models import User
+from webcompat.db import session_db
+from webcompat.db import User
 
 from webcompat import app
 from webcompat import github

webcompat/webhooks/__init__.py

@@ -11,28 +11,34 @@
 
 import json
 
-from flask import Blueprint
 from flask import abort
+from flask import Blueprint
 from flask import request
 
+from helpers import dump_to_db
 from helpers import parse_and_set_label
 
 webhooks = Blueprint('webhooks', __name__, url_prefix='/webhooks')
 
 
 @webhooks.route('/labeler', methods=['GET', 'POST'])
 def hooklistener():
-    '''Listen for the "issues" webhook event, parse the body,
-       post back labels. But only do that for the 'opened' action.'''
+    '''Listen for the "issues" webhook event, parse the body
+
+       Method posts back labels and dumps data to a local db.
+       Only in response to the 'opened' action, though.
+    '''
     if request.method == 'GET':
         abort(403)
     elif (request.method == 'POST' and
             request.headers.get('X-GitHub-Event') == 'issues'):
         payload = json.loads(request.data)
         if payload.get('action') == 'opened':
             issue_body = payload.get('issue')['body']
+            issue_title = payload.get('issue')['title']
             issue_number = payload.get('issue')['number']
             parse_and_set_label(issue_body, issue_number)
+            dump_to_db(issue_title, issue_body, issue_number)
             return ('gracias, amigo.', 200)
         else:
             return ('cool story, bro.', 200)

webcompat/webhooks/helpers.py

@@ -5,12 +5,13 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
-import os
 import re
 import requests
 
 from webcompat import app
-
+from webcompat.db import issue_db
+from webcompat.db import WCIssue
+from webcompat.helpers import extract_url
 
 def api_post(endpoint, payload, issue):
     '''Helper method to post junk to GitHub.'''
@@ -23,7 +24,8 @@ def api_post(endpoint, payload, issue):
 
 
 def parse_and_set_label(body, issue_number):
-    '''Parse the labels from the body in comments like so:
+    '''Parse the labels from the body in comment:
+
     <!-- @browser: value -->. Currently this only handles a single label,
     because that's all that we set in webcompat.com.
     '''
@@ -39,9 +41,14 @@ def parse_and_set_label(body, issue_number):
 
 
 def set_label(label, issue_number):
-    '''Do a GitHub POST request using one of our bots, which has push access
-    and set a label for the issue.'''
+    '''Do a GitHub POST request to set a label for the issue.'''
     # POST /repos/:owner/:repo/issues/:number/labels
     # ['Label1', 'Label2']
     payload = [label]
     api_post('labels', payload, issue_number)
+
+
+def dump_to_db(title, body, issue_number):
+    url = extract_url(body)
+    issue_db.add(WCIssue(issue_number, title, url, body))
+    issue_db.commit()