Merge pull request #3185 from gratipay/friends

find friends

error.spt

@@ -2,21 +2,33 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 from aspen.http import status_strings
 
+from gratipay.utils import LazyResponse
+from gratipay.utils.i18n import HTTP_ERRORS
+
 [----------------------------------------]
 
-style = ''
-msg = status_strings.get(response.code, 'Unknown Error').title()
+code = response.code
+msg = _(HTTP_ERRORS.get(code, status_strings.get(code, '')))
+try:
+    assert msg
+except Exception as e:
+    website.tell_sentry(e, state)
+
+if isinstance(response, LazyResponse):
+    response.render_body(state)
 err = response.body
+if code == 500 and not err:
+    err = _("Looks like you've found a bug! Sorry for the inconvenience, we'll get it fixed ASAP!")
 
-title = str(response.code)
 [----------------------------------------] text/html
 {% extends "templates/base.html" %}
+{% set title = code %}
 {% block content %}
     <p>{{ msg }}</p>
     <pre>{{ err }}</pre>
 {% endblock %}
 [----------------------------------------] application/json via json_dump
-{ "error_code": response.code
+{ "error_code": code
 , "error_message_short": msg
 , "error_message_long": err
  }

gratipay/elsewhere/__init__.py

@@ -12,9 +12,11 @@
 
 from aspen import log, Response
 from aspen.utils import to_age, utc
+from oauthlib.oauth2 import TokenExpiredError
 from requests_oauthlib import OAuth1Session, OAuth2Session
 
 from gratipay.elsewhere._extractors import not_available
+from gratipay.utils import LazyResponse
 
 
 ACTIONS = {'opt-in', 'connect', 'lock', 'unlock'}
@@ -110,29 +112,42 @@ def api_get(self, path, sess=None, **kw):
         The response is returned, after checking its status code and ratelimit
         headers.
         """
+        is_user_session = bool(sess)
         if not sess:
             sess = self.get_auth_session()
         response = sess.get(self.api_url+path, **kw)
 
-        self.check_api_response_status(response)
-        self.check_ratelimit_headers(response)
+        limit, remaining, reset = self.get_ratelimit_headers(response)
+        if not is_user_session:
+            self.log_ratelimit_headers(limit, remaining, reset)
 
-        return response
-
-    def check_api_response_status(self, response):
-        """Pass through any 404, convert any other non-200 into a 500.
-        """
+        # Check response status
         status = response.status_code
+        if status == 401 and isinstance(self, PlatformOAuth1):
+            # https://tools.ietf.org/html/rfc5849#section-3.2
+            if is_user_session:
+                raise TokenExpiredError
+            raise Response(500)
         if status == 404:
             raise Response(404, response.text)
-        elif status != 200:
+        if status == 429 and is_user_session:
+            def msg(_, to_age):
+                if remaining == 0 and reset:
+                    return _("You've consumed your quota of requests, you can try again in {0}.", to_age(reset))
+                else:
+                    return _("You're making requests too fast, please try again later.")
+            raise LazyResponse(status, msg)
+        if status != 200:
             log('{} api responded with {}:\n{}'.format(self.name, status, response.text)
                , level=logging.ERROR)
-            raise Response(500, '{} lookup failed with {}'.format(self.name, status))
+            msg = lambda _: _("{0} returned an error, please try again later.",
+                              self.display_name)
+            raise LazyResponse(502, msg)
 
-    def check_ratelimit_headers(self, response):
-        """Emit log messages if we're running out of ratelimit.
-        """
+        return response
+
+    def get_ratelimit_headers(self, response):
+        limit, remaining, reset = None, None, None
         prefix = getattr(self, 'ratelimit_headers_prefix', None)
         if prefix:
             limit = response.headers.get(prefix+'limit')
@@ -141,26 +156,31 @@ def check_ratelimit_headers(self, response):
 
             try:
                 limit, remaining, reset = int(limit), int(remaining), int(reset)
+                reset = datetime.fromtimestamp(reset, tz=utc)
             except (TypeError, ValueError):
-                limit, remaining, reset = None, None, None
-
-            if None in (limit, remaining, reset):
                 d = dict(limit=limit, remaining=remaining, reset=reset)
                 log('Got weird rate headers from %s: %s' % (self.name, d))
-            else:
-                percent_remaining = remaining/limit
-                if percent_remaining < 0.5:
-                    reset = to_age(datetime.fromtimestamp(reset, tz=utc))
-                    log_msg = (
-                        '{0} API: {1:.1%} of ratelimit has been consumed, '
-                        '{2} requests remaining, resets {3}.'
-                    ).format(self.name, 1 - percent_remaining, remaining, reset)
-                    log_lvl = logging.WARNING
-                    if percent_remaining < 0.2:
-                        log_lvl = logging.ERROR
-                    elif percent_remaining < 0.05:
-                        log_lvl = logging.CRITICAL
-                    log(log_msg, log_lvl)
+                limit, remaining, reset = None, None, None
+
+        return limit, remaining, reset
+
+    def log_ratelimit_headers(self, limit, remaining, reset):
+        """Emit log messages if we're running out of ratelimit.
+        """
+        if None in (limit, remaining, reset):
+            return
+        percent_remaining = remaining/limit
+        if percent_remaining < 0.5:
+            log_msg = (
+                '{0} API: {1:.1%} of ratelimit has been consumed, '
+                '{2} requests remaining, resets {3}.'
+            ).format(self.name, 1 - percent_remaining, remaining, to_age(reset))
+            log_lvl = logging.WARNING
+            if percent_remaining < 0.2:
+                log_lvl = logging.ERROR
+            elif percent_remaining < 0.05:
+                log_lvl = logging.CRITICAL
+            log(log_msg, log_lvl)
 
     def extract_user_info(self, info):
         """
@@ -194,23 +214,31 @@ def extract_user_info(self, info):
         r.extra_info = info
         return r
 
-    def get_team_members(self, team_name, page_url=None):
-        """Given a team_name on the platform, return the team's membership list
-        from the API.
+    def get_team_members(self, account, page_url=None):
+        """Given an AccountElsewhere, return its membership list from the API.
         """
-        default_url = self.api_team_members_path.format(user_name=quote(team_name))
-        r = self.api_get(page_url or default_url)
+        if not page_url:
+            page_url = self.api_team_members_path.format(
+                user_id=quote(account.user_id),
+                user_name=quote(account.user_name or ''),
+            )
+        r = self.api_get(page_url)
         members, count, pages_urls = self.api_paginator(r, self.api_parser(r))
         members = [self.extract_user_info(m) for m in members]
         return members, count, pages_urls
 
-    def get_user_info(self, user_name, sess=None):
-        """Given a user_name on the platform, get the user's info from the API.
+    def get_user_info(self, key, value, sess=None):
+        """Given a user_name or user_id, get the user's info from the API.
         """
-        try:
-            path = self.api_user_info_path.format(user_name=quote(user_name))
-        except KeyError:
-            raise Response(404)
+        if key == 'user_id':
+            path = 'api_user_info_path'
+        else:
+            assert key == 'user_name'
+            path = 'api_user_name_info_path'
+        path = getattr(self, path, None)
+        if not path:
+            raise Response(400)
+        path = path.format(**{key: value})
         info = self.api_parser(self.api_get(path, sess=sess))
         return self.extract_user_info(info)
 
@@ -224,6 +252,19 @@ def get_user_self_info(self, sess):
             info.token = json.dumps(token)
         return info
 
+    def get_friends_for(self, account, page_url=None, sess=None):
+        if not page_url:
+            page_url = self.api_friends_path.format(
+                user_id=quote(account.user_id),
+                user_name=quote(account.user_name or ''),
+            )
+        r = self.api_get(page_url, sess=sess)
+        friends, count, pages_urls = self.api_paginator(r, self.api_parser(r))
+        friends = [self.extract_user_info(f) for f in friends]
+        if count == -1 and hasattr(self, 'x_friends_count'):
+            count = self.x_friends_count(None, account.extra_info, -1)
+        return friends, count, pages_urls
+
 
 class PlatformOAuth1(Platform):
 

gratipay/elsewhere/_paginators.py

@@ -2,35 +2,86 @@
 """
 from __future__ import unicode_literals
 
+from urllib import urlencode
+from urlparse import parse_qs, urlsplit, urlunsplit
 
-def _relativize_urls(base, urls):
-    i = len(base)
-    r = {}
-    for link_key, url in urls.items():
-        if not url.startswith(base):
-            raise ValueError('"%s" is not a prefix of "%s"' % (base, url))
-        r[link_key] = url[i:]
-    return r
+
+def _modify_query(url, key, value):
+    scheme, netloc, path, query, fragment = urlsplit(url)
+    query = parse_qs(query)
+    if value is None:
+        query.pop(key, None)
+    else:
+        query[key] = [value]
+    query = urlencode(query, doseq=True)
+    return urlunsplit((scheme, netloc, path, query, fragment))
+
+
+def _strip_prefix(prefix, s):
+    """
+    >>> str(_strip_prefix('https://api.example.com', 'https://api.example.com/foo/bar'))
+    '/foo/bar'
+    >>> _strip_prefix('https://api.example.org', 'https://api.example.com/baz')
+    Traceback (most recent call last):
+        ...
+    ValueError: "https://api.example.org" is not a prefix of "https://api.example.com/baz"
+    """
+    i = len(prefix)
+    if s[:i] == prefix:
+        return s[i:]
+    raise ValueError('"%s" is not a prefix of "%s"' % (prefix, s))
 
 
 links_keys = set('prev next first last'.split())
 
 
+def query_param_paginator(param, **kw):
+    # https://developers.google.com/+/api/#pagination
+    # https://dev.twitter.com/overview/api/cursoring
+    page_key = kw.get('page')
+    total_key = kw.get('total')
+    links_keys_map = tuple((k, v) for k, v in kw.items() if k in links_keys)
+    def f(self, response, parsed):
+        url = _strip_prefix(self.api_url, response.request.url)
+        links = {k: _modify_query(url, param, parsed[k2])
+                 for k, k2 in links_keys_map
+                 if parsed.get(k2)}
+        if links.get('prev') and not links.get('first'):
+            links['first'] = _modify_query(url, param, None)
+        if page_key:
+            page = parsed[page_key]
+        else:
+            lists = [a for a in parsed.values() if isinstance(a, list)]
+            assert len(lists) == 1
+            page = next(iter(lists))
+        total_count = parsed.get(total_key, -1) if links else len(page)
+        return page, total_count, links
+    return f
+
+
 def header_links_paginator():
+    # https://developer.github.com/v3/#pagination
     def f(self, response, parsed):
-        links = {k: v['url'] for k, v in response.links.items() if k in links_keys}
+        links = {k: _strip_prefix(self.api_url, v['url'])
+                 for k, v in response.links.items()
+                 if k in links_keys}
         total_count = -1 if links else len(parsed)
-        return parsed, total_count, _relativize_urls(self.api_url, links)
+        return parsed, total_count, links
     return f
 
 
-def keys_paginator(**kw):
-    page_key = kw.get('page', 'values')
-    total_count_key = kw.get('total_count', 'size')
+def keys_paginator(page_key, **kw):
+    # https://confluence.atlassian.com/display/BITBUCKET/Version+2#Version2-Pagingthroughobjectcollections
+    # https://developers.facebook.com/docs/graph-api/using-graph-api/v2.2#paging
+    paging_key = kw.get('paging')
+    total_key = kw.get('total')
     links_keys_map = tuple((k, kw.get(k, k)) for k in links_keys)
     def f(self, response, parsed):
         page = parsed[page_key]
-        links = {k: parsed[k2] for k, k2 in links_keys_map if parsed.get(k2)}
-        total_count = parsed.get(total_count_key, -1) if links else len(page)
-        return page, total_count, _relativize_urls(self.api_url, links)
+        paging = parsed.get(paging_key, {}) if paging_key else parsed
+        links = {k: _strip_prefix(self.api_url, paging[k2])
+                 for k, k2 in links_keys_map
+                 if paging.get(k2)}
+        total_count = paging.get(total_key, -1) if links else len(page)
+        return page, total_count, links
     return f

gratipay/elsewhere/bitbucket.py

@@ -19,11 +19,13 @@ class Bitbucket(PlatformOAuth1):
 
     # API attributes
     api_format = 'json'
-    api_paginator = keys_paginator(prev='previous')
+    api_paginator = keys_paginator('values', prev='previous', total='size')
     api_url = 'https://bitbucket.org/api'
-    api_user_info_path = '/2.0/users/{user_name}'
+    api_user_info_path = '/2.0/users/{user_id}'
+    api_user_name_info_path = '/2.0/users/{user_name}'
     api_user_self_info_path = '/2.0/user'
     api_team_members_path = '/2.0/teams/{user_name}/members'
+    api_friends_path = '/2.0/users/{user_name}/following'
 
     # User info extractors
     x_user_info = key('user')

gratipay/elsewhere/bountysource.py

@@ -20,6 +20,7 @@ class Bountysource(Platform):
     name = 'bountysource'
     display_name = 'Bountysource'
     account_url = '{platform_data.auth_url}/people/{user_id}'
+    optional_user_name = True
 
     # API attributes
     api_format = 'json'

gratipay/elsewhere/facebook.py

@@ -2,6 +2,7 @@
 
 from gratipay.elsewhere import PlatformOAuth2
 from gratipay.elsewhere._extractors import key
+from gratipay.elsewhere._paginators import keys_paginator
 
 
 class Facebook(PlatformOAuth2):
@@ -10,17 +11,21 @@ class Facebook(PlatformOAuth2):
     name = 'facebook'
     display_name = 'Facebook'
     account_url = 'https://www.facebook.com/profile.php?id={user_id}'
+    optional_user_name = True
 
     # Auth attributes
     auth_url = 'https://www.facebook.com/dialog/oauth'
     access_token_url = 'https://graph.facebook.com/oauth/access_token'
-    oauth_default_scope = ['public_profile,email']
+    oauth_default_scope = ['public_profile,email,user_friends']
 
     # API attributes
     api_format = 'json'
+    api_paginator = keys_paginator('data', paging='paging', prev='previous')
     api_url = 'https://graph.facebook.com'
-    api_user_info_path = '/{user_name}'
+    api_user_name_info_path = '/{user_name}'
     api_user_self_info_path = '/me'
+    api_friends_path = '/v2.2/{user_id}/friends'
+    api_friends_limited = True
 
     # User info extractors
     x_user_id = key('id')

gratipay/elsewhere/github.py

@@ -23,9 +23,11 @@ class GitHub(PlatformOAuth2):
     api_format = 'json'
     api_paginator = header_links_paginator()
     api_url = 'https://api.github.com'
-    api_user_info_path = '/users/{user_name}'
+    api_user_info_path = '/user/{user_id}'
+    api_user_name_info_path = '/users/{user_name}'
     api_user_self_info_path = '/user'
     api_team_members_path = '/orgs/{user_name}/public_members'
+    api_friends_path = '/users/{user_name}/following'
     ratelimit_headers_prefix = 'x-ratelimit-'
 
     # User info extractors