Merge pull request #4159 from gratipay/readmes

Here's a first crack at syncing readmes

bin/sync-npm.sh

@@ -31,5 +31,4 @@ pip install -e .
 # Sync with npm.
 # ==============
 
-URL=https://registry.npmjs.com/-/all
-curl $URL | sync-npm serialize /dev/stdin | sync-npm upsert /dev/stdin
+curl https://registry.npmjs.com/-/all | sync-npm serialize | sync-npm upsert

gratipay/billing/payday.py

@@ -13,7 +13,6 @@
 
 import os
 import itertools
-from multiprocessing.dummy import Pool as ThreadPool
 
 import braintree
 
@@ -24,35 +23,14 @@
 )
 from gratipay.exceptions import NegativeBalance
 from gratipay.models import check_db
+from gratipay.utils.threaded_map import threaded_map
 from psycopg2 import IntegrityError
 
 
 with open(os.path.join(os.path.dirname(__file__), '../../sql/payday.sql')) as f:
     PAYDAY = f.read()
 
 
-class ExceptionWrapped(Exception): pass
-
-
-def threaded_map(func, iterable, threads=5):
-    pool = ThreadPool(threads)
-    def g(*a, **kw):
-        # Without this wrapper we get a traceback from inside multiprocessing.
-        try:
-            return func(*a, **kw)
-        except Exception as e:
-            import traceback
-            raise ExceptionWrapped(e, traceback.format_exc())
-    try:
-        r = pool.map(g, iterable)
-    except ExceptionWrapped as e:
-        print(e.args[1])
-        raise e.args[0]
-    pool.close()
-    pool.join()
-    return r
-
-
 class NoPayday(Exception):
     __str__ = lambda self: "No payday found where one was expected."
 

gratipay/package_managers/readmes.py

@@ -0,0 +1,59 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sys
+
+import requests
+
+from gratipay.utils import markdown
+from gratipay.utils.threaded_map import threaded_map
+from threading import Lock
+
+
+log_lock = Lock()
+
+def log(*a, **kw):
+    with log_lock:
+        print(*a, file=sys.stderr, **kw)
+
+
+def http_fetch(package_name):
+    r = requests.get('https://registry.npmjs.com/' + package_name)
+    if r.status_code != 200:
+        log(r.status_code, 'for', package_name)
+        return None
+    return r.json()
+
+
+def Syncer(db):
+    def sync(dirty, fetch=http_fetch):
+        """Update all info for one package.
+        """
+        log(dirty.name)
+        full = fetch(dirty.name)
+        if not full:
+            return  # try again later
+        assert full['name'] == dirty.name
+
+        db.run('''
+
+            UPDATE packages
+               SET readme=%s
+                 , readme_raw=%s
+                 , readme_type=%s
+             WHERE package_manager=%s
+               AND name=%s
+
+        ''', ( markdown.marky(full['readme'])
+             , full['readme']
+             , 'x-markdown/npm'
+             , dirty.package_manager
+             , dirty.name
+              ))
+
+    return sync
+
+
+def sync_all(db):
+    dirty = db.all('SELECT package_manager, name FROM packages WHERE readme_raw IS NULL '
+                   'ORDER BY package_manager DESC, name DESC')
+    threaded_map(Syncer(db), dirty, 10)

gratipay/package_managers/sync.py

@@ -9,6 +9,7 @@
 import uuid
 
 import ijson.backends.yajl2_cffi as ijson
+from gratipay.package_managers import readmes as _readmes
 
 
 log = lambda *a: print(*a, file=sys.stderr)
@@ -121,16 +122,16 @@ def upsert(args):
         """)
 
 
+def readmes(args):
+    from gratipay import wireup
+    db = wireup.db(wireup.env())
+    _readmes.sync_all(db)
+
+
 def parse_args(argv):
     p = argparse.ArgumentParser()
-    p.add_argument('command', choices=['serialize', 'upsert'])
-    p.add_argument('path', help="the path to the input file")
-    p.add_argument( '-i', '--if_modified_since'
-                  , help='a number of minutes in the past, past which we would like to see new '
-                         'updates (only meaningful for `serialize`; -1 means all!)'
-                  , type=int
-                  , default=-1
-                   )
+    p.add_argument('command', choices=['serialize', 'upsert', 'readmes'])
+    p.add_argument('path', help='the path to the input file', nargs='?', default='/dev/stdin')
     return p.parse_args(argv)
 
 

gratipay/utils/markdown.py

@@ -15,5 +15,7 @@ def render(markdown):
 def marky(markdown):
     """Process markdown the same way npm does.
     """
+    if type(markdown) is unicode:
+        markdown = markdown.encode('utf8')
     marky = Popen(("marky-markdown", "/dev/stdin"), stdin=PIPE, stdout=PIPE)
     return Markup(marky.communicate(markdown)[0])

gratipay/utils/threaded_map.py

@@ -0,0 +1,26 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from multiprocessing.dummy import Pool as ThreadPool
+
+
+class ExceptionWrapped(Exception):
+    pass
+
+
+def threaded_map(func, iterable, threads=5):
+    pool = ThreadPool(threads)
+    def g(*a, **kw):
+        # Without this wrapper we get a traceback from inside multiprocessing.
+        try:
+            return func(*a, **kw)
+        except Exception as e:
+            import traceback
+            raise ExceptionWrapped(e, traceback.format_exc())
+    try:
+        r = pool.map(g, iterable)
+    except ExceptionWrapped as e:
+        print(e.args[1])
+        raise e.args[0]
+    pool.close()
+    pool.join()
+    return r

sql/branch.sql

@@ -0,0 +1,5 @@
+BEGIN;
+    ALTER TABLE packages ALTER COLUMN readme_raw DROP NOT NULL;
+    ALTER TABLE packages ALTER COLUMN readme_raw SET DEFAULT NULL;
+    UPDATE packages SET readme_raw=NULL;
+END;

tests/py/test_npm_sync.py

@@ -5,6 +5,7 @@
 from subprocess import Popen, PIPE
 
 from gratipay.testing import Harness
+from gratipay.package_managers import readmes
 
 
 def load(raw):
@@ -80,3 +81,28 @@ def test_sn_handles_empty_description_and_emails(self):
         assert package.name == 'empty-description'
         assert package.description == ''
         assert package.emails == []
+
+
+    # rs - readmes.Syncer
+
+    def test_rs_syncs_a_readme(self):
+        self.db.run("INSERT INTO packages (package_manager, name, description, emails) "
+                    "VALUES ('npm', 'foo-package', 'A package', ARRAY[]::text[])")
+
+        class DirtyPackage:
+            package_manager = 'npm'
+            name = 'foo-package'
+
+        def fetch(name):
+            return {'name': 'foo-package', 'readme': '# Greetings, program!'}
+
+        readmes.Syncer(self.db)(DirtyPackage(), fetch=fetch)
+
+        package = self.db.one('SELECT * FROM packages')
+        assert package.name == 'foo-package'
+        assert package.description == 'A package'
+        assert package.readme == '<h1 id="user-content-greetings-program" class="deep-link">' \
+                                 '<a href="#greetings-program">Greetings, program!</a></h1>\n'
+        assert package.readme_raw == '# Greetings, program!'
+        assert package.readme_type == 'x-markdown/npm'
+        assert package.emails == []