Merge pull request #4211 from gratipay/rip-out-readmes

Rip out READMEs after all

gratipay/sync_npm/__init__.py

@@ -11,6 +11,8 @@
 import sys
 from threading import Lock
 
+from gratipay import wireup
+
 
 log_lock = Lock()
 
@@ -19,3 +21,23 @@ def log(*a, **kw):
     """
     with log_lock:
         print(*a, file=sys.stderr, **kw)
+
+
+class sentry(object):
+    """This is a context manager to log to sentry. You have to pass in an ``Environment``
+    object with a ``sentry_dsn`` attribute.
+    """
+
+    def __init__(self, env, noop=None):
+        try:
+            sys.stdout = sys.stderr  # work around aspen.log_dammit limitation; sigh
+            self.tell_sentry = wireup.make_sentry_teller(env, noop)
+        finally:
+            sys.stdout = sys.__stdout__
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.tell_sentry(exc_type, {})
+        return False

gratipay/sync_npm/cli.py

@@ -7,20 +7,18 @@
 import argparse
 
 from gratipay import wireup
-from gratipay.sync_npm import serialize, upsert, fetch_readmes, process_readmes
+from gratipay.sync_npm import serialize, upsert
 
 
 def parse_args(argv):
     p = argparse.ArgumentParser()
-    p.add_argument('command', choices=['serialize', 'upsert', 'fetch-readmes', 'process-readmes'])
+    p.add_argument('command', choices=['serialize', 'upsert'])
     p.add_argument('path', help='the path to the input file', nargs='?', default='/dev/stdin')
     return p.parse_args(argv)
 
 
 subcommands = { 'serialize': serialize.main
               , 'upsert': upsert.main
-              , 'fetch-readmes': fetch_readmes.main
-              , 'process-readmes': process_readmes.main
                }
 
 
@@ -30,10 +28,9 @@ def main(argv=sys.argv):
 
     Usage::
 
-      sync-npm {serialize,upsert,fetch-readmes,process-readmes} {<filepath>}
+      sync-npm {serialize,upsert} {<filepath>}
 
-    ``<filepath>`` defaults to stdin. It's necessary for ``serialize`` and
-    ``upsert``, and silently ignored for ``{fetch,process}-readmes``.
+    ``<filepath>`` defaults to stdin.
 
     .. note:: Sphinx is expanding ``sys.argv`` in the parameter list. Sorry. :-/
 
@@ -42,19 +39,4 @@ def main(argv=sys.argv):
     args = parse_args(argv[1:])
     db = wireup.db(env)
 
-    try:
-        sys.stdout = sys.stderr  # work around aspen.log_dammit limitation; sigh
-        tell_sentry = wireup.make_sentry_teller(env)
-    finally:
-        sys.stdout = sys.__stdout__
-
-    def sentrified(func):
-        def _(*a, **kw):
-            try:
-                func(*a, **kw)
-            except:
-                e = sys.exc_info()[0]
-                tell_sentry(e, {})
-        return _
-
-    subcommands[args.command](env, args, db, sentrified)
+    subcommands[args.command](env, args, db)

gratipay/sync_npm/serialize.py

@@ -7,7 +7,7 @@
 import sys
 import time
 
-from . import log
+from . import log, sentry
 
 
 def import_ijson(env):
@@ -97,10 +97,11 @@ def log_stats():
     log_stats()
 
 
-def main(env, args, db, sentrified):
+def main(env, args, db):
     """Consume raw JSON from the npm registry via ``args.path``, and spit out
     CSV for Postgres to stdout. Uses ``ijson``, requiring the ``yajl_cffi``
     backend if ``env.require_yajl`` is ``True``.
 
     """
-    sentrified(serialize)(env, args, db)
+    with sentry(env):
+        serialize(env, args, db)

gratipay/sync_npm/upsert.py

@@ -5,6 +5,8 @@
 
 import uuid
 
+from . import sentry
+
 
 # Coordinate with Postgres on how to say "NULL".
 # ==============================================
@@ -45,10 +47,11 @@ def upsert(env, args, db):
         """)
 
 
-def main(env, args, db, sentrified):
+def main(env, args, db):
     """Take a CSV file from stdin and load it into Postgres using an `ingenious algorithm`_.
 
     .. _ingenious algorithm:  http://tapoueh.org/blog/2013/03/15-batch-update.html
 
     """
-    sentrified(upsert)(env, args, db)
+    with sentry(env):
+        upsert(env, args, db)

gratipay/testing/__init__.py

@@ -4,12 +4,8 @@
 
 from decimal import Decimal
 
-import pytest
-from aspen import log_dammit
-
 from ..models.participant import Participant
 from ..models.team import Team
-from ..utils import markdown
 
 D = Decimal                     #:
 P = Participant.from_username   #:
@@ -50,17 +46,3 @@ def debug_http():
     requests_log = logging.getLogger("requests.packages.urllib3")
     requests_log.setLevel(logging.DEBUG)
     requests_log.propagate = True
-
-
-# Provide a decorator to skip tests when marky-markdown is missing.
-
-try:
-    markdown.render_like_npm('test')
-except OSError as exc:
-    MISSING_MARKY_MARKDOWN = True
-    log_dammit('Will skip marky-markdown-related tests because:', exc.args[0])
-else:
-    MISSING_MARKY_MARKDOWN = False
-
-def skipif_missing_marky_markdown(func):
-    return pytest.mark.skipif(MISSING_MARKY_MARKDOWN, reason="missing marky-markdown")(func)

gratipay/utils/markdown.py

@@ -1,8 +1,6 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from subprocess import Popen, PIPE
 
-import json
 import misaka as m  # http://misaka.61924.nl/
 from markupsafe import Markup
 
@@ -19,28 +17,3 @@ def render(markdown):
         extensions=m.EXT_AUTOLINK | m.EXT_STRIKETHROUGH | m.EXT_NO_INTRA_EMPHASIS,
         render_flags=m.HTML_SKIP_HTML | m.HTML_TOC | m.HTML_SMARTYPANTS | m.HTML_SAFELINK
     ))
-
-
-def render_like_npm(markdown, package=None):
-    """Process markdown the same way npm does.
-
-    Package should be a dict representing the package. If it includes `name`
-    and `description` then the first h1 and paragraph will have a
-    'package-{name,description}-redundant' class added to them if they're
-    similar enough. If it includes `repository.url` then links will be changed
-    somehow. For details consult the docs and code:
-
-    https://github.com/npm/marky-markdown
-
-    """
-    if type(markdown) is unicode:
-        markdown = markdown.encode('utf8')
-    cmd = ("bin/our-marky-markdown.js", "/dev/stdin")
-    if package:
-        cmd += (json.dumps(package),)
-    marky = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-    out, err = marky.communicate(markdown)
-    if marky.wait() > 0:
-        raise OSError(err)
-    return out
-

gratipay/wireup.py

@@ -111,11 +111,10 @@ def username_restrictions(website):
     gratipay.RESTRICTED_USERNAMES = os.listdir(website.www_root)
 
 
-def make_sentry_teller(env):
+def make_sentry_teller(env, _noop=None):
     if not env.sentry_dsn:
         aspen.log_dammit("Won't log to Sentry (SENTRY_DSN is empty).")
-        def noop(*a, **kw):
-            pass
+        noop = _noop or (lambda *a, **kw: None)
         Participant._tell_sentry = noop
         return noop
 

sql/branch.sql

@@ -0,0 +1,6 @@
+BEGIN;
+    ALTER TABLE packages DROP COLUMN readme;
+    ALTER TABLE packages DROP COLUMN readme_raw;
+    ALTER TABLE packages DROP COLUMN readme_type;
+    ALTER TABLE packages DROP COLUMN readme_needs_to_be_processed;
+END;