Make some improvements

- more test coverage
- avoid thrashing so hard
- update deps

gratipay/cli/sync_npm.py

@@ -3,14 +3,17 @@
 """
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import time
+
 from couchdb import Database
 
 from gratipay import wireup
 from gratipay.utils import sentry
 
 
 def production_change_stream(seq):
-    """Given a sequence number in the npm registry change stream, start streaming!
+    """Given a sequence number in the npm registry change stream, start
+    streaming from there!
     """
     npm = Database('https://skimdb.npmjs.com/registry')
     return npm.changes(feed='continuous', include_docs=True, since=seq)
@@ -74,3 +77,4 @@ def main():
     while 1:
         with sentry.teller(env):
             consume_change_stream(db, production_change_stream)
+        time.sleep(60)  # avoid a busy loop if thrashing

gratipay/testing/harness.py

@@ -134,6 +134,7 @@ def clear_tables(self):
             except (IntegrityError, InternalError):
                 tablenames.insert(0, tablename)
         self.db.run("ALTER SEQUENCE participants_id_seq RESTART WITH 1")
+        self.db.run("INSERT INTO worker_coordination DEFAULT VALUES")
 
 
     def make_elsewhere(self, platform, user_id, user_name, **kw):

requirements.txt

@@ -68,6 +68,6 @@
   ./vendor/ipaddress-1.0.16.tar.gz
 ./vendor/cryptography-1.5.3.tar.gz
 
-./vendor/ijson-2.3.tar.gz
+./vendor/CouchDB-1.1.tar.gz
 
 -e .

tests/py/test_sync_npm.py

@@ -58,3 +58,23 @@ def test_consumes_change_stream(self):
         assert package.name == 'foo'
         assert package.description == 'Foo!'
         assert package.emails == []
+
+
+    def test_picks_up_with_last_seq(self):
+        docs = [ {'name': 'foo', 'description': 'Foo.'}
+               , {'name': 'foo', 'description': 'See alice?', 'maintainers': [{'email': 'alice'}]}
+               , {'name': 'foo', 'description': "No, I don't see alice!"}
+                ]
+        self.db.run('update worker_coordination set npm_last_seq=2')
+        sync_npm.consume_change_stream(self.change_stream(docs), self.db)
+
+        package = self.db.one('select * from packages')
+        assert package.description == "No, I don't see alice!"
+        assert package.emails == []
+
+
+    def test_sets_last_seq(self):
+        docs = [{'name': 'foo', 'description': 'Foo.'}] * 13
+        assert self.db.one('select npm_last_seq from worker_coordination') == -1
+        sync_npm.consume_change_stream(self.change_stream(docs), self.db)
+        assert self.db.one('select npm_last_seq from worker_coordination') == 12