[WIP] More impro

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: a11d9314b22d8f8c7556443875b731ef05965464
+    hooks:
+    -   id: trailing-whitespace
+    -   id: flake8
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: debug-statements
+    -   id: end-of-file-fixer
+-   repo: git://github.com/FalconSocial/pre-commit-python-sorter
+    sha: 5991d2aea26858d3c3538e0b4e09255b6b99413e
+    hooks:
+    -   id: python-import-sorter
+        args:
+        - --silent-overwrite
+-   repo: git://github.com/pre-commit/mirrors-eslint
+    sha: v3.14.0
+    hooks:
+    -   id: eslint
+        additional_dependencies: [ 'eslint', 'eslint-plugin-html', 'eslint-config-airbnb', 'eslint-plugin-import', 'eslint-plugin-jsx-a11y']

.travis.yml

@@ -0,0 +1,35 @@
+# Config file for automatic testing at travis-ci.org
+
+language: python
+
+sudo: false
+
+matrix:
+  include:
+    - python: "2.7"
+      env:
+        - TOX_ENV=py27
+    - python: "3.4"
+      env:
+        - TOX_ENV=py34
+    - python: "3.5"
+      env:
+        - TOX_ENV=py35
+    - python: "pypy"
+      env:
+        - TOX_ENV=pypypy
+
+before_install:
+  - pip install -U pip
+  - pip install tox
+
+# command to run tests, e.g. python setup.py test
+script:
+  - tox -e $TOX_ENV
+
+after_success:
+  - codecov
+
+notifications:
+  slack:
+    secure: RH3xqTkkoA2TATYo7onoLlAlw5t7Bek3HqN/e48e1mj3nazTUs05k12e9Cj7I/y7rBu4b1g0Fl4nJ/DOJRD/81o2ML8OAqu4Ngfg149EQysvgHki20CY1LMph0aS5HSACL1yvp8VvDmmpF0F8YGh99ZMeV3qdp0t3AVWSQVIQgVOEjUSPklsf25ikCOqzTRCz90Dp5aIlDUuXtubTATaKQiLfqW8rc/S7Q/JLepmZau5ANFz8cHXn750y6EvEREIE/0gBwH6OWOa1qWCavJPc4z9953zX7rhI85f0eaHVYQ0ojeXJjQG4MOu5kP13tkaLppe1On3fBZLPpeF791EJwgcmEFUd8hDsomfHcxXhHf7+LPwbGjelGr8iY/2ZIWuILY17FmnWyruWyusrHpA1YKBFwjqgo7E06uzwVj9npxbc+WEHuBxTDdaeFYjYRLORvOeJ9y3n+rNs2c9gCvwuq3MzhMWT+KsrrngAibns1Fz/I1YZAY3voipmabeXVLqbLvb8f8xBmwwK14ba0tGMHfnnCfivlruS49qLwpMjASEPY3H8lKyROX94aaWj+B1Fld2kX0L5GN3xkVoT1aJOgOxCF9c7w9Kf1aTcf/T4bfEzjPNf3ZH3rwy/b3YABbQAEdV8SLdrD02+CIYNPzv3/U1y3rmTSg0wLT2elumhv8=

README.rst

@@ -0,0 +1,52 @@
+barbequeue
+==========
+
+A self-contained pure python messaging and task queue system, with
+optional non-Python dependencies
+
+Installation
+============
+
+Requirements
+------------
+
+The base installation requires either Python 2.7, or 3.4+.
+
+Goals
+=====
+
+1. Work on Windows 7 and above.
+2. Work on Python 2.7 and 3.
+3. Should the least amount of dependencies for the minimal use case. To
+   use other backends and services (Django, Google PubSub etc.) will
+   require more dependencies.
+
+Problems with current pure python task queues
+---------------------------------------------
+
+1. Does not work with Windows out of the box.
+2. Does not support progress tracking.
+
+Design
+======
+
+Barbequeue (or bbq for short) will be built as a layer on top of a
+messaging system. By allowing the messaging system to have multiple
+backends, from a simple in-process and in-memory function, to a database
+backend, to a full blown global messaging service like Amazon SQS, bbq
+can scale up or change its features based on the application's
+requirements.
+
+Out of the box, bbq.messaging supports the following backends:
+
+1. asyncio (lowest resources used, available on 3.4+)
+2. socketserver (recommended for Windows machines, allows for
+   multi-machine task delegation)
+
+You can install additional bbq plugins to support the following
+backends:
+
+1. Google Pub/Sub (TODO)
+2. Amazon SQS (TODO)
+
+for bbq.queue, there are the following task

requirements.txt

@@ -6,4 +6,6 @@ py==1.4.33
 pyparsing==2.2.0
 pytest==3.0.7
 pytest-mock==1.6.0
+pytest-cov
 six==1.10.0
+enum34>=1.1,<2

setup.py

@@ -21,14 +21,16 @@ def read_file(fname):
     """
     Read file and decode in py2k
     """
+    # This is necessary because we cannot use unicode_literals in
+    # setup.py
     if IS_PYTHON_2:
         return open(fname).read().decode("utf-8")
     return open(fname).read()
 
 
 dist_name = 'barbequeue'
 
-readme = "Empty for now."  # read_file('README.rst')
+readme = read_file('README.rst')
 
 # Default description of the distributed package
 description = ("""A queueing library with support for Windows and Unix.""")
@@ -49,11 +51,16 @@ def enable_log_to_stdout(logname):
     log.addHandler(ch)
 
 
+INSTALL_REQUIRES = [
+    'six',
+    'enum34>=1.1,<2',
+]
+
 PYTHON_2_BACKPORTS = [
     "futures>=3.1.1",
 ]
 
-INSTALL_REQUIRES = PYTHON_2_BACKPORTS if IS_PYTHON_2 else []
+INSTALL_REQUIRES += PYTHON_2_BACKPORTS if IS_PYTHON_2 else []
 
 setup(
     name=dist_name,

src/barbequeue/common/classes.py

@@ -6,6 +6,7 @@
 from collections import namedtuple
 
 from barbequeue import humanhash
+from barbequeue.common import compat
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +30,8 @@ def __init__(self, func_string, *args, **kwargs):
         self.kwargs = kwargs
 
     def get_lambda_to_execute(self):
+        if not isinstance(self.func, str):
+            return self.func
         fqn = self.func
         modulename, funcname = fqn.rsplit('.', 1)
         mod = importlib.import_module(modulename)
@@ -66,7 +69,7 @@ class BaseCloseableThread(threading.Thread):
     DEFAULT_TIMEOUT_SECONDS = 0.2
 
     def __init__(self, shutdown_event, thread_name, *args, **kwargs):
-        assert isinstance(shutdown_event, threading.Event)
+        assert isinstance(shutdown_event, compat.Event)
 
         self.shutdown_event = shutdown_event
         self.thread_name = thread_name

src/barbequeue/common/compat.py

@@ -0,0 +1,19 @@
+"""
+Everything in this module should have a Python 3 interface, but should be
+compatible with Python 2
+"""
+
+import sys
+import threading
+
+# Because "Queue" was renamed to "queue" in Py3.
+is_py2 = sys.version[0] == '2'
+if is_py2:
+    import Queue as queue  # noqa @UnusedImport
+else:
+    import queue as queue  # noqa @UnusedImport @Reimport
+
+if is_py2:
+    Event = threading._Event
+else:
+    Event = threading.Event

src/barbequeue/messaging/backends/inmem.py

@@ -2,11 +2,11 @@
 import time
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
-from queue import Queue, Empty
+from ...common.compat import queue
 
 logger = logging.getLogger(__name__)
 
-INMEM_SUPER_MAILBOX = defaultdict(lambda: Queue())
+INMEM_SUPER_MAILBOX = defaultdict(lambda: queue.Queue())
 
 
 class BaseBackend(object):
@@ -59,7 +59,7 @@ def wait(self, mailbox, timeout=None):
 
                 return True
             elif timeout <= 0:  # we've gone past our alloted timeout, so raise an error
-                raise Empty
+                raise queue.Empty
             else:
                 time.sleep(timeout_increment)
                 timeout -= timeout_increment

src/barbequeue/scheduler/classes.py

@@ -1,5 +1,5 @@
 import time
-from queue import Full, Empty
+from barbequeue.common.compat import queue
 from threading import Event
 
 from barbequeue.common.classes import BaseCloseableThread
@@ -55,7 +55,7 @@ def main_loop(self, timeout):
     def handle_worker_messages(self, timeout):
         try:
             msg = self.messaging_backend.pop(self.incoming_message_mailbox, timeout=timeout)
-        except Empty:
+        except queue.Empty:
             self.logger.debug("No new messages from workers.")
             return
 
@@ -78,7 +78,7 @@ def schedule_next_job(self, timeout):
 
         try:
             self.worker_queue.put(next_job, timeout=timeout)
-        except Full:
+        except queue.Full:
             self.logger.debug("Worker queue full; skipping scheduling of job {} for now.".format(next_job.job_id))
             return
 

src/barbequeue/worker/backends/inmem.py

@@ -1,10 +1,10 @@
 import logging
-import queue
 import traceback
 from threading import Event, Thread
 
 from barbequeue import humanhash
 from barbequeue.common.classes import BaseCloseableThread, Job
+from barbequeue.common.compat import queue
 from barbequeue.messaging.backends.inmem import Backend as MsgBackend
 from barbequeue.messaging.classes import MessageType, UnknownMessageError, Message
 

tests/common/test_classes.py

@@ -1,10 +1,10 @@
-import queue
 from threading import Event
 
 import pytest
 
 from barbequeue.client import Client, InMemClient
 from barbequeue.common.classes import Job
+from barbequeue.common.compat import queue
 from barbequeue.storage.backends import inmem
 from barbequeue.worker.backends import inmem as worker_inmem
 

tests/worker/backends/test_inmem.py

@@ -1,3 +1,4 @@
+import logging
 import pytest
 
 from barbequeue.common.classes import Job
@@ -12,6 +13,7 @@ def mailbox():
 
 @pytest.fixture
 def worker(mailbox):
+    logging.info("test_inmem.worker() working")
     b = inmem.Backend(incoming_message_mailbox=mailbox, outgoing_message_mailbox=mailbox)
     yield b
     b.shutdown()
@@ -25,7 +27,8 @@ def startmsg(job):
 
 @pytest.fixture
 def simplejob():
-    job = Job("builtins.id", 'test', job_id='simplejob')
+    logging.info("simplejob spawning a Job")
+    job = Job(id, 'test', job_id='simplejob')
     return job
 
 
@@ -49,7 +52,9 @@ class TestWorker:
     def test_successful_job_adds_to_report_queue(self, worker, simplejob, mocker):
         mocker.spy(worker.reportqueue, 'put')
 
+        logging.info("TestWorker starting simplejob...")
         worker.start_job(simplejob)
+        logging.info("TestWorker joining jobqueue...")
         worker.jobqueue.join()
 
         assert worker.reportqueue.put.call_count == 1

tox.ini

@@ -0,0 +1,16 @@
+[tox]
+envlist = py{27,34,35,pypy}
+
+[testenv]
+usedevelop = True
+setenv =
+    PYTHONPATH = {toxinidir}
+basepython =
+    py27: python2.7
+    py34: python3.4
+    py35: python3.5
+    pypypy: pypy
+deps =
+    -r{toxinidir}/requirements.txt
+commands =
+    py.test --cov=src/barbequeue --color=no {posargs}