From 6b1217522e9d00fc2bdc1b44a09e7e71232dac23 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 23 May 2017 16:54:43 +0200 Subject: [PATCH 01/10] Add pre commit hooks (copied from Kolibri) --- .pre-commit-config.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..26df105 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +- repo: git://github.com/pre-commit/pre-commit-hooks + sha: a11d9314b22d8f8c7556443875b731ef05965464 + hooks: + - id: trailing-whitespace + - id: flake8 + - id: check-yaml + - id: check-added-large-files + - id: debug-statements + - id: end-of-file-fixer +- repo: git://github.com/FalconSocial/pre-commit-python-sorter + sha: 5991d2aea26858d3c3538e0b4e09255b6b99413e + hooks: + - id: python-import-sorter + args: + - --silent-overwrite +- repo: git://github.com/pre-commit/mirrors-eslint + sha: v3.14.0 + hooks: + - id: eslint + additional_dependencies: [ 'eslint', 'eslint-plugin-html', 'eslint-config-airbnb', 'eslint-plugin-import', 'eslint-plugin-jsx-a11y'] From aa6eb43e20c6f938bcbdb387f356e9799466dd8b Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 23 May 2017 17:01:45 +0200 Subject: [PATCH 02/10] Add tox support and gather coverage --- requirements.txt | 1 + tox.ini | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tox.ini diff --git a/requirements.txt b/requirements.txt index fee0060..37aaf0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ py==1.4.33 pyparsing==2.2.0 pytest==3.0.7 pytest-mock==1.6.0 +pytest-cov six==1.10.0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..72c2c89 --- /dev/null +++ b/tox.ini @@ -0,0 +1,16 @@ +[tox] +envlist = py{27,34,35,pypy} + +[testenv] +usedevelop = True +setenv = + PYTHONPATH = {toxinidir} +basepython = + py27: python2.7 + py34: python3.4 + py35: python3.5 + pypypy: pypy +deps = + -r{toxinidir}/requirements.txt +commands = + py.test --cov=src/barbequeue --color=no {posargs} From dcc9b8884dbd43e0ac128ff1eb63d2faa1e37d9f Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 23 May 2017 17:15:45 +0200 Subject: [PATCH 03/10] Add Travis support --- .travis.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d4785d5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +# Config file for automatic testing at travis-ci.org + +language: python + +sudo: false + +matrix: + include: + - python: "2.7" + env: + - TOX_ENV=py27 + - python: "3.4" + env: + - TOX_ENV=py34 + - python: "3.5" + env: + - TOX_ENV=py35 + - python: "pypy" + env: + - TOX_ENV=pypypy + +before_install: + - pip install -U pip + - pip install tox + +# command to run tests, e.g. python setup.py test +script: + - tox -e $TOX_ENV + +after_success: + - codecov + +notifications: + slack: + secure: RH3xqTkkoA2TATYo7onoLlAlw5t7Bek3HqN/e48e1mj3nazTUs05k12e9Cj7I/y7rBu4b1g0Fl4nJ/DOJRD/81o2ML8OAqu4Ngfg149EQysvgHki20CY1LMph0aS5HSACL1yvp8VvDmmpF0F8YGh99ZMeV3qdp0t3AVWSQVIQgVOEjUSPklsf25ikCOqzTRCz90Dp5aIlDUuXtubTATaKQiLfqW8rc/S7Q/JLepmZau5ANFz8cHXn750y6EvEREIE/0gBwH6OWOa1qWCavJPc4z9953zX7rhI85f0eaHVYQ0ojeXJjQG4MOu5kP13tkaLppe1On3fBZLPpeF791EJwgcmEFUd8hDsomfHcxXhHf7+LPwbGjelGr8iY/2ZIWuILY17FmnWyruWyusrHpA1YKBFwjqgo7E06uzwVj9npxbc+WEHuBxTDdaeFYjYRLORvOeJ9y3n+rNs2c9gCvwuq3MzhMWT+KsrrngAibns1Fz/I1YZAY3voipmabeXVLqbLvb8f8xBmwwK14ba0tGMHfnnCfivlruS49qLwpMjASEPY3H8lKyROX94aaWj+B1Fld2kX0L5GN3xkVoT1aJOgOxCF9c7w9Kf1aTcf/T4bfEzjPNf3ZH3rwy/b3YABbQAEdV8SLdrD02+CIYNPzv3/U1y3rmTSg0wLT2elumhv8= From 2b2c8cfdecb4d73e78d7cbe2cb9c29fafbbf2ea9 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 26 May 2017 16:21:59 +0200 Subject: [PATCH 04/10] Add enum34 dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 37aaf0d..e0cbfb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pytest==3.0.7 pytest-mock==1.6.0 pytest-cov six==1.10.0 +enum34>=1.1,<2 From af17a116a5473baf32051e6561cd81579139bb75 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 26 May 2017 18:24:17 +0200 Subject: [PATCH 05/10] Translate README to rst (using pandoc) --- README.md | 42 ------------------------------------------ README.rst | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 42 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index 9a965b6..0000000 --- a/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# barbequeue -A self-contained pure python messaging and task queue system, with optional non-Python dependencies - -# Installation - -## Requirements - - -The base installation requires either Python 2.7, or 3.4+. - -# Goals - - -1. Work on Windows 7 and above. -2. Work on Python 2.7 and 3. -3. Should the least amount of dependencies for the minimal use case. To use other backends and services (Django, Google PubSub etc.) will require more dependencies. - - -## Problems with current pure python task queues - -1. Does not work with Windows out of the box. -2. Does not support progress tracking. - -# Design - -Barbequeue (or bbq for short) will be built as a layer on top of a messaging system. By allowing the messaging system to have multiple backends, from a -simple in-process and in-memory function, to a database backend, to a full blown global messaging service like Amazon SQS, bbq can scale up or change its features -based on the application's requirements. - -Out of the box, bbq.messaging supports the following backends: - - -1. asyncio (lowest resources used, available on 3.4+) -1. socketserver (recommended for Windows machines, allows for multi-machine task delegation) - -You can install additional bbq plugins to support the following backends: - -1. Google Pub/Sub (TODO) -1. Amazon SQS (TODO) - - -for bbq.queue, there are the following task \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..3826fd8 --- /dev/null +++ b/README.rst @@ -0,0 +1,52 @@ +barbequeue +========== + +A self-contained pure python messaging and task queue system, with +optional non-Python dependencies + +Installation +============ + +Requirements +------------ + +The base installation requires either Python 2.7, or 3.4+. + +Goals +===== + +1. Work on Windows 7 and above. +2. Work on Python 2.7 and 3. +3. Should the least amount of dependencies for the minimal use case. To + use other backends and services (Django, Google PubSub etc.) will + require more dependencies. + +Problems with current pure python task queues +--------------------------------------------- + +1. Does not work with Windows out of the box. +2. Does not support progress tracking. + +Design +====== + +Barbequeue (or bbq for short) will be built as a layer on top of a +messaging system. By allowing the messaging system to have multiple +backends, from a simple in-process and in-memory function, to a database +backend, to a full blown global messaging service like Amazon SQS, bbq +can scale up or change its features based on the application's +requirements. + +Out of the box, bbq.messaging supports the following backends: + +1. asyncio (lowest resources used, available on 3.4+) +2. socketserver (recommended for Windows machines, allows for + multi-machine task delegation) + +You can install additional bbq plugins to support the following +backends: + +1. Google Pub/Sub (TODO) +2. Amazon SQS (TODO) + +for bbq.queue, there are the following task From a21cb781a9aa7f64add63a0cf43319222df38164 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 26 May 2017 18:25:02 +0200 Subject: [PATCH 06/10] Use README.rst for setup.py --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b8dd8e..2b803ec 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,8 @@ def read_file(fname): """ Read file and decode in py2k """ + # This is necessary because we cannot use unicode_literals in + # setup.py if IS_PYTHON_2: return open(fname).read().decode("utf-8") return open(fname).read() @@ -28,7 +30,7 @@ def read_file(fname): dist_name = 'barbequeue' -readme = "Empty for now." # read_file('README.rst') +readme = read_file('README.rst') # Default description of the distributed package description = ("""A queueing library with support for Windows and Unix.""") From 9637b3bb06a470e2044ba5790503aa45b1e76ff3 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 26 May 2017 18:27:01 +0200 Subject: [PATCH 07/10] Additional requirements missing in setup.py --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b803ec..c48f970 100644 --- a/setup.py +++ b/setup.py @@ -51,11 +51,16 @@ def enable_log_to_stdout(logname): log.addHandler(ch) +INSTALL_REQUIRES = [ + 'six', + 'enum34>=1.1,<2', +] + PYTHON_2_BACKPORTS = [ "futures>=3.1.1", ] -INSTALL_REQUIRES = PYTHON_2_BACKPORTS if IS_PYTHON_2 else [] +INSTALL_REQUIRES += PYTHON_2_BACKPORTS if IS_PYTHON_2 else [] setup( name=dist_name, From 866b9e9cf6f9b4293b159da5d47033caab234c17 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Mon, 29 May 2017 00:58:41 +0200 Subject: [PATCH 08/10] Python 2+3 import of queue --- src/barbequeue/common/compat.py | 8 ++++++++ src/barbequeue/messaging/backends/inmem.py | 6 +++--- src/barbequeue/scheduler/classes.py | 6 +++--- src/barbequeue/worker/backends/inmem.py | 2 +- tests/common/test_classes.py | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 src/barbequeue/common/compat.py diff --git a/src/barbequeue/common/compat.py b/src/barbequeue/common/compat.py new file mode 100644 index 0000000..b637820 --- /dev/null +++ b/src/barbequeue/common/compat.py @@ -0,0 +1,8 @@ +import sys + +# Because "Queue" was renamed to "queue" in Py3. +is_py2 = sys.version[0] == '2' +if is_py2: + import Queue as queue # noqa @UnusedImport +else: + import queue as queue # noqa @UnusedImport @Reimport diff --git a/src/barbequeue/messaging/backends/inmem.py b/src/barbequeue/messaging/backends/inmem.py index 28fd85b..a3ff8e4 100644 --- a/src/barbequeue/messaging/backends/inmem.py +++ b/src/barbequeue/messaging/backends/inmem.py @@ -2,11 +2,11 @@ import time from abc import ABCMeta, abstractmethod from collections import defaultdict -from queue import Queue, Empty +from ...common.compat import queue logger = logging.getLogger(__name__) -INMEM_SUPER_MAILBOX = defaultdict(lambda: Queue()) +INMEM_SUPER_MAILBOX = defaultdict(lambda: queue.Queue()) class BaseBackend(object): @@ -59,7 +59,7 @@ def wait(self, mailbox, timeout=None): return True elif timeout <= 0: # we've gone past our alloted timeout, so raise an error - raise Empty + raise queue.Empty else: time.sleep(timeout_increment) timeout -= timeout_increment diff --git a/src/barbequeue/scheduler/classes.py b/src/barbequeue/scheduler/classes.py index dcab42d..afa54af 100644 --- a/src/barbequeue/scheduler/classes.py +++ b/src/barbequeue/scheduler/classes.py @@ -1,5 +1,5 @@ import time -from queue import Full, Empty +from barbequeue.common.compat import queue from threading import Event from barbequeue.common.classes import BaseCloseableThread @@ -55,7 +55,7 @@ def main_loop(self, timeout): def handle_worker_messages(self, timeout): try: msg = self.messaging_backend.pop(self.incoming_message_mailbox, timeout=timeout) - except Empty: + except queue.Empty: self.logger.debug("No new messages from workers.") return @@ -78,7 +78,7 @@ def schedule_next_job(self, timeout): try: self.worker_queue.put(next_job, timeout=timeout) - except Full: + except queue.Full: self.logger.debug("Worker queue full; skipping scheduling of job {} for now.".format(next_job.job_id)) return diff --git a/src/barbequeue/worker/backends/inmem.py b/src/barbequeue/worker/backends/inmem.py index a788ed7..e049ec1 100644 --- a/src/barbequeue/worker/backends/inmem.py +++ b/src/barbequeue/worker/backends/inmem.py @@ -1,10 +1,10 @@ import logging -import queue import traceback from threading import Event, Thread from barbequeue import humanhash from barbequeue.common.classes import BaseCloseableThread, Job +from barbequeue.common.compat import queue from barbequeue.messaging.backends.inmem import Backend as MsgBackend from barbequeue.messaging.classes import MessageType, UnknownMessageError, Message diff --git a/tests/common/test_classes.py b/tests/common/test_classes.py index 3ebbf4d..2246ed2 100644 --- a/tests/common/test_classes.py +++ b/tests/common/test_classes.py @@ -1,10 +1,10 @@ -import queue from threading import Event import pytest from barbequeue.client import Client, InMemClient from barbequeue.common.classes import Job +from barbequeue.common.compat import queue from barbequeue.storage.backends import inmem from barbequeue.worker.backends import inmem as worker_inmem From ce3a0891e457c21de242d3af7fe52a274693b65d Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Mon, 29 May 2017 00:58:50 +0200 Subject: [PATCH 09/10] Add missing __init__.py files --- src/barbequeue/scheduler/__init__.py | 0 src/barbequeue/storage/__init__.py | 0 src/barbequeue/worker/backends/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/barbequeue/scheduler/__init__.py create mode 100644 src/barbequeue/storage/__init__.py create mode 100644 src/barbequeue/worker/backends/__init__.py diff --git a/src/barbequeue/scheduler/__init__.py b/src/barbequeue/scheduler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/barbequeue/storage/__init__.py b/src/barbequeue/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/barbequeue/worker/backends/__init__.py b/src/barbequeue/worker/backends/__init__.py new file mode 100644 index 0000000..e69de29 From d448d61bc5169757603ef8dcb64ca9e9be4b3801 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Mon, 29 May 2017 21:12:54 +0200 Subject: [PATCH 10/10] Wrapping up non-finished Py2 compatibility for this PR --- src/barbequeue/common/classes.py | 5 ++++- src/barbequeue/common/compat.py | 11 +++++++++++ tests/worker/backends/test_inmem.py | 7 ++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/barbequeue/common/classes.py b/src/barbequeue/common/classes.py index 80a5cef..8f0481a 100644 --- a/src/barbequeue/common/classes.py +++ b/src/barbequeue/common/classes.py @@ -6,6 +6,7 @@ from collections import namedtuple from barbequeue import humanhash +from barbequeue.common import compat logger = logging.getLogger(__name__) @@ -29,6 +30,8 @@ def __init__(self, func_string, *args, **kwargs): self.kwargs = kwargs def get_lambda_to_execute(self): + if not isinstance(self.func, str): + return self.func fqn = self.func modulename, funcname = fqn.rsplit('.', 1) mod = importlib.import_module(modulename) @@ -66,7 +69,7 @@ class BaseCloseableThread(threading.Thread): DEFAULT_TIMEOUT_SECONDS = 0.2 def __init__(self, shutdown_event, thread_name, *args, **kwargs): - assert isinstance(shutdown_event, threading.Event) + assert isinstance(shutdown_event, compat.Event) self.shutdown_event = shutdown_event self.thread_name = thread_name diff --git a/src/barbequeue/common/compat.py b/src/barbequeue/common/compat.py index b637820..1bac80f 100644 --- a/src/barbequeue/common/compat.py +++ b/src/barbequeue/common/compat.py @@ -1,4 +1,10 @@ +""" +Everything in this module should have a Python 3 interface, but should be +compatible with Python 2 +""" + import sys +import threading # Because "Queue" was renamed to "queue" in Py3. is_py2 = sys.version[0] == '2' @@ -6,3 +12,8 @@ import Queue as queue # noqa @UnusedImport else: import queue as queue # noqa @UnusedImport @Reimport + +if is_py2: + Event = threading._Event +else: + Event = threading.Event diff --git a/tests/worker/backends/test_inmem.py b/tests/worker/backends/test_inmem.py index 1ae36fe..e8f0af7 100644 --- a/tests/worker/backends/test_inmem.py +++ b/tests/worker/backends/test_inmem.py @@ -1,3 +1,4 @@ +import logging import pytest from barbequeue.common.classes import Job @@ -12,6 +13,7 @@ def mailbox(): @pytest.fixture def worker(mailbox): + logging.info("test_inmem.worker() working") b = inmem.Backend(incoming_message_mailbox=mailbox, outgoing_message_mailbox=mailbox) yield b b.shutdown() @@ -25,7 +27,8 @@ def startmsg(job): @pytest.fixture def simplejob(): - job = Job("builtins.id", 'test', job_id='simplejob') + logging.info("simplejob spawning a Job") + job = Job(id, 'test', job_id='simplejob') return job @@ -49,7 +52,9 @@ class TestWorker: def test_successful_job_adds_to_report_queue(self, worker, simplejob, mocker): mocker.spy(worker.reportqueue, 'put') + logging.info("TestWorker starting simplejob...") worker.start_job(simplejob) + logging.info("TestWorker joining jobqueue...") worker.jobqueue.join() assert worker.reportqueue.put.call_count == 1