From 4af1460278abb63c1356664d68d9e246481217d6 Mon Sep 17 00:00:00 2001 From: mike wakerly Date: Thu, 28 Jul 2022 18:41:05 +0100 Subject: [PATCH 1/2] core: replace celery with rq --- docs/source/releases/changelog.rst | 1 + poetry.lock | 258 ++++-------------- pykeg/celery.py | 18 -- pykeg/contrib/webhook/tasks.py | 6 +- pykeg/core/management/commands/run_workers.py | 26 +- pykeg/core/tasks.py | 13 +- pykeg/plugin/util.py | 11 +- pykeg/settings.py | 37 +-- pykeg/util/runner.py | 2 - .../templates/kegadmin/nav-items.html | 2 +- .../templates/kegadmin/system_status.html | 82 ------ pykeg/web/kegadmin/urls.py | 1 - pykeg/web/kegadmin/views.py | 47 +--- pykeg/web/urls.py | 1 + pyproject.toml | 3 +- 15 files changed, 98 insertions(+), 410 deletions(-) delete mode 100644 pykeg/celery.py delete mode 100644 pykeg/web/kegadmin/templates/kegadmin/system_status.html diff --git a/docs/source/releases/changelog.rst b/docs/source/releases/changelog.rst index 38919754d..3c1c0a0a1 100644 --- a/docs/source/releases/changelog.rst +++ b/docs/source/releases/changelog.rst @@ -53,6 +53,7 @@ Several features have been removed in order to lower code or documentation compl * Upgraded to Python 3 and Django 3. * Internal: Improved static file serving (:issue:`368`) * Internal: Developer tests now use ``pytest`` +* Internal: Now using ``rq`` for worker queue Version 1.2.3 (2015-01-12) -------------------------- diff --git a/poetry.lock b/poetry.lock index 2aea39408..e73724078 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,17 +14,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "amqp" -version = "5.1.1" -description = "Low-level AMQP client for Python (fork of amqplib)." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -vine = ">=5.0.0" - [[package]] name = "asgiref" version = "3.5.2" @@ -92,14 +81,6 @@ soupsieve = ">1.2" html5lib = ["html5lib"] lxml = ["lxml"] -[[package]] -name = "billiard" -version = "3.6.4.0" -description = "Python multiprocessing fork with improvements and bugfixes" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "black" version = "22.6.0" @@ -121,57 +102,6 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] -[[package]] -name = "celery" -version = "5.2.7" -description = "Distributed Task Queue." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -billiard = ">=3.6.4.0,<4.0" -click = ">=8.0.3,<9.0" -click-didyoumean = ">=0.0.3" -click-plugins = ">=1.1.1" -click-repl = ">=0.2.0" -kombu = ">=5.2.3,<6.0" -pytz = ">=2021.3" -vine = ">=5.0.0,<6.0" - -[package.extras] -arangodb = ["pyArango (>=1.3.2)"] -auth = ["cryptography"] -azureblockblob = ["azure-storage-blob (==12.9.0)"] -brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"] -cassandra = ["cassandra-driver (<3.21.0)"] -consul = ["python-consul2"] -cosmosdbsql = ["pydocumentdb (==2.3.2)"] -couchbase = ["couchbase (>=3.0.0)"] -couchdb = ["pycouchdb"] -django = ["Django (>=1.11)"] -dynamodb = ["boto3 (>=1.9.178)"] -elasticsearch = ["elasticsearch"] -eventlet = ["eventlet (>=0.32.0)"] -gevent = ["gevent (>=1.5.0)"] -librabbitmq = ["librabbitmq (>=1.5.0)"] -memcache = ["pylibmc"] -mongodb = ["pymongo[srv] (>=3.11.1)"] -msgpack = ["msgpack"] -pymemcache = ["python-memcached"] -pyro = ["pyro4"] -pytest = ["pytest-celery"] -redis = ["redis (>=3.4.1,!=4.0.0,!=4.0.1)"] -s3 = ["boto3 (>=1.9.125)"] -slmq = ["softlayer-messaging (>=1.0.3)"] -solar = ["ephem"] -sqlalchemy = ["sqlalchemy"] -sqs = ["kombu"] -tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"] -yaml = ["PyYAML (>=3.10)"] -zookeeper = ["kazoo (>=1.3.1)"] -zstd = ["zstandard"] - [[package]] name = "certifi" version = "2022.6.15" @@ -213,44 +143,6 @@ python-versions = ">=3.7" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "click-didyoumean" -version = "0.3.0" -description = "Enables git-like *did-you-mean* feature in click" -category = "main" -optional = false -python-versions = ">=3.6.2,<4.0.0" - -[package.dependencies] -click = ">=7" - -[[package]] -name = "click-plugins" -version = "1.1.1" -description = "An extension module for click to enable registering CLI commands via setuptools entry-points." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -click = ">=4.0" - -[package.extras] -dev = ["pytest (>=3.6)", "pytest-cov", "wheel", "coveralls"] - -[[package]] -name = "click-repl" -version = "0.2.0" -description = "REPL plugin for Click" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -click = "*" -prompt-toolkit = "*" -six = "*" - [[package]] name = "colorama" version = "0.4.5" @@ -414,6 +306,23 @@ python-versions = ">=3.7" confusable-homoglyphs = ">=3.0,<4.0" Django = ">=3.2" +[[package]] +name = "django-rq" +version = "2.5.1" +description = "An app that provides django integration for RQ (Redis Queue)" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +django = ">=2.0" +redis = ">=3" +rq = ">=1.2" + +[package.extras] +sentry = ["raven (>=6.1.0)"] +testing = ["mock (>=2.0.0)"] + [[package]] name = "djangorestframework" version = "3.13.1" @@ -604,34 +513,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "kombu" -version = "5.2.4" -description = "Messaging library for Python." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -amqp = ">=5.0.9,<6.0.0" -vine = "*" - -[package.extras] -azureservicebus = ["azure-servicebus (>=7.0.0)"] -azurestoragequeues = ["azure-storage-queue"] -consul = ["python-consul (>=0.6.0)"] -librabbitmq = ["librabbitmq (>=2.0.0)"] -mongodb = ["pymongo (>=3.3.0,<3.12.1)"] -msgpack = ["msgpack"] -pyro = ["pyro4"] -qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"] -redis = ["redis (>=3.4.1,!=4.0.0,!=4.0.1)"] -slmq = ["softlayer-messaging (>=1.0.3)"] -sqlalchemy = ["sqlalchemy"] -sqs = ["boto3 (>=1.9.12)", "pycurl (>=7.44.1,<7.45.0)", "urllib3 (>=1.26.7)"] -yaml = ["PyYAML (>=3.10)"] -zookeeper = ["kazoo (>=1.3.1)"] - [[package]] name = "livereload" version = "2.6.3" @@ -757,19 +638,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "prompt-toolkit" -version = "3.0.30" -description = "Library for building powerful interactive command lines in Python" -category = "main" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -wcwidth = "*" +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "protobuf" @@ -983,6 +853,18 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] +[[package]] +name = "rq" +version = "1.10.1" +description = "RQ is a simple, lightweight, library for creating background jobs, and processing them." +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +click = ">=5.0.0" +redis = ">=3.5.0" + [[package]] name = "six" version = "1.16.0" @@ -1093,8 +975,8 @@ optional = false python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-devhelp" @@ -1105,8 +987,8 @@ optional = false python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-htmlhelp" @@ -1117,8 +999,8 @@ optional = false python-versions = ">=3.6" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] -test = ["pytest", "html5lib"] +test = ["html5lib", "pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-jsmath" @@ -1129,7 +1011,7 @@ optional = false python-versions = ">=3.5" [package.extras] -test = ["pytest", "flake8", "mypy"] +test = ["mypy", "flake8", "pytest"] [[package]] name = "sphinxcontrib-qthelp" @@ -1140,8 +1022,8 @@ optional = false python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-serializinghtml" @@ -1206,22 +1088,6 @@ six = ">=1.5" wrapt = "*" yarl = "*" -[[package]] -name = "vine" -version = "5.0.0" -description = "Promises, promises, promises." -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "wcwidth" -version = "0.2.5" -description = "Measures the displayed width of unicode strings in a terminal" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "whitenoise" version = "6.2.0" @@ -1281,7 +1147,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "^3.10.4" -content-hash = "2b1ce2a8c989eacc5bc6503f908750b1445ed406bd59a32ecaefec4154b1c400" +content-hash = "73b401fbeea31475719b4fd2c2a8fa8e210d7350d517bf6b386744e2f363af28" [metadata.files] addict = [ @@ -1292,10 +1158,6 @@ alabaster = [ {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] -amqp = [ - {file = "amqp-5.1.1-py3-none-any.whl", hash = "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359"}, - {file = "amqp-5.1.1.tar.gz", hash = "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2"}, -] asgiref = [ {file = "asgiref-3.5.2-py3-none-any.whl", hash = "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4"}, {file = "asgiref-3.5.2.tar.gz", hash = "sha256:4a29362a6acebe09bf1d6640db38c1dc3d9217c68e6f9f6204d72667fc19a424"}, @@ -1319,10 +1181,6 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, ] -billiard = [ - {file = "billiard-3.6.4.0-py3-none-any.whl", hash = "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b"}, - {file = "billiard-3.6.4.0.tar.gz", hash = "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547"}, -] black = [ {file = "black-22.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f586c26118bc6e714ec58c09df0157fe2d9ee195c764f630eb0d8e7ccce72e69"}, {file = "black-22.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b270a168d69edb8b7ed32c193ef10fd27844e5c60852039599f9184460ce0807"}, @@ -1348,10 +1206,6 @@ black = [ {file = "black-22.6.0-py3-none-any.whl", hash = "sha256:ac609cf8ef5e7115ddd07d85d988d074ed00e10fbc3445aee393e70164a2219c"}, {file = "black-22.6.0.tar.gz", hash = "sha256:6c6d39e28aed379aec40da1c65434c77d75e65bb59a1e1c283de545fb4e7c6c9"}, ] -celery = [ - {file = "celery-5.2.7-py3-none-any.whl", hash = "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14"}, - {file = "celery-5.2.7.tar.gz", hash = "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d"}, -] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -1430,18 +1284,6 @@ click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] -click-didyoumean = [ - {file = "click-didyoumean-0.3.0.tar.gz", hash = "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"}, - {file = "click_didyoumean-0.3.0-py3-none-any.whl", hash = "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667"}, -] -click-plugins = [ - {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, - {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, -] -click-repl = [ - {file = "click-repl-0.2.0.tar.gz", hash = "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8"}, - {file = "click_repl-0.2.0-py3-none-any.whl", hash = "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b"}, -] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, @@ -1498,6 +1340,10 @@ django-registration = [ {file = "django-registration-3.3.tar.gz", hash = "sha256:884a4cc9ec87b9f1c0ceb6b6c4b7ba491c1877997a3cd29cc923697dac785eb8"}, {file = "django_registration-3.3-py3-none-any.whl", hash = "sha256:dfa176f594fb465c93495caa55686be723a15829769511383e25172d2efbd0e6"}, ] +django-rq = [ + {file = "django-rq-2.5.1.tar.gz", hash = "sha256:f08486602664d73a6e335872c868d79663e380247e6307496d01b8fa770fefd8"}, + {file = "django_rq-2.5.1-py2.py3-none-any.whl", hash = "sha256:7be1e10e7091555f9f36edf100b0dbb205ea2b98683d74443d2bdf3c6649a03f"}, +] djangorestframework = [ {file = "djangorestframework-3.13.1-py3-none-any.whl", hash = "sha256:24c4bf58ed7e85d1fe4ba250ab2da926d263cd57d64b03e8dcef0ac683f8b1aa"}, {file = "djangorestframework-3.13.1.tar.gz", hash = "sha256:0c33407ce23acc68eca2a6e46424b008c9c02eceb8cf18581921d0092bc1f2ee"}, @@ -1642,10 +1488,6 @@ jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -kombu = [ - {file = "kombu-5.2.4-py3-none-any.whl", hash = "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4"}, - {file = "kombu-5.2.4.tar.gz", hash = "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610"}, -] livereload = [ {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, ] @@ -1852,10 +1694,6 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -prompt-toolkit = [ - {file = "prompt_toolkit-3.0.30-py3-none-any.whl", hash = "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"}, - {file = "prompt_toolkit-3.0.30.tar.gz", hash = "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0"}, -] protobuf = [ {file = "protobuf-3.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9d0f3aca8ca51c8b5e204ab92bd8afdb2a8e3df46bd0ce0bd39065d79aabcaa4"}, {file = "protobuf-3.20.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:001c2160c03b6349c04de39cf1a58e342750da3632f6978a1634a3dcca1ec10e"}, @@ -1993,6 +1831,10 @@ requests-oauthlib = [ {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, ] +rq = [ + {file = "rq-1.10.1-py2.py3-none-any.whl", hash = "sha256:92f4cf38b2364c1697b541e77c0fe62b7e5242fa864324f262be126ee2a07e3a"}, + {file = "rq-1.10.1.tar.gz", hash = "sha256:62d06b44c3acfa5d1933c5a4ec3fbc2484144a8af60e318d0b8447c5236271e2"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2074,14 +1916,6 @@ vcrpy = [ {file = "vcrpy-4.2.0-py2.py3-none-any.whl", hash = "sha256:7ec280c8d5385652f1117fe32a200e6676614007d9f946af9f07df1e5f92254c"}, {file = "vcrpy-4.2.0.tar.gz", hash = "sha256:94520b86fb765925adc8c77ff934e89a5e156c28e74a314320217ef1b454afe0"}, ] -vine = [ - {file = "vine-5.0.0-py2.py3-none-any.whl", hash = "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30"}, - {file = "vine-5.0.0.tar.gz", hash = "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e"}, -] -wcwidth = [ - {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, - {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, -] whitenoise = [ {file = "whitenoise-6.2.0-py3-none-any.whl", hash = "sha256:8e9c600a5c18bd17655ef668ad55b5edf6c24ce9bdca5bf607649ca4b1e8e2c2"}, {file = "whitenoise-6.2.0.tar.gz", hash = "sha256:8fa943c6d4cd9e27673b70c21a07b0aa120873901e099cd46cab40f7cc96d567"}, diff --git a/pykeg/celery.py b/pykeg/celery.py deleted file mode 100644 index c28d83c0b..000000000 --- a/pykeg/celery.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -from celery import Celery -from django.conf import settings - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pykeg.settings") - -app = Celery("pykeg") -app.config_from_object("django.conf:settings") -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) - - -def plugin_tasks(): - return [".".join(x.split(".")[:-2]) for x in settings.KEGBOT_PLUGINS] - - -app.autodiscover_tasks(plugin_tasks()) diff --git a/pykeg/contrib/webhook/tasks.py b/pykeg/contrib/webhook/tasks.py index a286eba20..f580d2f4b 100644 --- a/pykeg/contrib/webhook/tasks.py +++ b/pykeg/contrib/webhook/tasks.py @@ -1,8 +1,8 @@ -"""Celery tasks for Webhook plugin.""" +"""RQ tasks for Webhook plugin.""" import requests +from django_rq import job -from pykeg.celery import app from pykeg.core.util import get_version from pykeg.plugin import util from pykeg.util import kbjson @@ -10,7 +10,7 @@ logger = util.get_logger(__name__) -@app.task(name="webhook_post", expires=180) +@job def webhook_post(url, event_dict): """Posts an event to the supplied URL. diff --git a/pykeg/core/management/commands/run_workers.py b/pykeg/core/management/commands/run_workers.py index 66641d5fc..221b9e196 100644 --- a/pykeg/core/management/commands/run_workers.py +++ b/pykeg/core/management/commands/run_workers.py @@ -1,4 +1,7 @@ import os +import sys + +from django.conf import settings from pykeg.core.management.commands.common import RunnerCommand @@ -8,24 +11,13 @@ class Command(RunnerCommand): pidfile_name = "kegbot_run_workers.pid" def get_commands(self, options): - default_log = stats_log = beat_log = "" + default_log = "" logs_dir = options.get("logs_dir") - if logs_dir: - default_log = ' --logfile="{}"'.format(os.path.join(logs_dir, "celery_default.log")) - stats_log = ' --logfile="{}"'.format(os.path.join(logs_dir, "celery_stats.log")) - ret = [] - - base_cmd = "celery -A pykeg worker -l info " - - ret.append( - ("celery_default", base_cmd + '-Q default --hostname="default@%h"' + default_log) - ) - ret.append( - ( - "celery_stats", - base_cmd + '-Q stats --concurrency=1 --hostname="stats@%h"' + stats_log, - ) - ) + default_log = ' --logfile="{}"'.format(os.path.join(logs_dir, "workers.log")) + queue_names = " ".join(settings.RQ_QUEUES.keys()) + ret = [ + ("rq", f"{sys.argv[0]} rqworker {queue_names}{default_log} -v 3"), + ] return ret diff --git a/pykeg/core/tasks.py b/pykeg/core/tasks.py index 4e0c75461..8d71c4fd7 100644 --- a/pykeg/core/tasks.py +++ b/pykeg/core/tasks.py @@ -1,16 +1,17 @@ """Tasks for the Kegbot core.""" -from celery.utils.log import get_task_logger +import logging + from django.db import transaction +from django_rq import job from pykeg import notification from pykeg.backup import backup -from pykeg.celery import app from pykeg.plugin import util as plugin_util from . import stats -logger = get_task_logger(__name__) +logger = logging.getLogger(__name__) def schedule_tasks(events): @@ -23,7 +24,7 @@ def schedule_tasks(events): notification.handle_new_system_events(events) -@app.task(name="build_stats", queue="stats", expires=60 * 60) +@job("stats") def build_stats(drink_id, rebuild_following): logger.info("build_stats drink_id={} rebuild_following={}".format(drink_id, rebuild_following)) with transaction.atomic(): @@ -33,8 +34,8 @@ def build_stats(drink_id, rebuild_following): stats.build_for_id(drink_id) -@app.task(name="build_backup", bind=True) -def build_backup(self): +@job +def build_backup(): logger.info("build_backup") with transaction.atomic(): backup.backup() diff --git a/pykeg/plugin/util.py b/pykeg/plugin/util.py index 95c45ef1f..fa8cb79f2 100644 --- a/pykeg/plugin/util.py +++ b/pykeg/plugin/util.py @@ -1,4 +1,5 @@ import datetime +import logging from importlib import import_module from django.conf import settings @@ -73,12 +74,4 @@ def is_stale(time, now=None): def get_logger(name): - try: - from celery.utils.log import get_task_logger - - logger = get_task_logger(name) - except ImportError: - import logging - - logger = logging.getLogger(name) - return logger + return logging.getLogger(name) diff --git a/pykeg/settings.py b/pykeg/settings.py index 5d91266cb..c92b95e20 100644 --- a/pykeg/settings.py +++ b/pykeg/settings.py @@ -53,12 +53,9 @@ "gunicorn", "corsheaders", "rest_framework", + "django_rq", ) -if KEGBOT_ENV == ENV_TEST: - # Run all celery tasks synchronously. - CELERY_ALWAYS_EAGER = True - LOGIN_REDIRECT_URL = "/account/" KEGBOT_ADMIN_LOGIN_URL = "auth_login" @@ -154,7 +151,16 @@ "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", }, - } + "KEY_PREFIX": "kb:cache", + }, + "rq": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": KEGBOT["REDIS_URL"], + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + "KEY_PREFIX": "kb:rq", + }, } INTERNAL_IPS = ("127.0.0.1",) @@ -166,20 +172,19 @@ "pykeg.contrib.webhook.plugin.WebhookPlugin", ] -# Celery - -BROKER_URL = KEGBOT["REDIS_URL"] -CELERY_RESULT_BACKEND = KEGBOT["REDIS_URL"] +# RQ (workers) -CELERY_QUEUES = { - "default": {"exchange": "default", "binding_key": "default"}, - "stats": {"exchange": "default", "binding_key": "stats"}, +RQ_QUEUES = { + "default": { + "USE_REDIS_CACHE": "rq", + "ASYNC": KEGBOT_ENV != ENV_TEST, + }, + "stats": { + "USE_REDIS_CACHE": "rq", + "ASYNC": KEGBOT_ENV != ENV_TEST, + }, } -CELERY_DEFAULT_QUEUE = "default" -CELERYD_CONCURRENCY = 3 - - # logging LOGGING = { diff --git a/pykeg/util/runner.py b/pykeg/util/runner.py index 02d281013..71e0b5f5b 100644 --- a/pykeg/util/runner.py +++ b/pykeg/util/runner.py @@ -130,8 +130,6 @@ def preexec(): os.setsid() # Set umask to default to safe file permissions for root. os.umask(0o27) - # Switch to a "safe" directory. - os.chdir("/") proc = subprocess.Popen( command, diff --git a/pykeg/web/kegadmin/templates/kegadmin/nav-items.html b/pykeg/web/kegadmin/templates/kegadmin/nav-items.html index 3251ca939..e5ed971ed 100644 --- a/pykeg/web/kegadmin/templates/kegadmin/nav-items.html +++ b/pykeg/web/kegadmin/templates/kegadmin/nav-items.html @@ -45,7 +45,7 @@ {% navitem "kegadmin-bugreport" "Bugreport" %} {% navitem "kegadmin-export" "Export Data" %} {% if KEGBOT_ENABLE_ADMIN %} -{% navitem "kegadmin-system-status" "System Status" %}
  • Database Admin »
    • +
  • Workers »
    • {% endif %}
  • Report a Bug »
    • diff --git a/pykeg/web/kegadmin/templates/kegadmin/system_status.html b/pykeg/web/kegadmin/templates/kegadmin/system_status.html deleted file mode 100644 index a6f62b93f..000000000 --- a/pykeg/web/kegadmin/templates/kegadmin/system_status.html +++ /dev/null @@ -1,82 +0,0 @@ -{% extends "kegadmin/base.html" %} -{% load kegweblib %} -{% load crispy_forms_tags %} - -{% block title %}Kegbot Admin: System Status | {{ block.super }}{% endblock %} -{% block pagetitle %}Kegbot Admin: System Status{% endblock %} - -{% block kegadmin-main %} -

    Runtime info

    -{% if runtime_info %} - - - - - - - - - -{% for k, v in runtime_info.items %} - - - - -{% endfor %} - -
    NameValue
    {{ k }}
    {{ v }}
    - -{% else %} -Runtime info unknown. -{% endif %} - -

    Background worker status

    - -{% if error %} -
    -

    Connection Error

    -

    - Could not retrieve worker status. Error was: {{ error }} -

    -
    -{% endif %} - -{% if worker_status %} - - - - - - - - - - -{% for worker_name, status in worker_status.items %} - - - - -{% endfor %} - - -
    Worker NameStatusTasks — Run Count
    {{ worker_name }}{{ status.status }} - {% for task_name, task_count in status.stats.total.items %} - {{ task_name }} — {{ task_count }}
    - {% endfor %} -
    - -{% endif %} - - - Show Raw Stats » - -

    -
    -
    -{{ raw_stats }}
-
    -
    - - -{% endblock %} diff --git a/pykeg/web/kegadmin/urls.py b/pykeg/web/kegadmin/urls.py index 7bef86dd6..21dff4d3b 100644 --- a/pykeg/web/kegadmin/urls.py +++ b/pykeg/web/kegadmin/urls.py @@ -56,7 +56,6 @@ path("email/", views.email, name="kegadmin-email"), path("logs/", views.logs, name="kegadmin-logs"), path("users/create/", views.add_user, name="kegadmin-add-user"), - path("system/", views.system_status, name="kegadmin-system-status"), ] if util.get_plugins(): diff --git a/pykeg/web/kegadmin/views.py b/pykeg/web/kegadmin/views.py index d785181c5..0d23a73b6 100644 --- a/pykeg/web/kegadmin/views.py +++ b/pykeg/web/kegadmin/views.py @@ -20,7 +20,6 @@ from django.views.decorators.http import require_http_methods from pykeg.backup import backup -from pykeg.celery import app as celery_app from pykeg.core import models, tasks from pykeg.core.util import get_runtime_version_info from pykeg.logging.handlers import RedisListHandler @@ -40,12 +39,11 @@ def dashboard(request): email_configured = request.kbsite.email_is_configured() context["email_configured"] = email_configured - if settings.BROKER_URL.startswith("redis:"): - try: - r = redis.StrictRedis.from_url(settings.BROKER_URL) - r.ping() - except redis.RedisError as e: - context["redis_error"] = e.message if e.message else "Unknown error." + try: + r = redis.StrictRedis.from_url(settings.KEGBOT["REDIS_URL"]) + r.ping() + except redis.RedisError as e: + context["redis_error"] = e.message if e.message else "Unknown error." active_users = models.User.objects.filter(is_active=True).exclude(username="guest") context["num_users"] = len(active_users) @@ -202,41 +200,6 @@ def bugreport(request): return render(request, "kegadmin/bugreport.html", context=context) -@staff_member_required -def system_status(request): - context = {} - - context["runtime_info"] = get_runtime_version_info() - - try: - inspector = celery_app.control.inspect() - pings = inspector.ping() or {} - stats = inspector.stats() or {} - queues = inspector.active_queues() or {} - except redis.RedisError as e: - context["error"] = e - - status = {} - if not pings and "error" not in context: - context["error"] = "No response from workers. Not running?" - else: - for k, v in list(pings.items()): - status[k] = { - "status": "ok" if v.get("ok") else "unknown", - } - for k, v in list(stats.items()): - if k in status: - status[k]["stats"] = v - for k, v in list(queues.items()): - if k in status: - status[k]["active_queues"] = v - - context["worker_status"] = status - context["raw_stats"] = kbjson.dumps(context["worker_status"], indent=2) - - return render(request, "kegadmin/system_status.html", context=context) - - @staff_member_required def controller_list(request): context = {} diff --git a/pykeg/web/urls.py b/pykeg/web/urls.py index e8a627425..35fa81964 100644 --- a/pykeg/web/urls.py +++ b/pykeg/web/urls.py @@ -38,6 +38,7 @@ if settings.KEGBOT_ENABLE_ADMIN: urlpatterns += [ + path("admin/rq/", include("django_rq.urls")), path("admin/", admin.site.urls), ] diff --git a/pyproject.toml b/pyproject.toml index 224c01fec..3b88822a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ packages = [ [tool.poetry.dependencies] python = "^3.10.4" -celery = "*" coloredlogs = "*" dj-database-url = "*" dj-email-url = "*" @@ -53,6 +52,8 @@ future = "^0.18.2" freezegun = "^1.2.1" djangorestframework = "^3.13.1" django-cors-headers = "^3.13.0" +rq = "^1.10.1" +django-rq = "^2.5.1" [tool.poetry.dev-dependencies] black = "*" From cc274216d9b3a9e648b40b208a1488479d547937 Mon Sep 17 00:00:00 2001 From: mike wakerly Date: Fri, 29 Jul 2022 11:07:54 +0100 Subject: [PATCH 2/2] bugfix: always serve media from /media/ --- pykeg/web/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pykeg/web/urls.py b/pykeg/web/urls.py index 35fa81964..82dd7330b 100644 --- a/pykeg/web/urls.py +++ b/pykeg/web/urls.py @@ -34,7 +34,7 @@ urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) # Serve media uploads in all environments. -urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +urlpatterns += static("/media/", document_root=settings.MEDIA_ROOT) if settings.KEGBOT_ENABLE_ADMIN: urlpatterns += [