Merge pull request #45 from juntossomosmais/feat/update-dependencies

chore(dependencies): update packages and optimize publish query

.pre-commit-config.yaml

@@ -2,28 +2,28 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/pycqa/pylint
-    rev: v2.17.1
+    rev: v3.2.5
     hooks:
       - id: pylint
         additional_dependencies: [ django, 'stomp.py' ]
         exclude: migrations
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 7.1.0
     hooks:
       - id: flake8
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 24.4.2
     hooks:
       - id: black
         exclude: migrations/
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort

Dockerfile

@@ -1,7 +1,11 @@
-FROM python:3.7-slim
+FROM python:3.9-slim
 
 WORKDIR /app
 
+# Git is required for pre-commit
+RUN apt update
+RUN apt install -y git
+
 RUN pip install poetry
 
 COPY . .

README.md

@@ -388,3 +388,7 @@ The `OUTBOX_PATTERN_PUBLISHER_CACHE_KEY` variable controls the key name of the c
 **OUTBOX_PATTERN_CONSUMER_CACHE_KEY**
 
 The `OUTBOX_PATTERN_CONSUMER_CACHE_KEY` variable controls the key name of the cache used to store the outbox pattern publisher. Default: `remove_old_messages_django_outbox_pattern_consumer`.
+
+**DEFAULT_PUBLISHED_CHUNK_SIZE**
+
+The `DEFAULT_PUBLISHED_CHUNK_SIZE` variable controls chunk size for the `publish` command in get message to publish action. Default: 200

azure-pipelines.yml

@@ -8,10 +8,9 @@ trigger:
       - tests
       - Dockerfile
       - poetry.lock
+      - pyproject.toml
 pr:
   - main
-  - develop
-  - release/*
 
 resources:
   repositories:
@@ -20,7 +19,6 @@ resources:
       name: juntossomosmais/azure-pipelines-templates
       endpoint: github.com
       ref: main
-      PoetryVersion: 1.4.2
 
 extends:
   template: python/library.yaml@templates

django_outbox_pattern/management/commands/publish.py

@@ -46,7 +46,7 @@ def _publish(self):
             try:
                 published = self.published_class.objects.filter(
                     status=StatusChoice.SCHEDULE, expires_at__gte=timezone.now()
-                )
+                ).iterator(chunk_size=int(settings.DEFAULT_PUBLISHED_CHUNK_SIZE))
                 for message in published:
                     try:
                         attempts = self.producer.send(message)

django_outbox_pattern/migrations/0006_published_published_status_27c9ec_btree.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.14 on 2024-07-15 16:14
+
+import django.contrib.postgres.indexes
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("django_outbox_pattern", "0005_published_headers"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="published",
+            index=django.contrib.postgres.indexes.BTreeIndex(fields=["status"], name="published_status_27c9ec_btree"),
+        ),
+    ]

django_outbox_pattern/models.py

@@ -1,6 +1,7 @@
 import uuid
 from datetime import timedelta
 
+from django.contrib.postgres.indexes import BTreeIndex
 from django.db import models
 from django.utils import timezone
 
@@ -32,6 +33,9 @@ class Published(models.Model):
     class Meta:
         verbose_name = "published"
         db_table = "published"
+        indexes = [
+            BTreeIndex(fields=["status"]),
+        ]
 
     def __str__(self):
         return f"{self.destination} - {self.body}"

django_outbox_pattern/settings.py

@@ -17,6 +17,7 @@
     "DEFAULT_PRODUCER_LISTENER_CLASS", "django_outbox_pattern.listeners.ProducerListener"
 )
 DEFAULT_PUBLISHED_CLASS = DJANGO_OUTBOX_PATTERN.get("DEFAULT_PUBLISHED_CLASS", "django_outbox_pattern.models.Published")
+DEFAULT_PUBLISHED_CHUNK_SIZE = DJANGO_OUTBOX_PATTERN.get("DEFAULT_PUBLISHED_CHUNK_SIZE", 200)
 DEFAULT_RECEIVED_CLASS = DJANGO_OUTBOX_PATTERN.get("DEFAULT_RECEIVED_CLASS", "django_outbox_pattern.models.Received")
 DEFAULT_STOMP_HOST_AND_PORTS = DJANGO_OUTBOX_PATTERN.get("DEFAULT_STOMP_HOST_AND_PORTS", [("127.0.0.1", 61613)])
 DEFAULT_STOMP_QUEUE_HEADERS = DJANGO_OUTBOX_PATTERN.get(

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   integration-tests:
     build: .
@@ -16,7 +15,7 @@ services:
       - djangooutboxpattern
 
   rabbitmq:
-    image : rabbitmq:3.8-management
+    image : rabbitmq:3-management
     volumes:
        - ./tests/resources/rabbitmq:/etc/rabbitmq/
     healthcheck:
@@ -39,7 +38,9 @@ services:
       - djangooutboxpattern
 
   db:
-    image: postgres:12-alpine
+    image: postgres:16-alpine
+    ports:
+      - 5432:5432
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
@@ -52,5 +53,13 @@ services:
     networks:
       - djangooutboxpattern
 
+  lint-formatter:
+    build: .
+    volumes:
+      - .:/app
+    command: [ "./scripts/start-formatter-lint.sh" ]
+    networks:
+      - djangooutboxpattern
+
 networks:
   djangooutboxpattern: