From be4c924d599644b67fb52637b501a14e2c263f0c Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 11:45:39 +0100 Subject: [PATCH 01/17] starting point --- README.md | 11 +++++-- backend/ciso_assistant/settings.py | 22 +++++++------- backend/core/tasks.py | 49 ++++++++++++++++++++++++++++++ backend/poetry.lock | 34 ++++++++++++++++++++- backend/pyproject.toml | 1 + 5 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 backend/core/tasks.py diff --git a/README.md b/README.md index c33726081..3a3f6b94c 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Check out the online documentation on =0.800)", "pytest", "pytest-asyncio"] +[[package]] +name = "async-timeout" +version = "5.0.1" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_full_version < \"3.11.3\"" +files = [ + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, +] + [[package]] name = "attrs" version = "24.3.0" @@ -2359,6 +2372,25 @@ all = ["pillow (>=9.1.0)", "pypng"] pil = ["pillow (>=9.1.0)"] png = ["pypng"] +[[package]] +name = "redis" +version = "5.2.1" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, + {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + [[package]] name = "referencing" version = "0.36.1" @@ -2915,4 +2947,4 @@ test = ["pytest"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "fd698305960a2142ad6c8e97fd84af13b0f44d07d921fe9c0b44d9524320c9a6" +content-hash = "5427c1785e44066ec03ac778d11aa01be1e13599da22cde6813603487ab455bc" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b367643e7..4700ff6b1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -37,6 +37,7 @@ python-docx = "^1.1.2" docxtpl = "^0.19.0" numpy = "^2.1.3" matplotlib = "^3.9.3" +redis = "^5.2.1" [tool.poetry.group.dev.dependencies] pytest-django = "4.8.0" From 06b2f7274bbcc69714af24d23e0e774f9ff68774 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 13:04:04 +0100 Subject: [PATCH 02/17] variant with filesystem - experimental --- backend/ciso_assistant/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index d6976a64b..eb26010f0 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -415,9 +415,11 @@ def set_ciso_assistant_url(_, __, event_dict): # for dev: docker run -d -p 6379:6379 redis:alpine ## Huey settings HUEY = { - "huey_class": "huey.RedisHuey", + "huey_class": "huey.FileHuey", + # "huey_class": "huey.RedisHuey", "name": "ciso_assistant", "results": True, "immediate": False, # True for local dev, set to False to run in "live" regardless of DEBUG - "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), + "path": BASE_DIR / "db" / "huey-tasks", + # "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), } From 996494a744c1fb6f2fd534f7d762c529305b11e2 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 14:10:08 +0100 Subject: [PATCH 03/17] First implementation --- backend/core/tasks.py | 60 +++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index ee0f1cf05..38a65d614 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -7,43 +7,35 @@ # basic placeholders from the official doc # https://huey.readthedocs.io/en/latest/django.html -# -@task() -def count_beans(number): - print("-- counted %s beans --" % number) - return "Counted %s beans" % number - - -@task() -def send_email(): - print("sending an email") - return "email sent" -@periodic_task(crontab(minute="*/1")) -def every_min(): - print("Every five minutes this will be printed by the consumer") - - -@db_task() -def do_some_queries(): - # This task executes queries. Once the task finishes, the connection - # will be closed. - pass +@db_periodic_task(crontab(minute="*/1")) +def check_controls_with_expired_eta(): + expired_controls = ( + AppliedControl.objects.exclude(status="active") + .filter(eta__lt=date.today(), eta__isnull=False) + .prefetch_related("owner") + ) + # Group by individual owner + owner_controls = {} + for control in expired_controls: + for owner in control.owner.all(): + if owner.email not in owner_controls: + owner_controls[owner.email] = [] + owner_controls[owner.email].append(control) -@db_periodic_task(crontab(minute="*/5")) -def every_five_mins_db(): - print("Every five minutes this will be printed by the consumer") + # Send personalized email to each owner + for owner_email, controls in owner_controls.items(): + send_notification_email(owner_email, controls) -# unrralistic, just for testing -@db_periodic_task(crontab(minute="*/1")) -def check_controls_with_expired_eta(): - expired_eta_controls = AppliedControl.objects.exclude(status="active").filter( - eta__lt=date.today(), eta__isnull=False - ) - print(f"Found {expired_eta_controls.count()} expired controls") - for ac in expired_eta_controls: - print(":: ", ac.name) - send_email() +@task() +def send_notification_email(owner_email, controls): + subject = f"You have {len(controls)} expired controls" + message = "The following controls have expired:\n\n" + for control in controls: + message += f"- {control.name} (ETA: {control.eta})\n" + + print(f"Sending email to {owner_email}") + print(message) From ee99fb8bca8f10d665d7988c91f9533dd7d0ca97 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 14:58:13 +0100 Subject: [PATCH 04/17] checkpoint --- backend/ciso_assistant/settings.py | 4 ++++ backend/core/tasks.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index eb26010f0..bc700445a 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -412,6 +412,10 @@ def set_ciso_assistant_url(_, __, event_dict): }, } +if DEBUG: + EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" + DEFAULT_FROM_EMAIL = "noreply@example.com" + # for dev: docker run -d -p 6379:6379 redis:alpine ## Huey settings HUEY = { diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 38a65d614..97945e7a1 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -1,9 +1,10 @@ -from datetime import date, datetime, timezone +from datetime import date from huey import crontab from huey.contrib.djhuey import periodic_task, task, db_periodic_task, db_task from core.models import AppliedControl from django.db.models import Q - +from django.core.mail import send_mail +from django.conf import settings # basic placeholders from the official doc # https://huey.readthedocs.io/en/latest/django.html @@ -32,10 +33,16 @@ def check_controls_with_expired_eta(): @task() def send_notification_email(owner_email, controls): - subject = f"You have {len(controls)} expired controls" + subject = f"CISO Assistant: You have {len(controls)} expired control(s)" message = "The following controls have expired:\n\n" for control in controls: message += f"- {control.name} (ETA: {control.eta})\n" - - print(f"Sending email to {owner_email}") - print(message) + message += "\nThis reminder will stop once the control is marked as active or you update the ETA.\n" + # think templating and i18n + send_mail( + subject=subject, + message=message, + from_email=settings.DEFAULT_FROM_EMAIL, + recipient_list=[owner_email], + fail_silently=False, + ) From 3615f39dbf0a969afbbf060e1cf0bdacf8588eb9 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 15:21:16 +0100 Subject: [PATCH 05/17] checkpoint --- backend/core/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 97945e7a1..e81f65150 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -34,10 +34,12 @@ def check_controls_with_expired_eta(): @task() def send_notification_email(owner_email, controls): subject = f"CISO Assistant: You have {len(controls)} expired control(s)" - message = "The following controls have expired:\n\n" + message = "Hello,\n\nThe following controls have expired:\n\n" for control in controls: message += f"- {control.name} (ETA: {control.eta})\n" message += "\nThis reminder will stop once the control is marked as active or you update the ETA.\n" + message += "Log in to your CISO Assistant portal and check 'my assignments' section to get to your controls directly.\n\n" + message += "Thank you." # think templating and i18n send_mail( subject=subject, From a10aab7594724a54ba7aec7bffee03e69d65b2ea Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 15:58:12 +0100 Subject: [PATCH 06/17] checkpoint valkey --- backend/ciso_assistant/settings.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index bc700445a..17897c9c8 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -416,14 +416,15 @@ def set_ciso_assistant_url(_, __, event_dict): EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" DEFAULT_FROM_EMAIL = "noreply@example.com" -# for dev: docker run -d -p 6379:6379 redis:alpine +# for dev: docker run -d -p 6379:6379 valkey/valkey:alpine +# There are reference to redis but we're using valkey which is a drop-in replacement ## Huey settings HUEY = { - "huey_class": "huey.FileHuey", - # "huey_class": "huey.RedisHuey", + # "huey_class": "huey.FileHuey", + "huey_class": "huey.RedisHuey", "name": "ciso_assistant", - "results": True, + "results": False, # would be interesting for debug "immediate": False, # True for local dev, set to False to run in "live" regardless of DEBUG - "path": BASE_DIR / "db" / "huey-tasks", - # "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), + # "path": BASE_DIR / "db" / "huey-tasks", + "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), } From 9622fc11ccd19f645beaba47926fba50ce88b2a0 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 16:17:25 +0100 Subject: [PATCH 07/17] sqlite backend for huey, compatible with pg for the rest of the app --- backend/ciso_assistant/settings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index 17897c9c8..28934534e 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -421,10 +421,11 @@ def set_ciso_assistant_url(_, __, event_dict): ## Huey settings HUEY = { # "huey_class": "huey.FileHuey", - "huey_class": "huey.RedisHuey", + "huey_class": "huey.SqliteHuey", "name": "ciso_assistant", - "results": False, # would be interesting for debug + "filename": BASE_DIR / "db" / "huey.db", + "results": True, # would be interesting for debug "immediate": False, # True for local dev, set to False to run in "live" regardless of DEBUG # "path": BASE_DIR / "db" / "huey-tasks", - "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), + # "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), } From a5dd2ce46311400ec603476cb959150fb9ffaa35 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 17:19:18 +0100 Subject: [PATCH 08/17] compose preperation --- docker-compose-build.yml | 35 +++++++++------- docker-compose-pg-build.yml | 83 +++++++++++++++++++++++++++++++++++++ docker-compose-pg.yml | 39 ++++++++++------- docker-compose.yml | 19 +++++++++ 4 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 docker-compose-pg-build.yml diff --git a/docker-compose-build.yml b/docker-compose-build.yml index f55da3050..1b0567681 100644 --- a/docker-compose-build.yml +++ b/docker-compose-build.yml @@ -12,22 +12,25 @@ services: volumes: - ./db:/code/db - # huey: - # container_name: huey - # build: - # context: ./backend - # dockerfile: Dockerfile - # restart: always - # environment: - # - ALLOWED_HOSTS=backend,localhost - # - DJANGO_DEBUG=False - # volumes: - # - ./db:/code/db - # entrypoint: - # - /bin/sh - # - -c - # - | - # poetry run python manage.py run_huey + huey: + container_name: huey + build: + context: ./backend + dockerfile: Dockerfile + depends_on: + - backend + restart: always + environment: + - ALLOWED_HOSTS=backend,localhost + - CISO_ASSISTANT_URL=https://localhost:8443 + - DJANGO_DEBUG=False + volumes: + - ./db:/code/db + entrypoint: + - /bin/sh + - -c + - | + poetry run python manage.py run_huey -w 2 -k process frontend: container_name: frontend diff --git a/docker-compose-pg-build.yml b/docker-compose-pg-build.yml new file mode 100644 index 000000000..fe6167d52 --- /dev/null +++ b/docker-compose-pg-build.yml @@ -0,0 +1,83 @@ +services: + backend: + container_name: backend + build: + context: ./backend + dockerfile: Dockerfile + restart: always + depends_on: + - postgres + environment: + - ALLOWED_HOSTS=backend,localhost + - CISO_ASSISTANT_URL=https://localhost:8443 + - DJANGO_DEBUG=True + - POSTGRES_NAME=ciso_assistant + - POSTGRES_USER=ciso_assistant + - POSTGRES_PASSWORD=ciso_assistant + - DB_HOST=postgres + volumes: + - ./db:/code/db + + huey: + container_name: huey + build: + context: ./backend + dockerfile: Dockerfile + depends_on: + - backend + restart: always + environment: + - ALLOWED_HOSTS=backend,localhost + - CISO_ASSISTANT_URL=https://localhost:8443 + - DJANGO_DEBUG=False + - POSTGRES_NAME=ciso_assistant + - POSTGRES_USER=ciso_assistant + - POSTGRES_PASSWORD=ciso_assistant + - DB_HOST=postgres + volumes: + - ./db:/code/db + entrypoint: + - /bin/sh + - -c + - | + poetry run python manage.py run_huey -w 2 -k process + + frontend: + container_name: frontend + environment: + - PUBLIC_BACKEND_API_URL=http://backend:8000/api + - PROTOCOL_HEADER=x-forwarded-proto + - HOST_HEADER=x-forwarded-host + + build: + context: ./frontend + dockerfile: Dockerfile + depends_on: + - backend + + postgres: + container_name: postgres + image: postgres:16 + restart: always + environment: + POSTGRES_DB: ciso_assistant + POSTGRES_USER: ciso_assistant + POSTGRES_PASSWORD: ciso_assistant + volumes: + - ./db/pg:/var/lib/postgresql/data + + caddy: + container_name: caddy + image: caddy:2.8.4 + restart: unless-stopped + ports: + - 8443:8443 + command: + - caddy + - reverse-proxy + - --from + - https://localhost:8443 + - --to + - frontend:3000 + volumes: + - ./db:/data diff --git a/docker-compose-pg.yml b/docker-compose-pg.yml index 2336c9412..e01e76818 100644 --- a/docker-compose-pg.yml +++ b/docker-compose-pg.yml @@ -6,9 +6,9 @@ services: depends_on: - postgres environment: - - ALLOWED_HOSTS=backend + - ALLOWED_HOSTS=backend,localhost - CISO_ASSISTANT_URL=https://localhost:8443 - - DJANGO_DEBUG=True + - DJANGO_DEBUG=False - POSTGRES_NAME=ciso_assistant - POSTGRES_USER=ciso_assistant - POSTGRES_PASSWORD=ciso_assistant @@ -16,20 +16,27 @@ services: volumes: - ./db:/code/db - # huey: - # container_name: huey - # image: ghcr.io/intuitem/ciso-assistant-community/backend:latest - # restart: always - # environment: - # - ALLOWED_HOSTS=backend,localhost - # - DJANGO_DEBUG=False - # volumes: - # - ./db:/code/db - # entrypoint: - # - /bin/sh - # - -c - # - | - # poetry run python manage.py run_huey + huey: + container_name: huey + image: ghcr.io/intuitem/ciso-assistant-community/backend:latest + depends_on: + - backend + restart: always + environment: + - ALLOWED_HOSTS=backend,localhost + - CISO_ASSISTANT_URL=https://localhost:8443 + - DJANGO_DEBUG=False + - POSTGRES_NAME=ciso_assistant + - POSTGRES_USER=ciso_assistant + - POSTGRES_PASSWORD=ciso_assistant + - DB_HOST=postgres + volumes: + - ./db:/code/db + entrypoint: + - /bin/sh + - -c + - | + poetry run python manage.py run_huey -w 2 -k process frontend: container_name: frontend diff --git a/docker-compose.yml b/docker-compose.yml index d723deee3..8f2cb9bf7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,25 @@ services: volumes: - ./db:/code/db + huey: + container_name: huey + image: ghcr.io/intuitem/ciso-assistant-community/backend:latest + depends_on: + - backend + restart: always + environment: + - ALLOWED_HOSTS=backend,localhost + - CISO_ASSISTANT_URL=https://localhost:8443 + - DJANGO_DEBUG=False + - AUTH_TOKEN_TTL=7200 + volumes: + - ./db:/code/db + entrypoint: + - /bin/sh + - -c + - | + poetry run python manage.py run_huey -w 2 -k process + frontend: container_name: frontend environment: From bc18760bca4448a53256dac366090670055d5dcf Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 19:39:32 +0100 Subject: [PATCH 09/17] Introduce MAIL_DEBUG env variable --- backend/ciso_assistant/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index 28934534e..807cd6a16 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -110,6 +110,7 @@ def set_ciso_assistant_url(_, __, event_dict): # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get("DJANGO_DEBUG", "False") == "True" +MAIL_DEBUG = os.environ.get("MAIL_DEBUG", "False") logger.info("DEBUG mode: %s", DEBUG) logger.info("CISO_ASSISTANT_URL: %s", CISO_ASSISTANT_URL) @@ -412,9 +413,9 @@ def set_ciso_assistant_url(_, __, event_dict): }, } -if DEBUG: +if MAIL_DEBUG: EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" - DEFAULT_FROM_EMAIL = "noreply@example.com" + DEFAULT_FROM_EMAIL = "noreply@ciso.assistant" # for dev: docker run -d -p 6379:6379 valkey/valkey:alpine # There are reference to redis but we're using valkey which is a drop-in replacement From 1ba609a6fc72d99d96d0b515697b8240cb9378bb Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 19:49:35 +0100 Subject: [PATCH 10/17] checkpoint --- README.md | 3 +-- backend/ciso_assistant/settings.py | 7 +------ backend/core/tasks.py | 3 ++- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3a3f6b94c..958bf9af7 100644 --- a/README.md +++ b/README.md @@ -410,9 +410,8 @@ ln -fs ../../git_hooks/post-merge . 11. for Huey (tasks runner) -- run redis as a broker `docker run -d -p 6379:6379 redis:alpine`. This should make it available on localhost. - run `python manage.py run_huey -w 2 -k process` or equivalent in a separate shell. - +- you can use `MAIL_DEBUG` to have mail on the console for easier debug ### Running the frontend diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index 807cd6a16..6f938b3a7 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -417,16 +417,11 @@ def set_ciso_assistant_url(_, __, event_dict): EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" DEFAULT_FROM_EMAIL = "noreply@ciso.assistant" -# for dev: docker run -d -p 6379:6379 valkey/valkey:alpine -# There are reference to redis but we're using valkey which is a drop-in replacement ## Huey settings HUEY = { - # "huey_class": "huey.FileHuey", "huey_class": "huey.SqliteHuey", "name": "ciso_assistant", "filename": BASE_DIR / "db" / "huey.db", "results": True, # would be interesting for debug - "immediate": False, # True for local dev, set to False to run in "live" regardless of DEBUG - # "path": BASE_DIR / "db" / "huey-tasks", - # "url": os.environ.get("REDIS_URL", "redis://localhost:6379/?db=1"), + "immediate": False, # set to False to run in "live" mode regardless of DEBUG, otherwise it will follow } diff --git a/backend/core/tasks.py b/backend/core/tasks.py index e81f65150..54542e9be 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -6,7 +6,7 @@ from django.core.mail import send_mail from django.conf import settings -# basic placeholders from the official doc +# official doc # https://huey.readthedocs.io/en/latest/django.html @@ -33,6 +33,7 @@ def check_controls_with_expired_eta(): @task() def send_notification_email(owner_email, controls): + # TODO: check that the mailer is properly set and log an error otherwise subject = f"CISO Assistant: You have {len(controls)} expired control(s)" message = "Hello,\n\nThe following controls have expired:\n\n" for control in controls: From c222a1382b73ef240e869be5d4c7a1a36a6f53f5 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 20:26:57 +0100 Subject: [PATCH 11/17] logging and error management --- backend/ciso_assistant/settings.py | 2 +- backend/core/tasks.py | 48 ++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index 6f938b3a7..16ca77985 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -110,7 +110,7 @@ def set_ciso_assistant_url(_, __, event_dict): # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get("DJANGO_DEBUG", "False") == "True" -MAIL_DEBUG = os.environ.get("MAIL_DEBUG", "False") +MAIL_DEBUG = os.environ.get("MAIL_DEBUG", "False") == "True" logger.info("DEBUG mode: %s", DEBUG) logger.info("CISO_ASSISTANT_URL: %s", CISO_ASSISTANT_URL) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 54542e9be..951a9cbf9 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -5,9 +5,13 @@ from django.db.models import Q from django.core.mail import send_mail from django.conf import settings +import logging -# official doc -# https://huey.readthedocs.io/en/latest/django.html +import logging.config +import structlog + +logging.config.dictConfig(settings.LOGGING) +logger = structlog.getLogger(__name__) @db_periodic_task(crontab(minute="*/1")) @@ -17,7 +21,6 @@ def check_controls_with_expired_eta(): .filter(eta__lt=date.today(), eta__isnull=False) .prefetch_related("owner") ) - # Group by individual owner owner_controls = {} for control in expired_controls: @@ -25,7 +28,6 @@ def check_controls_with_expired_eta(): if owner.email not in owner_controls: owner_controls[owner.email] = [] owner_controls[owner.email].append(control) - # Send personalized email to each owner for owner_email, controls in owner_controls.items(): send_notification_email(owner_email, controls) @@ -33,7 +35,23 @@ def check_controls_with_expired_eta(): @task() def send_notification_email(owner_email, controls): - # TODO: check that the mailer is properly set and log an error otherwise + # Check required email settings + required_settings = ["EMAIL_HOST", "EMAIL_PORT", "DEFAULT_FROM_EMAIL"] + missing_settings = [ + setting + for setting in required_settings + if not hasattr(settings, setting) or not getattr(settings, setting) + ] + + if missing_settings: + error_msg = f"Cannot send email notification: Missing email settings: {', '.join(missing_settings)}" + logger.error(error_msg) + return + + if not owner_email: + logger.error("Cannot send email notification: No recipient email provided") + return + subject = f"CISO Assistant: You have {len(controls)} expired control(s)" message = "Hello,\n\nThe following controls have expired:\n\n" for control in controls: @@ -41,11 +59,15 @@ def send_notification_email(owner_email, controls): message += "\nThis reminder will stop once the control is marked as active or you update the ETA.\n" message += "Log in to your CISO Assistant portal and check 'my assignments' section to get to your controls directly.\n\n" message += "Thank you." - # think templating and i18n - send_mail( - subject=subject, - message=message, - from_email=settings.DEFAULT_FROM_EMAIL, - recipient_list=[owner_email], - fail_silently=False, - ) + + try: + send_mail( + subject=subject, + message=message, + from_email=settings.DEFAULT_FROM_EMAIL, + recipient_list=[owner_email], + fail_silently=False, + ) + logger.info(f"Successfully sent notification email to {owner_email}") + except Exception as e: + logger.error(f"Failed to send notification email to {owner_email}: {str(e)}") From 40b88906df341536fe8991a1fbcb5af85304642d Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 22:51:27 +0100 Subject: [PATCH 12/17] wip --- backend/core/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 951a9cbf9..586d18423 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -6,6 +6,7 @@ from django.core.mail import send_mail from django.conf import settings import logging +import random import logging.config import structlog @@ -14,7 +15,7 @@ logger = structlog.getLogger(__name__) -@db_periodic_task(crontab(minute="*/1")) +@db_periodic_task(crontab(minute=str(random.randint(0, 59)), hour="21")) def check_controls_with_expired_eta(): expired_controls = ( AppliedControl.objects.exclude(status="active") From 7bfaa3b6c332e48174eff20dd733f9c995859e93 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sat, 25 Jan 2025 23:09:55 +0100 Subject: [PATCH 13/17] target value --- backend/core/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 586d18423..583ce4a22 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -15,7 +15,8 @@ logger = structlog.getLogger(__name__) -@db_periodic_task(crontab(minute=str(random.randint(0, 59)), hour="21")) +# @db_periodic_task(crontab(minute='*/1')) for testing +@db_periodic_task(crontab(hour="6")) def check_controls_with_expired_eta(): expired_controls = ( AppliedControl.objects.exclude(status="active") From 37e260cd97a3f7b321518d1641142cc99a315740 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sun, 26 Jan 2025 00:05:22 +0100 Subject: [PATCH 14/17] slow down the scheduler given the current use cases to save CPU --- docker-compose-build.yml | 2 +- docker-compose-pg-build.yml | 2 +- docker-compose-pg.yml | 2 +- docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-build.yml b/docker-compose-build.yml index 1b0567681..a29d6b383 100644 --- a/docker-compose-build.yml +++ b/docker-compose-build.yml @@ -30,7 +30,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process + poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 frontend: container_name: frontend diff --git a/docker-compose-pg-build.yml b/docker-compose-pg-build.yml index fe6167d52..8607233d7 100644 --- a/docker-compose-pg-build.yml +++ b/docker-compose-pg-build.yml @@ -40,7 +40,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process + poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 frontend: container_name: frontend diff --git a/docker-compose-pg.yml b/docker-compose-pg.yml index e01e76818..1dffc8ff8 100644 --- a/docker-compose-pg.yml +++ b/docker-compose-pg.yml @@ -36,7 +36,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process + poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 frontend: container_name: frontend diff --git a/docker-compose.yml b/docker-compose.yml index 8f2cb9bf7..5d7581a1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process + poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 frontend: container_name: frontend From 1027c9c0af28e02c69ca66fc7cbcac14163bcc6d Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sun, 26 Jan 2025 00:08:50 +0100 Subject: [PATCH 15/17] fixup --- README.md | 2 +- backend/poetry.lock | 34 +--------------------------------- backend/pyproject.toml | 1 - 3 files changed, 2 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 958bf9af7..e217ed15c 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,6 @@ For docker setup on a remote server or hypervisor, checkout the [specific instru - npm 10.2+ - pnpm 9.0+ - yaml-cpp (brew install yaml-cpp libyaml or apt install libyaml-cpp-dev) -- redis 5+ ### Running the backend @@ -410,6 +409,7 @@ ln -fs ../../git_hooks/post-merge . 11. for Huey (tasks runner) +- prepare a mailer for testing. - run `python manage.py run_huey -w 2 -k process` or equivalent in a separate shell. - you can use `MAIL_DEBUG` to have mail on the console for easier debug diff --git a/backend/poetry.lock b/backend/poetry.lock index 7442c6030..f86785ed4 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -74,19 +74,6 @@ files = [ [package.extras] tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_full_version < \"3.11.3\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - [[package]] name = "attrs" version = "24.3.0" @@ -2372,25 +2359,6 @@ all = ["pillow (>=9.1.0)", "pypng"] pil = ["pillow (>=9.1.0)"] png = ["pypng"] -[[package]] -name = "redis" -version = "5.2.1" -description = "Python client for Redis database and key-value store" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, - {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, -] - -[package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} - -[package.extras] -hiredis = ["hiredis (>=3.0.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] - [[package]] name = "referencing" version = "0.36.1" @@ -2947,4 +2915,4 @@ test = ["pytest"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "5427c1785e44066ec03ac778d11aa01be1e13599da22cde6813603487ab455bc" +content-hash = "fd698305960a2142ad6c8e97fd84af13b0f44d07d921fe9c0b44d9524320c9a6" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 4700ff6b1..b367643e7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -37,7 +37,6 @@ python-docx = "^1.1.2" docxtpl = "^0.19.0" numpy = "^2.1.3" matplotlib = "^3.9.3" -redis = "^5.2.1" [tool.poetry.group.dev.dependencies] pytest-django = "4.8.0" From c98f0ac1f8a14f0973e4602c8abebc986918b60a Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Sun, 26 Jan 2025 21:13:58 +0100 Subject: [PATCH 16/17] back to default worker class for now --- docker-compose-build.yml | 2 +- docker-compose-pg-build.yml | 2 +- docker-compose-pg.yml | 2 +- docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-build.yml b/docker-compose-build.yml index a29d6b383..03af43fe7 100644 --- a/docker-compose-build.yml +++ b/docker-compose-build.yml @@ -30,7 +30,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 + poetry run python manage.py run_huey -w 2 --scheduler-interval 60 frontend: container_name: frontend diff --git a/docker-compose-pg-build.yml b/docker-compose-pg-build.yml index 8607233d7..3010dd5df 100644 --- a/docker-compose-pg-build.yml +++ b/docker-compose-pg-build.yml @@ -40,7 +40,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 + poetry run python manage.py run_huey -w 2 --scheduler-interval 60 frontend: container_name: frontend diff --git a/docker-compose-pg.yml b/docker-compose-pg.yml index 1dffc8ff8..52362ec1d 100644 --- a/docker-compose-pg.yml +++ b/docker-compose-pg.yml @@ -36,7 +36,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 + poetry run python manage.py run_huey -w 2 --scheduler-interval 60 frontend: container_name: frontend diff --git a/docker-compose.yml b/docker-compose.yml index 5d7581a1f..cfbceab96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: - /bin/sh - -c - | - poetry run python manage.py run_huey -w 2 -k process --scheduler-interval 60 + poetry run python manage.py run_huey -w 2 --scheduler-interval 60 frontend: container_name: frontend From 704e444b439593cfc1f9453fe3f004792f9feef8 Mon Sep 17 00:00:00 2001 From: Abderrahmane Smimite Date: Mon, 27 Jan 2025 18:34:22 +0100 Subject: [PATCH 17/17] Use the global setting and default to false --- backend/core/tasks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 583ce4a22..bc896b062 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -7,6 +7,7 @@ from django.conf import settings import logging import random +from global_settings.models import GlobalSettings import logging.config import structlog @@ -15,7 +16,7 @@ logger = structlog.getLogger(__name__) -# @db_periodic_task(crontab(minute='*/1')) for testing +# @db_periodic_task(crontab(minute='*/1'))# for testing @db_periodic_task(crontab(hour="6")) def check_controls_with_expired_eta(): expired_controls = ( @@ -37,6 +38,16 @@ def check_controls_with_expired_eta(): @task() def send_notification_email(owner_email, controls): + # TODO this will probably will move to a common section later on. + notifications_enable_mailing = GlobalSettings.objects.get(name="general").value.get( + "notifications_enable_mailing", False + ) + if not notifications_enable_mailing: + logger.warning( + "Email notification is disabled. You can enable it under Extra/Settings. Skipping for now." + ) + return + # Check required email settings required_settings = ["EMAIL_HOST", "EMAIL_PORT", "DEFAULT_FROM_EMAIL"] missing_settings = [