Merge pull request #491 from i-dot-ai/feature/run-e2e-in-cd-pipeline

Run e2e in cd pipeline

.env.integration

@@ -1,8 +1,60 @@
 EMBEDDING_MODEL=paraphrase-albert-small-v2
-DJANGO_SECRET_KEY=1n53cur3K3y
-POSTGRES_PASSWORD=insecure
-COMPRESSION_ENABLED=False
 AWS_REGION=eu-west-2
-DJANGO_LOG_LEVEL=WARNING
 OPENAI_MODEL=azure/gpt-35-turbo-16k
+
+# === Database ===
+
+ELASTIC__HOST=elasticsearch
+ELASTIC__VERSION=8.11.0
+ELASTIC__USER=elastic
+ELASTIC__PASSWORD=redboxpass
+ELASTIC__PORT=9200
+ELASTIC__SCHEME=http
+ELASTIC__SUBSCRIPTION_LEVEL=basic
 ELASTIC_ROOT_INDEX=redbox-data-integration
+
+# === Redis ===
+
+EMBED_QUEUE_NAME=redbox-embedder-queue
+INGEST_QUEUE_NAME=redbox-ingester-queue
+
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+# === Object Storage ===
+
+MINIO_HOST=minio
+MINIO_PORT=9000
+# MINIO_ACCESS_KEY=minioadmin
+# MINIO_SECRET_KEY=minioadmin
+AWS_ACCESS_KEY=minioadmin
+AWS_SECRET_KEY=minioadmin
+
+# minio or s3
+OBJECT_STORE=minio
+BUCKET_NAME=redbox-storage-dev
+
+# === Django App ===
+DJANGO_SETTINGS_MODULE=redbox_app.settings
+DEBUG=True
+DJANGO_SECRET_KEY=1n53cur3K3y
+DJANGO_LOG_LEVEL=DEBUG
+ENVIRONMENT=INTEGRATION
+POSTGRES_USER=redbox-core
+POSTGRES_DB=redbox-core
+POSTGRES_PASSWORD=insecure
+[email protected]
+POSTGRES_HOST=db
+CORE_API_HOST=core-api
+CORE_API_PORT=5002
+
+EMAIL_BACKEND_TYPE=CONSOLE
+GOV_NOTIFY_API_KEY=f4k3_k3y
+[email protected]
+GOVUK_NOTIFY_PLAIN_EMAIL_TEMPLATE_ID=example-id
+EMAIL_FILE_PATH='/app/mail'
+
+USE_STREAMING=False
+COMPRESSION_ENABLED=False
+FILE_EXPIRY_IN_DAYS=30
+MAX_SECURITY_CLASSIFICATION=OFFICIAL_SENSITIVE

.github/workflows/integration.yml

@@ -54,23 +54,20 @@ jobs:
         cp .env.integration .env
         echo AZURE_OPENAI_ENDPOINT=${{ secrets.AZURE_OPENAI_ENDPOINT }} >> .env
         echo AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }} >> .env
-        docker compose up -d --wait elasticsearch
-        docker compose up -d --wait core-api worker
-        
+        docker compose up -d --wait db elasticsearch minio redis
+        sleep 60
+        docker compose up -d --wait worker core-api django-app
 
-    - name: Wait 60s for services to be ready
+    - name: Wait for services to be ready
       run: |
         sleep 60
         docker ps
 
     - name: Test integration with pytest
       run: |
-        pip install requests
-        pip install pytest
-        pip install boto3
-        pip install python-jose
-        pip install websockets
-        python -m pytest tests
+        poetry install --only dev
+        poetry run playwright install --with-deps chromium
+        poetry run pytest tests/ --browser chromium
 
     - name: notify slack failure
       id: slack-failure

Makefile

@@ -3,6 +3,8 @@
 
 .PHONY: app reqs
 
+BACKUP_ENV_FILENAME = ".env.$(shell date +"%Y-%m-%d-%H:%M:%S").backup"
+
 reqs:
 	poetry install
 
@@ -37,17 +39,15 @@ test-django:
 	docker compose up -d --wait db minio
 	docker compose run django-app venv/bin/pytest tests/ --ds redbox_app.settings -v --cov=redbox_app.redbox_core --cov-fail-under 80 -o log_cli=true
 
-test-integration:
-	docker compose down
-	cp .env .env.backup
-	cp .env.integration .env
-	docker compose build core-api worker minio
-	docker compose up -d core-api worker minio
-	poetry install --no-root --no-ansi --with dev --without ai,api,worker
+test-integration: stop
+#	cp .env $(BACKUP_ENV_FILENAME)
+#	cp .env.integration .env
+	docker compose up -d --wait elasticsearch db worker minio core-api django-app
+	poetry install --no-root --no-ansi --with dev --without ai,api,worker,docs
 	sleep 10
-	poetry run pytest tests
-	cp .env.backup .env
-	rm .env.backup
+	poetry run pytest tests/
+#	cp $(BACKUP_ENV_FILENAME) .env
+#	rm $(BACKUP_ENV_FILENAME)
 
 collect-static:
 	docker compose run django-app venv/bin/django-admin collectstatic --noinput

django_app/redbox_app/setting_enums.py

@@ -0,0 +1,29 @@
+from enum import StrEnum
+
+
+class Environment(StrEnum):
+    def __new__(cls, value: str, is_test: bool, hosts=list[str]):
+        obj = str.__new__(cls, [value])
+        obj._value_ = value
+        obj.is_test = is_test
+        obj.hosts = hosts
+        return obj
+
+    def is_local(self) -> bool:
+        return self is Environment.LOCAL
+
+    LOCAL = ("LOCAL", True, ["localhost", "127.0.0.1", "0.0.0.0"])  # noqa: S104 nosec: B104: Not in prod
+    INTEGRATION = ("INTEGRATION", True, ["localhost", "127.0.0.1", "0.0.0.0"])  # noqa: S104 nosec: B104: Not in prod
+    DEV = ("DEV", False, ["redbox-dev.ai.cabinetoffice.gov.uk"])
+    PREPROD = ("PREPROD", False, ["redbox-preprod.ai.cabinetoffice.gov.uk"])
+    PROD = ("PROD", False, ["redbox.ai.cabinetoffice.gov.uk"])
+
+
+class Classification(StrEnum):
+    """Security classifications
+    https://www.gov.uk/government/publications/government-security-classifications/"""
+
+    OFFICIAL = "Official"
+    OFFICIAL_SENSITIVE = "Official Sensitive"
+    SECRET = "Secret"  # noqa: S105
+    TOP_SECRET = "Top Secret"  # noqa: S105

django_app/redbox_app/settings.py

@@ -1,22 +1,20 @@
 # mypy: ignore-errors
 
 import socket
-from enum import StrEnum
 from pathlib import Path
 
 import environ
 from dotenv import load_dotenv
+from redbox_app.setting_enums import Classification, Environment
 from storages.backends import s3boto3
 
-from .hosting_environment import HostingEnvironment
-
 load_dotenv()
 
 env = environ.Env()
 
 SECRET_KEY = env.str("DJANGO_SECRET_KEY")
-ENVIRONMENT = env.str("ENVIRONMENT")
-WEBSOCKET_SCHEME = "ws" if HostingEnvironment.is_local() else "wss"
+ENVIRONMENT = Environment[env.str("ENVIRONMENT").upper()]
+WEBSOCKET_SCHEME = "ws" if ENVIRONMENT.is_local() else "wss"
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env.bool("DEBUG")
@@ -144,13 +142,6 @@
 LOGIN_REDIRECT_URL = "homepage"
 LOGIN_URL = "sign-in"
 
-HOST = (
-    "redbox.ai.cabinetoffice.gov.uk"
-    if ENVIRONMENT.lower() == "prod"
-    else f"redbox-{ENVIRONMENT.lower()}.ai.cabinetoffice.gov.uk"
-)
-
-
 # CSP settings https://content-security-policy.com/
 # https://django-csp.readthedocs.io/
 CSP_DEFAULT_SRC = (
@@ -171,7 +162,7 @@
 )
 CSP_STYLE_SRC = ("'self'",)
 CSP_FRAME_ANCESTORS = ("'none'",)
-CSP_CONNECT_SRC = ["'self'", f"wss://{HOST}/ws/chat/"]
+CSP_CONNECT_SRC = ["'self'", f"wss://{ENVIRONMENT.hosts[0]}/ws/chat/"]
 
 # https://pypi.org/project/django-permissions-policy/
 PERMISSIONS_POLICY: dict[str, list] = {
@@ -210,7 +201,7 @@
 AWS_S3_FILE_OVERWRITE = False  # allows users to have duplicate file names
 
 
-if HostingEnvironment.is_local():
+if ENVIRONMENT.is_test:
     AWS_S3_SECRET_ACCESS_KEY = env.str("AWS_SECRET_KEY")
     AWS_ACCESS_KEY_ID = env.str("AWS_ACCESS_KEY")
     MINIO_HOST = env.str("MINIO_HOST")
@@ -226,12 +217,6 @@
             "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
         },
     }
-
-    ALLOWED_HOSTS = [
-        "localhost",
-        "127.0.0.1",
-        "0.0.0.0",  # noqa: S104
-    ]  # nosec B104 - don't do this on server!
 else:
     STORAGES = {
         "default": {
@@ -242,15 +227,17 @@
         },
     }
 
-    LOCALHOST = socket.gethostbyname(socket.gethostname())
-    ALLOWED_HOSTS = [LOCALHOST, HOST]
-
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security
     # Mozilla guidance max-age 2 years
     SECURE_HSTS_SECONDS = 2 * 365 * 24 * 60 * 60
     SECURE_HSTS_INCLUDE_SUBDOMAINS = True
     SESSION_COOKIE_SECURE = True
 
+if ENVIRONMENT.is_test:
+    ALLOWED_HOSTS = ENVIRONMENT.hosts
+else:
+    LOCALHOST = socket.gethostbyname(socket.gethostname())
+    ALLOWED_HOSTS = [LOCALHOST, *ENVIRONMENT.hosts]
 
 DATABASES = {
     "default": {
@@ -330,16 +317,4 @@
 USE_STREAMING = env.bool("USE_STREAMING")
 FILE_EXPIRY_IN_SECONDS = env.int("FILE_EXPIRY_IN_DAYS") * 24 * 60 * 60
 SUPERUSER_EMAIL = env.str("SUPERUSER_EMAIL", None)
-
-
-class Classification(StrEnum):
-    """Security classifications
-    https://www.gov.uk/government/publications/government-security-classifications/"""
-
-    OFFICIAL = "Official"
-    OFFICIAL_SENSITIVE = "Official Sensitive"
-    SECRET = "Secret"  # noqa: S105
-    TOP_SECRET = "Top Secret"  # noqa: S105
-
-
 MAX_SECURITY_CLASSIFICATION = Classification[env.str("MAX_SECURITY_CLASSIFICATION")]