Add Mini datalake to crossfeed

backend/Makefile

@@ -9,6 +9,18 @@ syncdb:
 syncdb-populate:
 	docker compose exec backend python manage.py syncdb --populate
 
+syncdb-dangerously-force:
+	docker compose exec backend python manage.py syncdb --dangerouslyforce
+
+
+# Synchronize and populate the database
+syncmdl:
+	docker compose exec backend python manage.py syncmdl
+
+syncmdl-dangerously-force:
+	docker compose exec backend python manage.py syncmdl --dangerouslyforce
+
+
 # Pytest
 # i.e. make pytest FILE=xfd_api/tests/test_domain.py
 pytest:

backend/requirements.txt

@@ -17,6 +17,7 @@ click==8.1.7
 cryptography==38.0.0
 dill==0.3.9
 Django==5.1.4
+django-netfields==1.3.2
 dnspython==2.7.0
 docker==7.1.0
 elasticsearch==7.9.0

backend/src/xfd_django/xfd_api/management/commands/syncdb.py

@@ -39,11 +39,11 @@ def handle(self, *args, **options):
         # Step 1: Database Reset and Migration
         if dangerouslyforce:
             self.stdout.write("Dropping and recreating the database...")
-            drop_all_tables()
-            synchronize()
+            drop_all_tables(app_label="xfd_api")
+            synchronize(target_app_label="xfd_api")
         else:
             self.stdout.write("Applying migrations...")
-            synchronize()
+            synchronize(target_app_label="xfd_api")
 
         # Step 2: Elasticsearch Index Management
         manage_elasticsearch_indices(dangerouslyforce)

backend/src/xfd_django/xfd_api/management/commands/syncmdl.py

@@ -0,0 +1,88 @@
+"""Populate command."""
+# Standard Python Libraries
+import os
+
+# Third-Party Libraries
+from django.core.management.base import BaseCommand
+from django.db import connections
+from xfd_api.tasks.syncdb_helpers import drop_all_tables, synchronize
+
+
+class Command(BaseCommand):
+    """Syncmdl command."""
+
+    help = "Synchronizes the MDL with optional sample data, and manages Elasticsearch indices."
+
+    def add_arguments(self, parser):
+        """Add arguments."""
+        parser.add_argument(
+            "-d",
+            "--dangerouslyforce",
+            action="store_true",
+            help="Force drop and recreate the database.",
+        )
+
+    def handle(self, *args, **options):
+        """Handle method."""
+        dangerouslyforce = options["dangerouslyforce"]
+
+        mdl_username = os.getenv("MDL_USERNAME")
+        mdl_password = os.getenv("MDL_PASSWORD")
+        mdl_name = os.getenv("MDL_NAME")
+
+        if not (mdl_username and mdl_password and mdl_name):
+            self.stderr.write(
+                "Error: MDL_USERNAME, MDL_PASSWORD, and MDL_NAME must be set in the environment."
+            )
+            return
+
+        connection = connections["default"]
+
+        # Step 1: Database User and Database Setup
+        self.stdout.write("Setting up the MDL database and user...")
+
+        with connection.cursor() as cursor:
+            try:
+                cursor.execute(
+                    "CREATE USER {} WITH PASSWORD '{}';".format(
+                        mdl_username, mdl_password
+                    )
+                )
+            except Exception as e:
+                self.stdout.write(
+                    "User creation failed (likely already exists): {}".format(e)
+                )
+
+            try:
+                cursor.execute(
+                    "GRANT {} TO {};".format(mdl_username, os.getenv("DB_USERNAME"))
+                )
+            except Exception as e:
+                self.stdout.write("Granting role failed: {}".format(e))
+
+            try:
+                cursor.execute(
+                    "CREATE DATABASE {} OWNER {};".format(mdl_name, mdl_username)
+                )
+            except Exception as e:
+                self.stdout.write(
+                    "Database creation failed (likely already exists): {}".format(e)
+                )
+
+            try:
+                cursor.execute(
+                    "GRANT ALL PRIVILEGES ON DATABASE {} TO {};".format(
+                        mdl_name, mdl_username
+                    )
+                )
+            except Exception as e:
+                self.stdout.write("Granting privileges failed: {}".format(e))
+
+        # Step 2: Synchronize or Reset the Database
+        self.stdout.write("Synchronizing the MDL database schema...")
+        if dangerouslyforce:
+            self.stdout.write("Dropping and recreating the database...")
+            drop_all_tables(app_label="xfd_mini_dl")
+        synchronize(target_app_label="xfd_mini_dl")
+
+        self.stdout.write("Database synchronization complete.")