Dj db duplicate saves

.gitignore

@@ -17,6 +17,7 @@ src/pe_reports/assets/
 __pycache__
 .coverage
 .mypy_cache
+venv/
 .pytest_cache
 .python-version
 *.egg-info
@@ -53,4 +54,4 @@ dnstwist_output.txt
 adhoc_investigations/adhoc_investigation.ini
 adhoc_investigations/input_data
 adhoc_investigations/output_data
-adhoc_investigations/dnsmonitor_monitored_domains.csv
+adhoc_investigations/dnsmonitor_monitored_domains.csv

setup.py

@@ -129,8 +129,10 @@ def get_version(version_file):
         "matplotlib == 3.3.4",
         "nested-lookup",
         "openpyxl",
-        "pandas == 1.1.5",
+        "pandas",
         "pdfkit",
+        "presidio-analyzer",
+        "presidio-anonymizer",
         "psutil",
         "psycopg2-binary",
         "psycopg2-binary",
@@ -147,12 +149,17 @@ def get_version(version_file):
         "reportlab",
         "requests",
         "schema == 0.7.5",
+        "scrubadub",
         "setuptools == 58.1.0",
         "scikit-learn",
         "shodan == 1.27.0",
         "sshtunnel",
         "sslyze>=5.0.0",
-        # "spacy",
+        "spacy",
+        "spacy-loggers",
+        "spacy-legacy",
+        "spacy-transformers",
+        "spacy-alignments",
         "nltk",
         "beautifulsoup4",
         "sublist3r",

src/pe_reports/Dockerfile

@@ -0,0 +1,47 @@
+# Use the Python 3.10.2 image as the base image
+
+FROM python:3.10.2
+
+# Install required tools
+#RUN apt-get update && apt-get install -y curl build-essential
+
+# Install Rust and Cargo
+#RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+
+# Add Rust to PATH
+#ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install required tools
+RUN apt-get update && apt-get install -y bash g++ gcc make redis redis-tools
+
+# Create non-root user
+RUN useradd -m -u 1001 atc_api
+
+# Upgrade pip and certifi
+RUN python3 -m pip install --upgrade pip && pip install --upgrade certifi
+
+
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# Create working directory with correct ownership
+RUN mkdir /code && chown atc_api:atc_api /code
+WORKDIR /code
+
+# Install dependencies
+COPY --chown=atc_api:atc_api ./pe_reports_django_project/pe_reports_django/requirements.txt /code/
+RUN pip install --no-cache-dir -r requirements.txt
+# Copy the project code
+COPY --chown=atc_api:atc_api pe_reports_django_project /code
+
+# Switch to non-root user
+USER atc_api
+
+# Set Django environment variable
+ENV DJANGO_SETTINGS_MODULE=pe_reports_django.settings
+
+# Run the application
+CMD uvicorn --workers 4 pe_reports_django.asgi:app1 --host 0.0.0.0 --port 8000 --reload
+

src/pe_reports/__init__.py

@@ -1,105 +1,3 @@
 """The pe_reports library."""
-# We disable a Flake8 check for "Module imported but unused (F401)" here because
-# although this import is not directly used, it populates the value
-# package_name.__version__, which is used to get version information about this
-# Python package.
-
-# Standard Python Libraries
-import logging
-from logging.handlers import RotatingFileHandler
-import os
-
-# Third-Party Libraries
-# from celery import Celery
-from flask import Flask, render_template
-from flask_login import LoginManager
-from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy
-
-# cisagov Libraries
-from pe_reports.data.config import config
-
-from ._version import __version__  # noqa: F401
-
-# Stakeholder views
-# from pe_reports.home.views import home_blueprint
-# from pe_reports.report_gen.views import report_gen_blueprint
-# from pe_reports.stakeholder.views import stakeholder_blueprint
-# from pe_reports.stakeholder_bulk_upload.views import stakeholder_bulk_upload_blueprint
-# from pe_reports.stakeholder_full.views import stakeholder_full_blueprint
-
-
-params = config()
-login_manager = LoginManager()
-# Flask implementation
-app = Flask(__name__)
-app.config["SECRET_KEY"] = os.getenv("FLASK_SECRET_KEY", "dev")
-app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-app.config[
-    "SQLALCHEMY_DATABASE_URI"
-] = f'postgresql+psycopg2://{params["user"]}:{params["password"]}@{params["host"]}:{params["port"]}/{params["database"]}'
-
-
-# Configure the redis server
-# app.config["CELERY_BROKER_URL"] = "redis://localhost:6379/0"
-# app.config["CELERY_RESULT_BACKEND"] = "redis://localhost:6379/0"
-app.config["UPLOAD_FOLDER"] = "src/pe_reports/uploads/"
-app.config["ALLOWED_EXTENSIONS"] = {"txt", "csv"}
 
 CENTRAL_LOGGING_FILE = "pe_reports_logging.log"
-DEBUG = False
-# Setup Logging
-"""Set up logging and call the run_pe_script function."""
-if DEBUG is True:
-    level = "DEBUG"
-else:
-    level = "INFO"
-
-# Logging will rotate at 2GB
-logging.basicConfig(
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="%m/%d/%Y %I:%M:%S",
-    level=level,
-    handlers=[
-        RotatingFileHandler(CENTRAL_LOGGING_FILE, maxBytes=2000000, backupCount=10)
-    ],
-)
-
-app.config["LOGGER"] = logging.getLogger(__name__)
-
-# with open('username.txt', 'w') as file:
-#         file.write(pwd.getpwuid(os.getuid())[0])
-
-# Creates a Celery object
-# celery = Celery(app.name, broker=app.config["CELERY_BROKER_URL"])
-# celery.conf.update(app.config)
-
-# Config DB
-db = SQLAlchemy(app)
-Migrate(app, db)
-
-# TODO: Add a login page in the future. Issue #207 contains details
-# login_manager.init_app(app)
-# login_manager.login_view = "login"
-
-__all__ = ["app", "pages", "report_generator", "stylesheet"]
-
-
-# Register the flask apps
-# app.register_blueprint(stakeholder_blueprint)
-# app.register_blueprint(stakeholder_full_blueprint)
-# app.register_blueprint(stakeholder_bulk_upload_blueprint)
-# app.register_blueprint(report_gen_blueprint)
-# TODO: Add login blueprint. Issue #207 contains details
-# app.register_blueprint(manage_login_blueprint)
-# app.register_blueprint(home_blueprint)
-
-
-@app.errorhandler(404)
-def page_not_found(e):
-    return render_template("404.html")
-
-
-if __name__ == "__main__":
-    logging.info("The program has started...")
-    app.run(host="127.0.0.1", debug=DEBUG, port=8000)

src/pe_reports/data/db_query.py

@@ -62,7 +62,7 @@ def task_api_call(task_url, check_url, data={}, retry_time=3):
             # Ping task status endpoint and get status
             # check_task_resp = requests.get(check_task_url, headers=headers).json()
             check_task_resp = requests.get(check_task_url, headers=headers)
-            #print(check_task_resp)
+            # print(check_task_resp)
             check_task_resp = check_task_resp.json()
             task_status = check_task_resp.get("status")
             LOGGER.info(
@@ -2200,7 +2200,7 @@ def query_previous_period(org_uid, prev_end_date):
     return assets_dict
 
 
-#  ---------- PE-Score API Queries, Issue 635 ---------- 
+#  ---------- PE-Score API Queries, Issue 635 ----------
 # --- Issue 635 ---
 def pescore_hist_domain_alert(start_date, end_date):
     """
@@ -3609,7 +3609,7 @@ def upsert_new_cves_tsql(new_cves):
 
 
 # --- 018 atc-framework OLD TSQL ---
-def get_demo_orgs(conn):
+def get_demo_orgs_tsql(conn):
     """Query organizations table for orgs we report on."""
     try:
         cur = conn.cursor()
@@ -3623,4 +3623,4 @@ def get_demo_orgs(conn):
         LOGGER.error("There was a problem with your database query %s", error)
     finally:
         if conn is not None:
-            close(conn)
+            close(conn)

src/pe_reports/docker-compose.yaml

@@ -1,16 +1,15 @@
----
-version: "3.8"
+version: "3.9"
 services:
-  pe_reports_rabbitmq:
-    container_name: pe_reports_rabbitmq
-    hostname: pe_reports_rabbitmq
+  atc_rabbitmq:
+    container_name: atc_rabbitmq
+    hostname: atc_rabbitmq
     image: rabbitmq:3.8.14-management
     restart: always
     ports:
-      - 15672:15672
-      - 5672:5672
+      - 15674:15672
+      - 5674:5672
     env_file:
-      - ../../src/pe_reports/pe_reports_django_project/.env
+      - ./pe_reports_django_project/.env
     environment:
       - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER}
       - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASS}
@@ -24,37 +23,70 @@
       - LANGUAGE=C.UTF-8
       - LC_ALL=C.UTF-8
     volumes:
-      - /home/ubuntu/pe-reports/src/pe_reports/data/rabbitmq/data:/var/lib/rabbitmq/mnesia:rw
-      - /var/www/pe-reports/src/pe_reports/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
+      - ./pe_reports_django_project/data/rabbitmq/data:/var/lib/rabbitmq/mnesia:rw
+      - ./pe_reports_django_project/config:/etc/rabbitmq:rw
     networks:
-      - pe_reports_rabbitmq_network
+      - atc_network
 
-  pe_reports_redis:
-    container_name: pe_reports_redis
-    hostname: pe_reports_redis
-    image: redis:latest
+  atc_redis:
+    container_name: atc_redis
+    hostname: atc_redis
+    image: redis:7.4.1
     restart: always
     ports:
-      - 6379:6379
+      - 6378:6379
     volumes:
-      - redis_data:/data
+      - ./pe_reports_django_project/redis_data:/data
     networks:
-      - pe_reports_redis_network
+      - atc_network
+
+  web:
+    build: .
+    container_name: atc_web
+    volumes:
+      - ./pe_reports_django_project:/code
+      - ./pe_reports_django_project/config:/code/config
+    ports:
+      - "8002:8000"
+    env_file:
+      - pe_reports_django_project/.env
+    environment:
+      - DJANGO_SETTINGS_MODULE:pe_reports_django.settings
+    networks:
+      - atc_network
+
+
 
   nginx:
     image: nginx:1.25.0
+    container_name: atc_nginx
     ports:
-      - "8089:8089"
+      - "8091:8091"
     volumes:
-      - ./config/nginx_config_conf.d:/etc/nginx/conf.d
-      - ./pe_reports_django_project/static:/var/www/pe-reports/static
-      - ./pe-reports:/var/www/pe-reports
+      - ./pe_reports_django_project/config/nginx_config_conf.d:/etc/nginx/conf.d
     networks:
-      - pe_reports_nginx_network
+      - atc_network
+    depends_on:
+      - web
+
+  # database:
+  #   image: postgres
+  #   restart: always
+  #   env_file:
+  #     - ./pe_reports_django_project/.env
+  #   networks:
+  #     - atc_network
+  #   volumes:
+  #     - ./pe_reports_django_project/postgres_data:/var/lib/postgresql/data
+  #   ports:
+  #     - 5437:5432
+  #   container_name: atc_database
 
 networks:
-  pe_reports_network:
+  atc_network:
     driver: bridge
 
+
+
 volumes:
-  redis_data: {}
+  redis_data: {}

src/pe_reports/pe_reports_django_project/.env

@@ -5,6 +5,13 @@ password=
 host=
 port=
 
+# Mini Data Lake
+mdl_host=
+mdl_database=
+mdl_user=
+mdl_password=
+mdl_port=
+
 # The following key is for PE Service
 API_KEY=
 USER_REFRESH_TOKEN=

src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf

@@ -0,0 +1,12 @@
+server {
+    listen 8091;
+    server_name localhost;
+
+    location / {
+        proxy_pass http://web:8000; # Assuming 'web' is the service name and '8000' is the port where Gunicorn runs
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}

src/pe_reports/pe_reports_django_project/config/rabbitmq.conf

@@ -0,0 +1,4 @@
+loopback_users.guest = false
+listeners.tcp.default = 5672
+default_pass = guest1
+default_user = admin

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags

@@ -0,0 +1,2 @@
+[implicit_default_bindings,maintenance_mode_status,quorum_queue,user_limits,
+ virtual_host_metadata].

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid

@@ -0,0 +1 @@
+465

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config

@@ -0,0 +1 @@
+{[rabbit@atc_rabbitmq],[rabbit@atc_rabbitmq]}.

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost

@@ -0,0 +1 @@
+/

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/nodes_running_at_shutdown

@@ -0,0 +1 @@
+[rabbit@atc_rabbitmq].

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD

@@ -0,0 +1 @@
+
cXM

src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_serial

@@ -0,0 +1 @@
+26.