From 057fb574ed82db6964aa59aad895e56d90649f95 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:52:17 -0400 Subject: [PATCH 1/7] Add mini data lake app and models Add mini data lake app and models and router to point models too correct database --- src/pe_reports/pe_reports_django_project/.env | 7 + .../pe_reports_django_project/db_routers.py | 32 + .../dmz_mini_dl/__init__.py | 0 .../dmz_mini_dl/admin.py | 3 + .../dmz_mini_dl/apps.py | 6 + .../dmz_mini_dl/migrations/__init__.py | 0 .../dmz_mini_dl/models.py | 3735 +++++++++++++++++ .../dmz_mini_dl/tests.py | 3 + .../dmz_mini_dl/views.py | 3 + .../pe_reports_django/settings.py | 11 + 10 files changed, 3800 insertions(+) create mode 100644 src/pe_reports/pe_reports_django_project/db_routers.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/__init__.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/migrations/__init__.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py create mode 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py diff --git a/src/pe_reports/pe_reports_django_project/.env b/src/pe_reports/pe_reports_django_project/.env index 65d40d47..b46d7f3a 100644 --- a/src/pe_reports/pe_reports_django_project/.env +++ b/src/pe_reports/pe_reports_django_project/.env @@ -5,6 +5,13 @@ password= host= port= +# Mini Data Lake +mdl_host= +mdl_database= +mdl_user= +mdl_password= +mdl_port= + # The following key is for PE Service API_KEY= USER_REFRESH_TOKEN= diff --git a/src/pe_reports/pe_reports_django_project/db_routers.py b/src/pe_reports/pe_reports_django_project/db_routers.py new file mode 100644 index 00000000..e0a9bc66 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/db_routers.py @@ -0,0 +1,32 @@ +class MyAppRouter: + def db_for_read(self, model, **hints): + # Specify the app you want to route to the mini_data_lake database + if model._meta.app_label == 'dmz_mini_dl': + return 'mini_data_lake' + return 'default' # All other models go to the default database + + def db_for_write(self, model, **hints): + if model._meta.app_label == 'dmz_mini_dl': + return 'mini_data_lake' + return 'default' # All other models go to the default database + + def allow_relation(self, obj1, obj2, **hints): + # Check the app labels of both objects + app_label1 = obj1._meta.app_label + app_label2 = obj2._meta.app_label + + # If both objects are from the specific app, allow the relation + if app_label1 == 'dmz_mini_dl' and app_label2 == 'dmz_mini_dl': + return True + + # If only one of them is from the specific app, disallow the relation + if app_label1 == 'dmz_mini_dl' or app_label2 == 'dmz_mini_dl': + return False + + # Allow relations between all other models + return True + + def allow_migrate(self, db, app_label, model_name=None, **hints): + if app_label == 'dmz_mini_dl': + return db == 'mini_data_lake' # Migrate the specific app to the mini_data_lake database + return db == 'default' # All other apps migrate to the default database \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/__init__.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py new file mode 100644 index 00000000..781a3893 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DmzMiniDlConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "dmz_mini_dl" diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/migrations/__init__.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py new file mode 100644 index 00000000..8c67758d --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py @@ -0,0 +1,3735 @@ +# Create your models here. +""" Django ORM models """ + +from django.db import models +from django.contrib.postgres.fields import ArrayField +from django.contrib.auth.models import User as AuthUser +from netfields import InetAddressField, NetManager +import uuid + +manage_db = True +class ApiKey(models.Model): + """The ApiKey model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(auto_now_add=True, db_column="created_at") + updated_at = models.DateTimeField(auto_now=True, db_column="updated_at") + last_used = models.DateTimeField(db_column="last_used", blank=True, null=True) + hashed_key = models.TextField(db_column="hashed_key") + last_four = models.TextField(db_column="last_four") + user = models.ForeignKey( + "User", models.CASCADE, db_column="user_id", blank=True, null=True + ) + + class Meta: + """Meta class for ApiKey.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "api_key" + + +class Assessment(models.Model): + """The Assessment model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + rsc_id = models.CharField(db_column="rsc_id", unique=True, max_length=255) + type = models.CharField(max_length=255) + user = models.ForeignKey( + "User", db_column="user_id", blank=True, null=True, on_delete=models.CASCADE + ) + + class Meta: + """The Meta class for Assessment.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "assessment" + + +class Category(models.Model): + """The Category model.""" + + id = models.UUIDField(primary_key=True) + name = models.CharField(max_length=255) + number = models.CharField(max_length=255, unique=True) + short_name = models.CharField( + db_column="short_name", max_length=255, blank=True, null=True + ) + + class Meta: + """The Meta class for Category model.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "category" + + +class Cpe(models.Model): + """The Cpe model.""" + + id = models.UUIDField(primary_key=True) + name = models.CharField(max_length=255) + version = models.CharField(max_length=255) + vendor = models.CharField(max_length=255) + last_seen_at = models.DateTimeField(db_column="last_seen_at") + + class Meta: + """The Meta class for Cpe.""" + + app_label = 'dmz_mini_dl' + db_table = "cpe" + managed = manage_db # This ensures Django does not manage the table + unique_together = (("name", "version", "vendor"),) # Unique constraint + + +class Cve(models.Model): + """The Cve model.""" + + id = models.UUIDField(primary_key=True) + name = models.CharField(unique=True, blank=True, null=True, max_length=255) + published_at = models.DateTimeField(db_column="published_at", blank=True, null=True) + modified_at = models.DateTimeField(db_column="modified_at", blank=True, null=True) + status = models.CharField(blank=True, null=True, max_length=255) + description = models.TextField(blank=True, null=True) + cvss_v2_source = models.CharField(db_column="cvss_v2_source", blank=True, null=True, max_length=255) + cvss_v2_type = models.CharField(db_column="cvss_v2_type", blank=True, null=True, max_length=255) + cvss_v2_version = models.CharField(db_column="cvss_v2_version", blank=True, null=True, max_length=255) + cvss_v2_vector_string = models.CharField( + db_column="cvss_v2_vector_string", blank=True, null=True, max_length=255 + ) + cvss_v2_base_score = models.CharField( + db_column="cvss_v2_base_score", blank=True, null=True, max_length=255 + ) + cvss_v2_base_severity = models.CharField( + db_column="cvss_v2_base_severity", blank=True, null=True, max_length=255 + ) + cvss_v2_exploitability_score = models.CharField( + db_column="cvss_v2_exploitability_score", blank=True, null=True, max_length=255 + ) + cvss_v2_impact_score = models.CharField( + db_column="cvss_v2_impact_score", blank=True, null=True, max_length=255 + ) + cvss_v3_source = models.CharField(db_column="cvss_v3_source", blank=True, null=True, max_length=255) + cvss_v3_type = models.CharField(db_column="cvss_v3_type", blank=True, null=True, max_length=255) + cvss_v3_version = models.CharField(db_column="cvss_v3_version", blank=True, null=True, max_length=255) + cvss_v3_vector_string = models.CharField( + db_column="cvss_v3_vector_string", blank=True, null=True, max_length=255 + ) + cvss_v3_base_score = models.CharField( + db_column="cvss_v3_base_score", blank=True, null=True, max_length=255 + ) + cvss_v3_base_severity = models.CharField( + db_column="cvss_v3_base_severity", blank=True, null=True, max_length=255 + ) + cvss_v3_exploitability_score = models.CharField( + db_column="cvss_v3_exploitability_score", blank=True, null=True, max_length=255 + ) + cvss_v3_impact_score = models.CharField( + db_column="cvss_v3_impact_score", blank=True, null=True, max_length=255 + ) + cvss_v4_source = models.CharField(db_column="cvss_v4_source", blank=True, null=True, max_length=255) + cvss_v4_type = models.CharField(db_column="cvss_v4_type", blank=True, null=True, max_length=255) + cvss_v4_version = models.CharField(db_column="cvss_v4_version", blank=True, null=True, max_length=255) + cvss_v4_vector_string = models.CharField( + db_column="cvss_v4_vector_string", blank=True, null=True, max_length=255 + ) + cvss_v4_base_score = models.CharField( + db_column="cvss_v4_base_score", blank=True, null=True, max_length=255 + ) + cvss_v4_base_severity = models.CharField( + db_column="cvss_v4_base_severity", blank=True, null=True, max_length=255 + ) + cvss_v4_exploitability_score = models.CharField( + db_column="cvss_v4_exploitability_score", blank=True, null=True, max_length=255 + ) + cvss_v4_impact_score = models.CharField( + db_column="cvss_v4_impact_score", blank=True, null=True, max_length=255 + ) + weaknesses = models.TextField(blank=True, null=True) + references = models.TextField(blank=True, null=True) + dve_score = models.DecimalField( + max_digits=1000, decimal_places=1000, blank=True, null=True + ) + + cpes = models.ManyToManyField(Cpe, related_name='cves', blank=True) + # tickets = models.ManyToManyField("Ticket", related_name='cves', blank=True) + # vuln_scans = models.ManyToManyField("VulnScan", related_name='cves', blank=True) + + + class Meta: + """The Meta class for Cve.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cve" + + +# This will likely be handled via the many to many field +# class CveCpesCpe(models.Model): +# """The CveCpesCpe model.""" + +# cve_id = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column="cve_id") +# cpe_id = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column="cpe_id") + +# class Meta: +# """The Meta class for CveCpesCpe model.""" + +# db_table = "cve_cpes_cpe" +# managed = False # This ensures Django does not manage the table +# unique_together = (("cve", "cpe"),) # Unique constraint + +# This is crossfeeds domain model, which lines up better with the pe subdomain table +# class Domain(models.Model): +# """The Domain model.""" + +# id = models.UUIDField(primary_key=True) +# created_at = models.DateTimeField(db_column="created_at") +# updated_at = models.DateTimeField(db_column="updated_at") +# synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True) +# ip = models.CharField(max_length=255, blank=True, null=True) +# from_root_domain = models.CharField(db_column="from_root_domain", blank=True, null=True) +# subdomain_source = models.CharField( +# db_column="subdomain_source", max_length=255, blank=True, null=True +# ) +# ip_only = models.BooleanField(db_column="ip_only", default=False) +# reverse_name = models.CharField(db_column="reverse_name", max_length=512) +# name = models.CharField(max_length=512) +# screenshot = models.CharField(max_length=512, blank=True, null=True) +# country = models.CharField(max_length=255, blank=True, null=True) +# asn = models.CharField(max_length=255, blank=True, null=True) +# cloud_hosted = models.BooleanField(db_column="cloud_hosted", default=False) +# ssl = models.JSONField(blank=True, null=True) +# censys_certificates_results = models.JSONField( +# db_column="censys_certificates_results", default=dict +# ) +# trustymail_results = models.JSONField(db_column="trustymail_results", default=dict) +# discovered_by = models.ForeignKey( +# "Scan", +# on_delete=models.SET_NULL, +# db_column="discovered_by_id", +# blank=True, +# null=True, +# ) +# organization = models.ForeignKey( +# "Organization", on_delete=models.CASCADE, db_column="organization_id" +# ) + +# class Meta: +# """The meta class for Domain.""" + +# db_table = "domain" +# managed = False # This ensures Django does not manage the table +# unique_together = (("name", "organization"),) # Unique constraint + + def save(self, *args, **kwargs): + self.name = self.name.lower() + self.reverseName = ".".join(reversed(self.name.split("."))) + super().save(*args, **kwargs) + + +class Notification(models.Model): + """The Notification model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + start_datetime = models.DateTimeField( + db_column="start_datetime", blank=True, null=True + ) + end_datetime = models.DateTimeField(db_column="end_datetime", blank=True, null=True) + maintenance_type = models.CharField( + db_column="maintenance_type", blank=True, null=True, max_length=255 + ) + status = models.CharField(blank=True, null=True, max_length=255) + updated_by = models.CharField(db_column="updated_by", blank=True, null=True, max_length=255) + message = models.TextField(blank=True, null=True) + + class Meta: + """The Meta class for Notification.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "notification" + + +class Organization(models.Model): + """The Organization model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at", auto_now_add=True) + updated_at = models.DateTimeField(db_column="updated_at", auto_now=True) + acronym = models.CharField(unique=True, blank=True, null=True, max_length=255) + retired = models.BooleanField(default=False, null=True, blank=True) + name = models.CharField(max_length=255) + root_domains = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True, db_column="root_domains" + ) + ip_blocks = models.TextField(db_column="ip_blocks") # This field type is a guess. + is_passive = models.BooleanField(db_column="is_passive") + pending_domains = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True, db_column="pending_domains" + ) # This field type is a guess + date_pe_first_reported = models.DateTimeField(blank=True, null=True) + country = models.TextField(blank=True, null=True) + country_name = models.TextField(blank=True, null=True) + state = models.CharField(blank=True, null=True, max_length=255) + region_id = models.CharField(db_column="region_id", blank=True, null=True, max_length=255) + state_fips = models.IntegerField(db_column="state_fips", blank=True, null=True) + state_name = models.CharField(db_column="state_name", blank=True, null=True, max_length=255) + county = models.TextField(blank=True, null=True) + county_fips = models.IntegerField(db_column="county_fips", blank=True, null=True) + type = models.CharField(blank=True, null=True, max_length=255) + pe_report_on = models.BooleanField(default=False, null=True, blank=True) + pe_premium = models.BooleanField(default=False, null=True, blank=True) + pe_demo = models.BooleanField(default=False, null=True, blank=True) + agency_type = models.TextField(blank=True, null=True) + is_parent = models.BooleanField(blank=True, null=True) + pe_run_scans = models.BooleanField(default=False, null=True, blank=True) + stakeholder = models.BooleanField(default=False, null=True, blank=True) + election = models.BooleanField(blank=True, null=True) + was_stakeholder = models.BooleanField(default=False, null=True, blank=True) + vs_stakeholder = models.BooleanField(default=False, null=True, blank=True) + pe_stakeholder = models.BooleanField(default=False, null=True, blank=True) + receives_cyhy_report = models.BooleanField(blank=True, null=True) + receives_bod_report = models.BooleanField(blank=True, null=True) + receives_cybex_report = models.BooleanField(blank=True, null=True) + init_stage = models.CharField(max_length=255, null=True, blank=True) + scheduler = models.CharField(max_length=255, null=True, blank=True) + enrolled_in_vs_timestamp = models.DateTimeField(db_column="enrolled_in_vs_timestamp", auto_now=True) + period_start_vs_timestamp = models.DateTimeField(db_column="period_start_vs_timestamp", auto_now=True) + report_types = models.JSONField(null=True, blank=True, default=list) + scan_types = models.JSONField(null=True, blank=True, default=list) + scan_windows = models.JSONField(null=True, blank=True, default=list) + scan_limits = models.JSONField(null=True, blank=True, default=list) + password = models.TextField(blank=True, null=True) + cyhy_period_start = models.DateField(blank=True, null=True) + location = models.ForeignKey("Location", related_name='organizations', on_delete=models.SET_NULL, null=True, blank=True) + # sectors = models.ManyToManyField("Sector", related_name='organizations', blank=True) covered in sectors table already + # cidrs = models.ManyToManyField("Cidr", related_name='organizations', blank=True) covered in the cidr table already + # vuln_scans = models.ManyToManyField("VulnScan", related_name='organizations', blank=True) + # hosts = models.ManyToManyField("Host", related_name='organizations', blank=True) covered in hosts table already + # port_scans = models.ManyToManyField("PortScan", related_name='organizations', blank=True) + parent = models.ForeignKey( + "self", models.DO_NOTHING, db_column="parent_id", blank=True, null=True + ) + created_by = models.ForeignKey( + "User", models.DO_NOTHING, db_column="created_by_id", blank=True, null=True + ) + org_type = models.ForeignKey( + "OrgType", + on_delete=models.CASCADE, + db_column="org_type_id", + blank=True, + null=True, + ) + class Meta: + """The meta class for Organization.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "organization" + + +class OrganizationTag(models.Model): + """The OrganizationTag model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + name = models.CharField(unique=True, max_length=255) + organization = models.ManyToManyField("Organization", related_name='organization_tags', blank=True) + class Meta: + """The Meta class for OrganizationTag.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "organization_tag" + +# Probably can be removed and merged with a many to many relationship +# class OrganizationTagOrganizationsOrganization(models.Model): +# """The OrganizationTagOrganizationsOrganization model.""" + +# organization_tag_id = models.OneToOneField( +# OrganizationTag, +# models.DO_NOTHING, +# db_column="organizationTagId", +# primary_key=True, +# ) # The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. +# organization_id = models.ForeignKey( +# Organization, models.DO_NOTHING, db_column="organizationId" +# ) + +# class Meta: +# """The Meta class for OrganizationTagOrganizationsOrganization.""" + +# managed = False +# db_table = "organization_tag_organizations_organization" +# unique_together = (("organizationTagId", "organizationId"),) + + +class QueryResultCache(models.Model): + """The QueryResultCache model.""" + id = models.UUIDField(primary_key=True) + identifier = models.CharField(blank=True, null=True, max_length=255) + time = models.BigIntegerField() + duration = models.IntegerField() + query = models.TextField() + result = models.TextField() + + class Meta: + """The Meta class for QueryResultCache.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "query-result-cache" + + +class Question(models.Model): + """The Question model.""" + + id = models.UUIDField(primary_key=True) + name = models.CharField(max_length=255) + description = models.TextField(blank=True, null=True) + long_form = models.TextField(db_column="long_form") + number = models.CharField(max_length=255) + category = models.ForeignKey( + Category, models.DO_NOTHING, db_column="category_id", blank=True, null=True + ) + + class Meta: + """The Meta class for Question.""" + + db_table = "question" + app_label = 'dmz_mini_dl' + managed = manage_db + unique_together = (("category", "number"),) + +# Created via Many to Many field +# Question and Resource many-to-many +# class QuestionResourcesResource(models.Model): +# """The QuestionResourcesResource model.""" + +# question_id = models.ForeignKey( +# "Question", on_delete=models.CASCADE, db_column="questionId" +# ) +# resource_id = models.ForeignKey( +# "Resource", on_delete=models.CASCADE, db_column="resourceId" +# ) + +# class Meta: +# """The Meta class for QuestionResourcesResource.""" + +# db_table = "question_resources_resource" +# managed = False +# unique_together = (("question", "resource"),) + + +class Resource(models.Model): + """The Resource model.""" + + id = models.UUIDField(primary_key=True) + description = models.TextField() + name = models.CharField(max_length=255) + type = models.CharField(max_length=255) + url = models.TextField(unique=True) + questions = models.ManyToManyField(Question, related_name='resources', blank=True) + + class Meta: + """The Meta class for Resource.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "resource" + + +class Response(models.Model): + """The Response model.""" + + id = models.UUIDField(primary_key=True) + selection = models.TextField() + assessment = models.ForeignKey( + Assessment, models.DO_NOTHING, db_column="assessment_id", blank=True, null=True + ) + question = models.ForeignKey( + Question, models.DO_NOTHING, db_column="question_id", blank=True, null=True + ) + + class Meta: + """The Meta class for Resource.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "response" + unique_together = (("assessment_id", "question_id"),) + + +class Role(models.Model): + """The Role model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + role = models.CharField(max_length=255) + approved = models.BooleanField() + created_by = models.ForeignKey( + "User", models.DO_NOTHING, db_column="created_by_id", blank=True, null=True + ) + approved_by = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="approved_by_id", + related_name="role_approved_by_id_set", + blank=True, + null=True, + ) + user = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="user_id", + related_name="role_user_id_set", + blank=True, + null=True, + ) + organization = models.ForeignKey( + Organization, + models.DO_NOTHING, + db_column="organization_id", + blank=True, + null=True, + ) + + class Meta: + """The Meta class for Role.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "role" + unique_together = (("user_id", "organization_id"),) + + +class SavedSearch(models.Model): + """The SavedSearch model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + name = models.CharField(max_length=255) + search_term = models.CharField(db_column="search_term", max_length=255) + sort_direction = models.CharField(db_column="sort_direction", max_length=255) + sort_field = models.CharField(db_column="sort_field", max_length=255) + count = models.IntegerField() + filters = models.JSONField() + search_path = models.CharField(db_column="search_path", max_length=255) + create_vulnerabilities = models.BooleanField(db_column="create_vulnerabilities") + vulnerability_template = models.JSONField(db_column="vulnerability_template") + created_by = models.ForeignKey( + "User", models.DO_NOTHING, db_column="created_by_id", blank=True, null=True + ) + + class Meta: + """The Meta class for SavedSearch.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "saved_search" + + +class Scan(models.Model): + """The Scan model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + name = models.CharField(max_length=255) + arguments = models.JSONField() + frequency = models.IntegerField() + last_run = models.DateTimeField(db_column="last_run", blank=True, null=True) + is_granular = models.BooleanField(db_column="is_granular") + is_user_modifiable = models.BooleanField( + db_column="is_user_modifiable", blank=True, null=True + ) + is_single_scan = models.BooleanField(db_column="is_single_scan") + manual_run_pending = models.BooleanField(db_column="manual_run_pending") + created_by = models.ForeignKey( + "User", models.DO_NOTHING, db_column="created_by", blank=True, null=True + ) + organizations = models.ManyToManyField(Organization, related_name="scans", blank=True) + organization_tags = models.ManyToManyField(OrganizationTag, related_name="scans", blank=True) + + class Meta: + """The Meta class for Scan.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "scan" + +# Taken Care of via many to many field +# class ScanOrganizationsOrganization(models.Model): +# """The ScanOrganizationsOrganization model.""" + +# scan_id = models.OneToOneField( +# Scan, models.DO_NOTHING, db_column="scanId", primary_key=True +# ) # The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. +# organization_id = models.ForeignKey( +# Organization, models.DO_NOTHING, db_column="organizationId" +# ) + +# class Meta: +# """The Meta class for ScanOrganizationsOrganization.""" + +# managed = False +# db_table = "scan_organizations_organization" +# unique_together = (("scanId", "organizationId"),) + +# Completed via many to many +# class ScanTagsOrganizationTag(models.Model): +# """The ScanTagsOrganizationTag model.""" + +# scan_id = models.OneToOneField( +# Scan, models.DO_NOTHING, db_column="scanId", primary_key=True +# ) # The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. +# organization_tag_id = models.ForeignKey( +# OrganizationTag, models.DO_NOTHING, db_column="organizationTagId" +# ) + +# class Meta: +# """The Meta class for ScanTagsOrganizationTag.""" + +# managed = False +# db_table = "scan_tags_organization_tag" +# unique_together = (("scanId", "organizationTagId"),) + + +class ScanTask(models.Model): + """The ScanTask model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + status = models.TextField() + type = models.TextField() + fargate_task_arn = models.TextField(db_column="fargate_task_arn", blank=True, null=True) + input = models.TextField(blank=True, null=True) + output = models.TextField(blank=True, null=True) + requested_at = models.DateTimeField(db_column="requested_at", blank=True, null=True) + started_at = models.DateTimeField(db_column="started_at", blank=True, null=True) + finished_at = models.DateTimeField(db_column="finished_at", blank=True, null=True) + queued_at = models.DateTimeField(db_column="queued_at", blank=True, null=True) + organization = models.ForeignKey( + Organization, + models.DO_NOTHING, + db_column="organization_id", + blank=True, + null=True, + ) + scan = models.ForeignKey( + Scan, models.DO_NOTHING, db_column="scan_id", blank=True, null=True + ) + organization_tags = models.ManyToManyField(OrganizationTag, related_name="scan_tasks", blank=True) + + class Meta: + """The Meta class for ScanTask.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "scan_task" + +# Managed via many to many +# class ScanTaskOrganizationsOrganization(models.Model): +# """The ScanTaskOrganizationsOrganization model.""" + +# scan_task_id = models.OneToOneField( +# ScanTask, models.DO_NOTHING, db_column="scanTaskId", primary_key=True +# ) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. +# organization_id = models.ForeignKey( +# Organization, models.DO_NOTHING, db_column="organizationId" +# ) + +# class Meta: +# """The Meta class for ScanTaskOrganizationsOrganization.""" + +# managed = False +# db_table = "scan_task_organizations_organization" +# unique_together = (("scanTaskId", "organizationId"),) + + +class Service(models.Model): + """The Service model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + service_source = models.TextField(db_column="service_source", blank=True, null=True) + port = models.IntegerField() + service = models.TextField(blank=True, null=True) + last_seen = models.DateTimeField(db_column="last_seen", blank=True, null=True) + banner = models.TextField(blank=True, null=True) + products = models.JSONField() + censys_metadata = models.JSONField(db_column="censys_metadata") + censys_ipv4_results = models.JSONField(db_column="censys_ipv4_results") + intrigue_ident_results = models.JSONField(db_column="intrigue_ident_results") + shodan_results = models.JSONField(db_column="shodan_results") + wappalyzer_results = models.JSONField(db_column="wappalyzer_results") + domain = models.ForeignKey( + "SubDomains", models.DO_NOTHING, db_column="domain_id", blank=True, null=True + ) + discovered_by = models.ForeignKey( + Scan, models.DO_NOTHING, db_column="discovered_by_id", blank=True, null=True + ) + + class Meta: + """The Meta class for Service.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "service" + unique_together = (("port", "domain"),) + +# ????Not sure if this is necessary since we are removing typeorm????? +# class TypeormMetadata(models.Model): +# """The TypeormMetadata model.""" + +# type = models.CharField() +# database = models.CharField(blank=True, null=True) +# schema = models.CharField(blank=True, null=True) +# table = models.CharField(blank=True, null=True) +# name = models.CharField(blank=True, null=True) +# value = models.TextField(blank=True, null=True) + +# class Meta: +# """The Meta class for TypeormMetadata.""" + +# managed = False +# db_table = "typeorm_metadata" + + +class User(models.Model): + """The User model.""" + + id = models.UUIDField(primary_key=True) + cognito_id = models.CharField( + db_column="cognitoId", unique=True, blank=True, null=True, max_length=255 + ) + login_gov_id = models.CharField( + db_column="login_gov_id", unique=True, blank=True, null=True, max_length=255 + ) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + first_name = models.CharField(db_column="first_name", max_length=255) + last_name = models.CharField(db_column="last_name", max_length=255) + full_name = models.CharField(db_column="full_name", max_length=255) + email = models.CharField(unique=True, max_length=255) + invite_pending = models.BooleanField(db_column="invite_pending") + login_blocked_by_maintenance = models.BooleanField( + db_column="login_blocked_by_maintenance" + ) + date_accepted_terms = models.DateTimeField( + db_column="date_accepted_terms", blank=True, null=True + ) + accepted_terms_version = models.TextField( + db_column="accepted_terms_version", blank=True, null=True + ) + last_logged_in = models.DateTimeField(db_column="last_logged_in", blank=True, null=True) + user_type = models.TextField(db_column="user_type") + region_id = models.CharField(db_column="region_id", blank=True, null=True, max_length=255) + state = models.CharField(blank=True, null=True, max_length=255) + okta_id = models.CharField(db_column="okta_id", unique=True, blank=True, null=True, max_length=255) + + class Meta: + """The Meta class for User.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "user" + + +class Vulnerability(models.Model): + """The Vulnerability model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + last_seen = models.DateTimeField(db_column="last_seen", blank=True, null=True) + title = models.TextField() + cve = models.TextField(blank=True, null=True) + cwe = models.TextField(blank=True, null=True) + cpe = models.TextField(blank=True, null=True) + description = models.TextField() + references = models.JSONField() + cvss = models.DecimalField( + max_digits=100, decimal_places=5, blank=True, null=True + ) + severity = models.TextField(blank=True, null=True) + needs_population = models.BooleanField(db_column="needs_population") + state = models.TextField() + substate = models.TextField() + source = models.TextField() + notes = models.TextField() + actions = models.JSONField() + structured_data = models.JSONField(db_column="structured_data") + is_kev = models.BooleanField(db_column="is_kev", blank=True, null=True) + kev_results = models.JSONField(db_column="kev_results", blank=True, null=True) + domain = models.ForeignKey( + "SubDomains", models.DO_NOTHING, db_column="domain_id", blank=True, null=True + ) + service = models.ForeignKey( + Service, models.DO_NOTHING, db_column="service_id", blank=True, null=True + ) + + class Meta: + """The Meta class for Vulnerability.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "vulnerability" + unique_together = (("domain", "title"),) + + +class Webpage(models.Model): + """The Webpage model.""" + + id = models.UUIDField(primary_key=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True) + last_seen = models.DateTimeField(db_column="last_seen", blank=True, null=True) + s3key = models.TextField(db_column="s3Key", blank=True, null=True) + url = models.TextField() + status = models.DecimalField(max_digits=100, decimal_places=5) + response_size = models.DecimalField( + db_column="response_size", + max_digits=100, + decimal_places=5, + blank=True, + null=True, + ) + headers = models.JSONField() + domain = models.ForeignKey( + "SubDomains", models.DO_NOTHING, db_column="domain_id", blank=True, null=True + ) + discovered_by = models.ForeignKey( + Scan, models.DO_NOTHING, db_column="discovered_by_id", blank=True, null=True + ) + + class Meta: + """The Meta class for Webpage.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "webpage" + unique_together = (("url", "domain"),) + + +######### VS Models ######### +class TicketEvent(models.Model): + id = models.UUIDField(primary_key=True, editable=False) + reference = models.CharField(max_length=255, null=True, blank=True) + vuln_scan = models.ForeignKey("VulnScan", on_delete=models.CASCADE, db_column = "vuln_scan_id", null=True, blank=True, related_name='ticket_events') + action = models.CharField(max_length=255, null=True, blank=True) + reason = models.CharField(max_length=255, null=True, blank=True) + event_timestamp = models.DateTimeField(null=True, blank=True) + delta = models.JSONField(default=list) + ticket = models.ForeignKey("Ticket", on_delete=models.CASCADE, db_column = "ticket_id",null=True, blank=True, related_name='ticket_events') + + class Meta: + """The Meta class for TicketEvent.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "ticket_event" + unique_together = ('event_timestamp', 'ticket', 'action') + +class VulnScan(models.Model): + """The VS Vuln Scan model.""" + id = models.CharField(max_length=255, primary_key=True) + cert_id = models.CharField(max_length=255, blank=True, null=True) + cpe = models.CharField(max_length=255, blank=True, null=True) + cve_string = models.CharField(max_length=255, blank=True, null=True) + cve = models.ForeignKey(Cve, related_name='vuln_scans', on_delete=models.CASCADE, blank=True, null=True) + cvss_base_score = models.CharField(max_length=255, blank=True, null=True) + cvss_temporal_score = models.CharField(max_length=255, blank=True, null=True) + cvss_temporal_vector = models.CharField(max_length=255, blank=True, null=True) + cvss_vector = models.CharField(max_length=255, blank=True, null=True) + description = models.CharField(max_length=255, blank=True, null=True) + exploit_available = models.CharField(max_length=255, blank=True, null=True) + exploitability_ease = models.CharField(max_length=255, blank=True, null=True) + ip_string = models.CharField(max_length=255, blank=True, null=True) + ip = models.ForeignKey("Ip", related_name='vuln_scans', on_delete=models.CASCADE, blank=True, null=True) + latest = models.BooleanField(default=False) + owner = models.CharField(max_length=255, blank=True, null=True) + osvdb_id = models.CharField(max_length=255, blank=True, null=True) + organization = models.ForeignKey(Organization, related_name='vuln_scans', on_delete=models.CASCADE, blank=True, null=True) + patch_publication_timestamp = models.DateTimeField(blank=True, null=True) + cisa_known_exploited = models.DateTimeField(blank=True, null=True) + port = models.IntegerField(blank=True, null=True) + port_protocol = models.CharField(max_length=255, blank=True, null=True) + risk_factor = models.CharField(max_length=255, blank=True, null=True) + script_version = models.CharField(max_length=255, blank=True, null=True) + see_also = models.CharField(max_length=255, blank=True, null=True) + service = models.CharField(max_length=255, blank=True, null=True) + severity = models.IntegerField(blank=True, null=True) + solution = models.CharField(max_length=255, blank=True, null=True) + source = models.CharField(max_length=255, blank=True, null=True) + synopsis = models.CharField(max_length=255, blank=True, null=True) + vuln_detection_timestamp = models.DateTimeField(blank=True, null=True) + vuln_publication_timestamp = models.DateTimeField(blank=True, null=True) + xref = models.CharField(max_length=255, blank=True, null=True) + cwe = models.CharField(max_length=255, blank=True, null=True) + bid = models.CharField(max_length=255, blank=True, null=True) + exploited_by_malware = models.BooleanField(default=False) + thorough_tests = models.BooleanField(default=False) + cvss_score_rationale = models.CharField(max_length=255, blank=True, null=True) + cvss_score_source = models.CharField(max_length=255, blank=True, null=True) + cvss3_base_score = models.DecimalField(max_digits=5, decimal_places=2, blank=True, null=True) + cvss3_vector = models.CharField(max_length=255, blank=True, null=True) + cvss3_temporal_vector = models.CharField(max_length=255, blank=True, null=True) + cvss3_temporal_score = models.DecimalField(max_digits=5, decimal_places=2, blank=True, null=True) + asset_inventory = models.BooleanField(default=False) + plugin_id = models.CharField(max_length=255, blank=True, null=True) + plugin_modification_date = models.DateTimeField(blank=True, null=True) + plugin_publication_date = models.DateTimeField(blank=True, null=True) + plugin_name = models.CharField(max_length=255, blank=True, null=True) + plugin_type = models.CharField(max_length=255, blank=True, null=True) + plugin_family = models.CharField(max_length=255, blank=True, null=True) + f_name = models.CharField(max_length=255, blank=True, null=True) + cisco_bug_id = models.CharField(max_length=255, blank=True, null=True) + cisco_sa = models.CharField(max_length=255, blank=True, null=True) + plugin_output = models.TextField(blank=True, null=True) + # snapshots = models.ManyToManyField(Snapshot, related_name='vuln_scans') + # ticket_events = models.ManyToManyField(TicketEvent, related_name='vuln_scans') + other_findings = models.JSONField(default=dict, blank=True) + +class Meta: + """The Meta class for VulnScan.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "vuln_scan" + +class Cidr(models.Model): + id = models.UUIDField(primary_key=True, editable=False) + created_date = models.DateTimeField(auto_now_add=True) + network = InetAddressField(null=True, blank=True, unique=True) #models.TextField() # This field type is a guess. + start_ip = InetAddressField(null=True, blank=True) + end_ip = InetAddressField(null=True, blank=True) + retired = models.BooleanField(null=True, blank=True) + updated_at = models.DateTimeField(auto_now=True) + insert_alert = models.TextField(blank=True, null=True) + first_seen = models.DateField(blank=True, null=True) + last_seen = models.DateField(blank=True, null=True) + current = models.BooleanField(blank=True, null=True) + data_source = models.ForeignKey( + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, + ) + + organizations = models.ManyToManyField(Organization, related_name='cidrs', blank=True) + + class Meta: + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cidr" + indexes = [ + models.Index(fields=['network']) + ] + + + +class Location(models.Model): + id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) + name = models.CharField(max_length=255, null=True, blank=True) + country_abrv = models.CharField(max_length=255, null=True, blank=True) + country = models.CharField(max_length=255, null=True, blank=True) + county = models.CharField(max_length=255, null=True, blank=True) + county_fips = models.CharField(max_length=255, null=True, blank=True) + gnis_id = models.CharField(max_length=255, null=True, blank=True, unique=True) + state_abrv = models.CharField(max_length=255, null=True, blank=True) + state_fips = models.CharField(max_length=255, null=True, blank=True) + state = models.CharField(max_length=255, null=True, blank=True) + + class Meta: + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = 'location' + indexes = [ + models.Index(fields=['gnis_id']) + ] + +class Sector(models.Model): + id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) + name = models.CharField(max_length=255, null=True, blank=True) + acronym = models.CharField(max_length=255, null=True, blank=True, unique=True) + retired = models.BooleanField(null=True, blank=True) + + organizations = models.ManyToManyField(Organization, related_name='sectors', blank=True) + + class Meta: + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = 'sector' + indexes = [ + models.Index(fields=['acronym']) + ] + +class Host(models.Model): + id = models.CharField(max_length=255, primary_key=True) + ip_string = models.CharField(max_length=255, null=True, blank=True) + ip = models.ForeignKey("Ip", related_name='hosts', on_delete=models.SET_NULL, null=True, blank=True) + updated_timestamp = models.DateTimeField(null=True, blank=True) + latest_netscan_1_timestamp = models.DateTimeField(null=True, blank=True) + latest_netscan_2_timestamp = models.DateTimeField(null=True, blank=True) + latest_vulnscan_timestamp = models.DateTimeField(null=True, blank=True) + latest_portscan_timestamp = models.DateTimeField(null=True, blank=True) + latest_scan_completion_timestamp = models.DateTimeField(null=True, blank=True) + location_longitude = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) + location_latitude = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) + priority = models.IntegerField(null=True, blank=True) + next_scan_timestamp = models.DateTimeField(null=True, blank=True) + rand = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) + curr_stage = models.CharField(max_length=255, null=True, blank=True) + host_live = models.BooleanField(null=True, blank=True) + host_live_reason = models.CharField(max_length=255, null=True, blank=True) + status = models.CharField(max_length=255, null=True, blank=True) + organization = models.ForeignKey(Organization, related_name='hosts', on_delete=models.CASCADE, null=True, blank=True) + + class Meta: + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = 'host' + indexes = [ + models.Index(fields=['ip_string']), + ] + +class Ip(models.Model): + # id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) + ip_hash = models.TextField(primary_key=True) + organization = models.ForeignKey(Organization, related_name='ips', on_delete=models.CASCADE) + created_timestamp = models.DateTimeField(auto_now_add=True) + updated_timestamp = models.DateTimeField(null=True, blank=True, auto_now=True) + last_seen_timestamp = models.DateTimeField(null=True, blank=True) + ip = models.GenericIPAddressField(null=True, blank=True) + live = models.BooleanField(null=True, blank=True) + false_positive = models.BooleanField(null=True, blank=True) + from_cidr = models.BooleanField(null=True, blank=True) + retired = models.BooleanField(null=True, blank=True) + last_reverse_lookup = models.DateTimeField(blank=True, null=True) + from_cidr = models.BooleanField(blank=True, null=True) + + # domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) + # host_scans = models.ManyToManyField("HostScan", related_name='ips', blank=True) + # hosts = models.ManyToManyField(Host, related_name='ips', blank=True) + # tickets = models.ManyToManyField("Ticket", related_name='ips', blank=True) + # vuln_scans = models.ManyToManyField(VulnScan, related_name='ips', blank=True) + # port_scans = models.ManyToManyField("PortScan", related_name='ips', blank=True) + sub_domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) + has_shodan_results = models.BooleanField(blank=True, null=True) + origin_cidr = models.ForeignKey( + Cidr, on_delete=models.CASCADE, db_column="origin_cidr", blank=True, null=True + ) + current = models.BooleanField(blank=True, null=True) + + class Meta: + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = 'ip' + indexes = [ + models.Index(fields=['ip', 'organization']) + ] + unique_together = ['ip', 'organization'] + + +class Ticket(models.Model): + id = models.CharField(max_length=255, primary_key=True) # Assuming the UUID is represented as a string + cve_string = models.CharField(max_length=255, null=True, blank=True) + cve = models.ForeignKey(Cve, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + cvss_base_score = models.DecimalField(max_digits=5, decimal_places=2, null=True, blank=True) + cvss_version = models.CharField(max_length=255, null=True, blank=True) + # kev = models.ForeignKey(Kev, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + vuln_name = models.CharField(max_length=255, null=True, blank=True) + cvss_score_source = models.CharField(max_length=255, null=True, blank=True) + cvss_severity = models.DecimalField(max_digits=5, decimal_places=2, null=True, blank=True) + vpr_score = models.DecimalField(max_digits=5, decimal_places=2, null=True, blank=True) + false_positive = models.BooleanField(null=True, blank=True) + ip_string = models.CharField(max_length=255, null=True, blank=True) + ip = models.ForeignKey(Ip, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + updated_timestamp = models.DateTimeField(null=True, blank=True) + location_longitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True) + location_latitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True) + found_in_latest_host_scan = models.BooleanField(null=True, blank=True) + organization = models.ForeignKey(Organization, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + vuln_port = models.IntegerField(null=True, blank=True) + port_protocol = models.CharField(max_length=255, null=True, blank=True) + snapshots_bool = models.BooleanField(null=True, blank=True) + vuln_source = models.CharField(max_length=255, null=True, blank=True) + vuln_source_id = models.IntegerField(null=True, blank=True) + closed_timestamp = models.DateTimeField(null=True, blank=True) + opened_timestamp = models.DateTimeField(null=True, blank=True) + # snapshots = models.ManyToManyField(Snapshot, related_name='tickets', blank=True) + # ticket_events = models.ManyToManyField(TicketEvent, related_name='tickets', blank=True) + + class Meta: + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = 'ticket' + unique_together = ['id'] + +class PortScan(models.Model): + id = models.CharField(max_length=255, primary_key=True) # Assuming UUIDs are stored as strings + ip_string = models.CharField(max_length=255, null=True, blank=True) + ip = models.ForeignKey(Ip, related_name='port_scans', null=True, blank=True, on_delete=models.CASCADE) + latest = models.BooleanField(default=False) + port = models.IntegerField(null=True, blank=True) + protocol = models.CharField(max_length=255, null=True, blank=True) + reason = models.CharField(max_length=255, null=True, blank=True) + service = models.JSONField(default=dict) # Use JSONField to store JSON objects + service_name = models.CharField(max_length=255, null=True, blank=True) + service_confidence = models.IntegerField(null=True, blank=True) + service_method = models.CharField(max_length=255, null=True, blank=True) + source = models.CharField(max_length=255, null=True, blank=True) + state = models.CharField(max_length=255, null=True, blank=True) + time_scanned = models.DateTimeField(null=True, blank=True) + # snapshots = models.ManyToManyField(Snapshot, related_name='port_scans', blank=True) + organization = models.ForeignKey(Organization, related_name='port_scans', null=True, blank=True, on_delete=models.CASCADE) + + class Meta: + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = 'port_scan' +######### WAS Models ######### + +class WasTrackerCustomerdata(models.Model): + """Define WasTrackerCustomerdata model.""" + + customer_id = models.UUIDField( + db_column="customer_id", primary_key=True, default=uuid.uuid1 + ) + tag = models.TextField() + customer_name = models.TextField() + testing_sector = models.TextField() + ci_type = models.TextField() + jira_ticket = models.TextField() + ticket = models.TextField() + next_scheduled = models.TextField() + last_scanned = models.TextField() + frequency = models.TextField() + comments_notes = models.TextField() + was_report_poc = models.TextField() + was_report_email = models.TextField() + onboarding_date = models.TextField() + no_of_web_apps = models.IntegerField() + no_web_apps_last_updated = models.TextField(blank=True, null=True) + elections = models.BooleanField(blank=False, null=False) + fceb = models.BooleanField(blank=False, null=False) + special_report = models.BooleanField(blank=False, null=False) + report_password = models.TextField() + child_tags = models.TextField() + + class Meta: + """Set WasTrackerCustomerdata model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "was_tracker_customer_data" + +""" +-- WARNING: It may differ from actual native database DDL +CREATE TABLE information_schema.was_findings ( + finding_uid uuid NOT NULL, + finding_type varchar(10485760) NULL, + webapp_id int4 NULL, + was_org_id text NULL, + owasp_category varchar(10485760) NULL, + severity varchar(10485760) NULL, + times_detected int4 NULL, + base_score float8 NULL, + temporal_score float8 NULL, + fstatus varchar(10485760) NULL, + last_detected date NULL, + first_detected date NULL, + is_remediated bool NULL, + potential bool NULL, + webapp_url text NULL, + webapp_name text NULL, + "name" text NULL, + cvss_v3_attack_vector text NULL, + cwe_list _int4 NULL, + wasc_list jsonb NULL +); +""" +class WasFindings(models.Model): + """Define WasFindings model.""" + + finding_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + finding_type = models.TextField(blank=True, null=True) + webapp_id = models.IntegerField(blank=True, null=True) + was_org_id = models.TextField(blank=True, null=True) + owasp_category = models.TextField(blank=True, null=True) + severity = models.TextField(blank=True, null=True) + times_detected = models.IntegerField(blank=True, null=True) + base_score = models.FloatField(blank=True, null=True) + temporal_score = models.FloatField(blank=True, null=True) + fstatus = models.TextField(blank=True, null=True) + last_detected = models.DateField(blank=True, null=True) + first_detected = models.DateField(blank=True, null=True) + is_remediated = models.BooleanField(blank=True, null=True) + potential = models.BooleanField(blank=True, null=True) + webapp_url = models.TextField(blank=True, null=True) + webapp_name = models.TextField(blank=True, null=True) + name = models.TextField(blank=True, null=True) + cvss_v3_attack_vector = models.TextField(blank=True, null=True) + cwe_list = ArrayField( + models.IntegerField(blank=True, null=True), blank=True, null=True + ) + wasc_list = models.JSONField(blank=True, null=True) + last_tested = models.DateField(blank=True, null=True) + fixed_date = models.DateField(blank=True, null=True) + is_ignored = models.BooleanField(blank=True, null=True) + url = models.TextField(blank=True, null=True) + qid = models.IntegerField(blank=True, null=True) + response = models.TextField(blank=True, null=True) + class Meta: + """Set WasFindings model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "was_findings" + +class WasHistory(models.Model): + """Define WasHistory model.""" + was_org_id = models.TextField(blank=True, null=True) + date_scanned = models.DateField() + vuln_cnt = models.IntegerField(blank=True, null=True) + vuln_webapp_cnt = models.IntegerField(blank=True, null=True) + web_app_cnt = models.IntegerField(blank=True, null=True) + high_rem_time = models.IntegerField(blank=True, null=True) + crit_rem_time = models.IntegerField(blank=True, null=True) + crit_vuln_cnt = models.IntegerField(blank=True, null=True) + high_vuln_cnt = models.IntegerField(blank=True, null=True) + report_period = models.DateField(blank=True, null=True) + high_rem_cnt = models.IntegerField(blank=True, null=True) + crit_rem_cnt = models.IntegerField(blank=True, null=True) + total_potential = models.IntegerField(blank=True, null=True) + class Meta: + """Set WasHistory model metadata.""" + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "was_history" + unique_together = (('was_org_id', 'date_scanned'),) + + +class WasMap(models.Model): + """Define WasMap model.""" + was_org_id = models.TextField(blank=True, primary_key=True) + pe_org_id = models.UUIDField(blank=True, null=True) + report_on = models.BooleanField(blank=True, null=True) + last_scanned = models.DateField(blank=True, null=True) + class Meta: + """Set WasMap model metadata.""" + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "was_map" + + +class WasReport(models.Model): + org_name = models.TextField(blank=True, null=True) + date_pulled = models.DateTimeField(blank=True, null=True) + last_scan_date = models.DateTimeField(blank=True, null=True) + security_risk = models.TextField(blank=True, null=True) + total_info = models.IntegerField(blank=True, null=True) + num_apps = models.IntegerField(blank=True, null=True) + risk_color = models.TextField(blank=True, null=True) + sensitive_count = models.IntegerField(blank=True, null=True) + sensitive_color = models.TextField(blank=True, null=True) + max_days_open_urgent = models.IntegerField(blank=True, null=True) + max_days_open_critical = models.IntegerField(blank=True, null=True) + urgent_color = models.TextField(blank=True, null=True) + critical_color = models.TextField(blank=True, null=True) + org_was_acronym = models.TextField(blank=True, null=True) + name_len = models.TextField(blank=True, null=True) + vuln_csv_dict = models.JSONField(blank=True, null=True, default=dict) + ssn_cc_dict = models.JSONField(blank=True, null=True, default=dict) + app_overview_csv_dict = models.JSONField(blank=True, null=True, default=dict) + details_csv = models.JSONField(blank=True, null=True, default=list) + info_csv = models.JSONField(blank=True, null=True, default=list) + links_crawled = models.JSONField(blank=True, null=True, default=list) + links_rejected = models.JSONField(blank=True, null=True, default=list) + emails_found = models.JSONField(blank=True, null=True, default=list) + owasp_count_dict = models.JSONField(blank=True, null=True, default=dict) + group_count_dict = models.JSONField(blank=True, null=True, default=dict) + fixed = models.IntegerField(blank=True, null=True) + total = models.IntegerField(blank=True, null=True) + vulns_monthly_dict = models.JSONField(blank=True, null=True, default=dict) + path_disc = models.IntegerField(blank=True, null=True) + info_disc = models.IntegerField(blank=True, null=True) + cross_site = models.IntegerField(blank=True, null=True) + burp = models.IntegerField(blank=True, null=True) + sql_inj = models.IntegerField(blank=True, null=True) + bugcrowd = models.IntegerField(blank=True, null=True) + reopened = models.IntegerField(blank=True, null=True) + reopened_color = models.TextField(blank=True, null=True) + new_vulns = models.IntegerField(blank=True, null=True) + new_vulns_color = models.TextField(blank=True, null=True) + tot_vulns = models.IntegerField(blank=True, null=True) + tot_vulns_color = models.TextField(blank=True, null=True) + lev1 = models.IntegerField(blank=True, null=True) + lev2 = models.IntegerField(blank=True, null=True) + lev3 = models.IntegerField(blank=True, null=True) + lev4 = models.IntegerField(blank=True, null=True) + lev5 = models.IntegerField(blank=True, null=True) + severities = ArrayField(models.IntegerField(), blank=True, null=True, default=list) + ages = ArrayField(models.IntegerField(), blank=True, null=True, default=list) + pdf_obj = models.BinaryField(blank=True, null=True) + + class Meta: + db_table = 'was_report' + unique_together = ('last_scan_date', 'org_was_acronym') + app_label = 'dmz_mini_dl' + managed = manage_db + + +######### PE Models ######### +class PeUsers(models.Model): + """Define Users model.""" + + id = models.UUIDField(primary_key=True) + email = models.CharField(unique=True, max_length=64, blank=True, null=True) + username = models.CharField(unique=True, max_length=64, blank=True, null=True) + admin = models.IntegerField(blank=True, null=True) + role = models.IntegerField(blank=True, null=True) + password_hash = models.CharField(max_length=128, blank=True, null=True) + api_key = models.CharField(unique=True, max_length=128, blank=True, null=True) + + class Meta: + """Set User model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "pe_users" + +# ?????? not sure if we use this anywhere +class AlembicVersion(models.Model): + """Define AlembicVersion model.""" + + version_num = models.CharField(primary_key=True, max_length=32) + + class Meta: + """Set AlembicVersion model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "alembic_version" + +class SixgillAlerts(models.Model): + """Define Alerts model.""" + + alerts_uid = models.UUIDField(primary_key=True) + alert_name = models.TextField(blank=True, null=True) + content = models.TextField(blank=True, null=True) + date = models.DateField(blank=True, null=True) + sixgill_id = models.TextField(unique=True, blank=True, null=True) + read = models.TextField(blank=True, null=True) + severity = models.TextField(blank=True, null=True) + site = models.TextField(blank=True, null=True) + threat_level = models.TextField(blank=True, null=True) + threats = models.TextField(blank=True, null=True) + title = models.TextField(blank=True, null=True) + user_id = models.TextField(blank=True, null=True) + category = models.TextField(blank=True, null=True) + lang = models.TextField(blank=True, null=True) + organization = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization_uid" + ) + data_source = models.ForeignKey( + "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + ) + content_snip = models.TextField(blank=True, null=True) + asset_mentioned = models.TextField(blank=True, null=True) + asset_type = models.TextField(blank=True, null=True) + + class Meta: + """Set Alerts model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "sixgill_alerts" + + +class Alias(models.Model): + """Define Alias model.""" + + alias_uid = models.UUIDField(primary_key=True) + organization = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization_uid" + ) + alias = models.TextField(unique=True) + + class Meta: + """Set Alias model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "alias" + +# ?????? +class AssetHeaders(models.Model): + """Define AssetHeaders model.""" + + field_id = models.UUIDField( + db_column="_id", primary_key=True + ) # Field renamed because it started with '_'. + organization = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization_uid" + ) + sub_url = models.TextField() + tech_detected = models.TextField() # This field type is a guess. + interesting_header = models.TextField() # This field type is a guess. + ssl2 = models.TextField(blank=True, null=True) # This field type is a guess. + tls1 = models.TextField(blank=True, null=True) # This field type is a guess. + certificate = models.TextField(blank=True, null=True) # This field type is a guess. + scanned = models.BooleanField(blank=True, null=True) + ssl_scanned = models.BooleanField(blank=True, null=True) + + class Meta: + """Set AssetHeaders model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "asset_headers" + unique_together = (("organization", "sub_url"),) + +# # ?????? no data currently +# class AuthGroup(models.Model): +# """Define AuthGroup model.""" + +# name = models.CharField(unique=True, max_length=150) + +# class Meta: +# """Set AuthGroup model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_group" + +# # ?????? no data currently +# class AuthGroupPermissions(models.Model): +# """Define AuthGroupPermissions model.""" + +# id = models.BigAutoField(primary_key=True) +# group = models.ForeignKey(AuthGroup, on_delete=models.CASCADE) +# permission = models.ForeignKey("AuthPermission", on_delete=models.CASCADE) + +# class Meta: +# """Set AuthGroupPermissions model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_group_permissions" +# unique_together = (("group", "permission"),) + +# # ?????? +# class AuthPermission(models.Model): +# """Define AuthPermission model.""" +# id = models.BigAutoField(primary_key=True) +# name = models.CharField(max_length=255) +# content_type = models.ForeignKey("DjangoContentType", on_delete=models.CASCADE) +# codename = models.CharField(max_length=100) + +# class Meta: +# """Set AuthPermission model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_permission" +# unique_together = (("content_type", "codename"),) + +# # ?????? +# class AuthUser(models.Model): +# """Define AuthUser model.""" +# id = models.BigAutoField(primary_key=True) +# password = models.CharField(max_length=128) +# last_login = models.DateTimeField(blank=True, null=True) +# is_superuser = models.BooleanField() +# username = models.CharField(unique=True, max_length=150) +# first_name = models.CharField(max_length=150) +# last_name = models.CharField(max_length=150) +# email = models.CharField(max_length=254) +# is_staff = models.BooleanField() +# is_active = models.BooleanField() +# date_joined = models.DateTimeField() + +# class Meta: +# """Set AuthUser model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_user" + +# # ?????? currently empty +# class AuthUserGroups(models.Model): +# """Define AuthUserGroups model.""" + +# id = models.BigAutoField(primary_key=True) +# user = models.ForeignKey(AuthUser, on_delete=models.CASCADE) +# group = models.ForeignKey(AuthGroup, on_delete=models.CASCADE) + +# class Meta: +# """Set AuthUserGroups model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_user_groups" +# unique_together = (("user", "group"),) + +# # ?????? currently empty +# class AuthUserUserPermissions(models.Model): +# """Define AuthUserUserPermissions model.""" + +# id = models.BigAutoField(primary_key=True) +# user = models.ForeignKey(AuthUser, on_delete=models.CASCADE) +# permission = models.ForeignKey(AuthPermission, on_delete=models.CASCADE) + +# class Meta: +# """Set AuthUserUserPermissions model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_user_user_permissions" +# unique_together = (("user", "permission"),) + +# needs merging merged into cidr table +# class Cidrs(models.Model): +# """Define Cidrs model.""" + +# cidr_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) +# network = models.TextField() # This field type is a guess. +# organization_uid = models.ForeignKey( +# "Organizations", +# on_delete=models.CASCADE, +# db_column="organization_uid", +# blank=True, +# null=True, +# ) +# data_source_uid = models.ForeignKey( +# "DataSource", +# on_delete=models.CASCADE, +# db_column="data_source_uid", +# blank=True, +# null=True, +# ) +# insert_alert = models.TextField(blank=True, null=True) +# first_seen = models.DateField(blank=True, null=True) +# last_seen = models.DateField(blank=True, null=True) +# current = models.BooleanField(blank=True, null=True) + +# class Meta: +# """Set Cidrs model metadata.""" + +# managed = False +# db_table = "cidrs" +# unique_together = (("organization_uid", "network"),) + +class CredentialBreaches(models.Model): + """Define CredentialBreaches model.""" + + credential_breaches_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + breach_name = models.TextField(unique=True) + description = models.TextField(blank=True, null=True) + exposed_cred_count = models.BigIntegerField(blank=True, null=True) + breach_date = models.DateField(blank=True, null=True) + added_date = models.DateTimeField(blank=True, null=True) + modified_date = models.DateTimeField(blank=True, null=True) + data_classes = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) # This field type is a guess. + password_included = models.BooleanField(blank=True, null=True) + is_verified = models.BooleanField(blank=True, null=True) + is_fabricated = models.BooleanField(blank=True, null=True) + is_sensitive = models.BooleanField(blank=True, null=True) + is_retired = models.BooleanField(blank=True, null=True) + is_spam_list = models.BooleanField(blank=True, null=True) + data_source = models.ForeignKey( + "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + ) + + class Meta: + """Set CredentialBreaches model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "credential_breaches" + + +class CredentialExposures(models.Model): + """Define CredentialExposures model.""" + + credential_exposures_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + email = models.TextField() + organization_uid = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization_uid" + ) + root_domain = models.TextField(blank=True, null=True) + sub_domain = models.TextField(blank=True, null=True) + breach_name = models.TextField(blank=True, null=True) + modified_date = models.DateTimeField(blank=True, null=True) + credential_breaches = models.ForeignKey( + CredentialBreaches, + on_delete=models.CASCADE, + db_column="credential_breaches_uid", + ) + data_source = models.ForeignKey( + "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + ) + name = models.TextField(blank=True, null=True) + login_id = models.TextField(blank=True, null=True) + phone = models.TextField(blank=True, null=True) + password = models.TextField(blank=True, null=True) + hash_type = models.TextField(blank=True, null=True) + intelx_system_id = models.TextField(blank=True, null=True) + + class Meta: + """Set CredentialExposures model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "credential_exposures" + unique_together = (("breach_name", "email"),) + +# needs merging +# class CveInfo(models.Model): +# """Define CveInfo model.""" + +# cve_uuid = models.UUIDField(primary_key=True, default=uuid.uuid1) +# cve_name = models.TextField(unique=True, blank=True, null=True) +# cvss_2_0 = models.DecimalField( +# max_digits=1000, decimal_places=1000, blank=True, null=True +# ) +# cvss_2_0_severity = models.TextField(blank=True, null=True) +# cvss_2_0_vector = models.TextField(blank=True, null=True) +# cvss_3_0 = models.DecimalField( +# max_digits=1000, decimal_places=1000, blank=True, null=True +# ) +# cvss_3_0_severity = models.TextField(blank=True, null=True) +# cvss_3_0_vector = models.TextField(blank=True, null=True) +# dve_score = models.DecimalField( +# max_digits=1000, decimal_places=1000, blank=True, null=True +# ) + +# class Meta: +# """Set CveInfo model metadata.""" + +# managed = False +# db_table = "cve_info" + +class CyhyContacts(models.Model): + """Define CyhyContacts model.""" + + field_id = models.UUIDField( + db_column="_id", primary_key=True, default=uuid.uuid1 + ) # Field renamed because it started with '_'. + org_id = models.TextField() + organization = models.ForeignKey( + "Organization", models.DO_NOTHING, db_column="organization_uid" + ) + org_name = models.TextField() + phone = models.TextField(blank=True, null=True) + contact_type = models.TextField() + email = models.TextField(blank=True, null=True) + name = models.TextField(blank=True, null=True) + date_pulled = models.DateField(blank=True, null=True) + + class Meta: + """Set CyhyContacts model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cyhy_contacts" + unique_together = (("org_id", "contact_type", "email", "name"),) + +class CyhyDbAssets(models.Model): + """Define CyhyDbAssets model.""" + + field_id = models.UUIDField( + db_column="_id", primary_key=True, default=uuid.uuid1 + ) # Field renamed because it started with '_'. + org_id = models.TextField(blank=True, null=True) + organization = models.ForeignKey( + "Organization", models.DO_NOTHING, db_column="organization_uid" + ) + org_name = models.TextField(blank=True, null=True) + contact = models.TextField(blank=True, null=True) + network = models.GenericIPAddressField(blank=True, null=True) + type = models.TextField(blank=True, null=True) + first_seen = models.DateField(blank=True, null=True) + last_seen = models.DateField(blank=True, null=True) + currently_in_cyhy = models.BooleanField(blank=True, null=True) + + class Meta: + """Set CyhyDbAssets model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cyhy_db_assets" + unique_together = (("org_id", "network"),) + +# Probably included in VS models +# class CyhyPortScans(models.Model): +# """Define CyhyPortScans model.""" + +# cyhy_port_scans_uid = models.UUIDField(primary_key=True) +# organization_uid = models.ForeignKey( +# "Organization", models.DO_NOTHING, db_column="organization_uid" +# ) +# cyhy_id = models.TextField(unique=True, blank=True, null=True) +# cyhy_time = models.DateTimeField(blank=True, null=True) +# service_name = models.TextField(blank=True, null=True) +# port = models.TextField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# cpe = models.TextField(blank=True, null=True) +# first_seen = models.DateField(blank=True, null=True) +# last_seen = models.DateField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# state = models.TextField(blank=True, null=True) +# agency_type = models.TextField(blank=True, null=True) + +# class Meta: +# """Set CyhyPortScans model metadata.""" + +# managed = False +# db_table = "cyhy_port_scans" + +# TODO determine if we want user logic on both databases +# class PEDataapiApiuser(models.Model): +# """Define DataapiApiuser model.""" + +# id = models.BigAutoField(primary_key=True) +# apikey = models.CharField( +# db_column="apiKey", max_length=200, blank=True, null=True +# ) # Field name made lowercase. +# user = models.OneToOneField(AuthUser, on_delete=models.CASCADE) +# refresh_token = models.CharField(max_length=200, blank=True, null=True) + +# class Meta: +# """Set DataapiApiuser model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "pe_dataAPI_apiuser" + + +# ?????? +class DataSource(models.Model): + """Define DataSource model.""" + + data_source_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + name = models.TextField() + description = models.TextField() + last_run = models.DateField() + + class Meta: + """Set DataSource model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "data_source" + +# # ?????? +# class DjangoAdminLog(models.Model): +# """Define DjangoAdminLog model.""" +# id = models.BigAutoField(primary_key=True) +# action_time = models.DateTimeField() +# object_id = models.TextField(blank=True, null=True) +# object_repr = models.CharField(max_length=200) +# action_flag = models.SmallIntegerField() +# change_message = models.TextField() +# content_type = models.ForeignKey( +# "DjangoContentType", on_delete=models.CASCADE, blank=True, null=True +# ) +# user = models.ForeignKey(AuthUser, on_delete=models.CASCADE) + +# class Meta: +# """Set DjangoAdminLog model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_admin_log" + +# # ?????? +# class DjangoContentType(models.Model): +# """Define DjangoContentType model.""" +# id = models.BigAutoField(primary_key=True) +# app_label = models.CharField(max_length=100) +# model = models.CharField(max_length=100) + +# class Meta: +# """Set DjangoContentType model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_content_type" +# unique_together = (("app_label", "model"),) + +# # ?????? +# class DjangoMigrations(models.Model): +# """Define DjangoMigrations model.""" + +# id = models.BigAutoField(primary_key=True) +# app = models.CharField(max_length=255) +# name = models.CharField(max_length=255) +# applied = models.DateTimeField() + +# class Meta: +# """Set DjangoMigrations model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_migrations" + +# # ?????? +# class DjangoSession(models.Model): +# """Define DjangoSession model.""" + +# session_key = models.CharField(primary_key=True, max_length=40) +# session_data = models.TextField() +# expire_date = models.DateTimeField() + +# class Meta: +# """Set DjangoSession model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_session" + +class DnsRecords(models.Model): + """Define DnsRecords model.""" + + dns_record_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + domain_name = models.TextField(blank=True, null=True) + domain_type = models.TextField(blank=True, null=True) + created_date = models.DateTimeField(blank=True, null=True) + updated_date = models.DateTimeField(blank=True, null=True) + expiration_date = models.DateTimeField(blank=True, null=True) + name_servers = models.TextField( + blank=True, null=True + ) # This field type is a guess. + whois_server = models.TextField(blank=True, null=True) + registrar_name = models.TextField(blank=True, null=True) + status = models.TextField(blank=True, null=True) + clean_text = models.TextField(blank=True, null=True) + raw_text = models.TextField(blank=True, null=True) + registrant_name = models.TextField(blank=True, null=True) + registrant_organization = models.TextField(blank=True, null=True) + registrant_street = models.TextField(blank=True, null=True) + registrant_city = models.TextField(blank=True, null=True) + registrant_state = models.TextField(blank=True, null=True) + registrant_post_code = models.TextField(blank=True, null=True) + registrant_country = models.TextField(blank=True, null=True) + registrant_email = models.TextField(blank=True, null=True) + registrant_phone = models.TextField(blank=True, null=True) + registrant_phone_ext = models.TextField(blank=True, null=True) + registrant_fax = models.TextField(blank=True, null=True) + registrant_fax_ext = models.TextField(blank=True, null=True) + registrant_raw_text = models.TextField(blank=True, null=True) + administrative_name = models.TextField(blank=True, null=True) + administrative_organization = models.TextField(blank=True, null=True) + administrative_street = models.TextField(blank=True, null=True) + administrative_city = models.TextField(blank=True, null=True) + administrative_state = models.TextField(blank=True, null=True) + administrative_post_code = models.TextField(blank=True, null=True) + administrative_country = models.TextField(blank=True, null=True) + administrative_email = models.TextField(blank=True, null=True) + administrative_phone = models.TextField(blank=True, null=True) + administrative_phone_ext = models.TextField(blank=True, null=True) + administrative_fax = models.TextField(blank=True, null=True) + administrative_fax_ext = models.TextField(blank=True, null=True) + administrative_raw_text = models.TextField(blank=True, null=True) + technical_name = models.TextField(blank=True, null=True) + technical_organization = models.TextField(blank=True, null=True) + technical_street = models.TextField(blank=True, null=True) + technical_city = models.TextField(blank=True, null=True) + technical_state = models.TextField(blank=True, null=True) + technical_post_code = models.TextField(blank=True, null=True) + technical_country = models.TextField(blank=True, null=True) + technical_email = models.TextField(blank=True, null=True) + technical_phone = models.TextField(blank=True, null=True) + technical_phone_ext = models.TextField(blank=True, null=True) + technical_fax = models.TextField(blank=True, null=True) + technical_fax_ext = models.TextField(blank=True, null=True) + technical_raw_text = models.TextField(blank=True, null=True) + billing_name = models.TextField(blank=True, null=True) + billing_organization = models.TextField(blank=True, null=True) + billing_street = models.TextField(blank=True, null=True) + billing_city = models.TextField(blank=True, null=True) + billing_state = models.TextField(blank=True, null=True) + billing_post_code = models.TextField(blank=True, null=True) + billing_country = models.TextField(blank=True, null=True) + billing_email = models.TextField(blank=True, null=True) + billing_phone = models.TextField(blank=True, null=True) + billing_phone_ext = models.TextField(blank=True, null=True) + billing_fax = models.TextField(blank=True, null=True) + billing_fax_ext = models.TextField(blank=True, null=True) + billing_raw_text = models.TextField(blank=True, null=True) + zone_name = models.TextField(blank=True, null=True) + zone_organization = models.TextField(blank=True, null=True) + zone_street = models.TextField(blank=True, null=True) + zone_city = models.TextField(blank=True, null=True) + zone_state = models.TextField(blank=True, null=True) + zone_post_code = models.TextField(blank=True, null=True) + zone_country = models.TextField(blank=True, null=True) + zone_email = models.TextField(blank=True, null=True) + zone_phone = models.TextField(blank=True, null=True) + zone_phone_ext = models.TextField(blank=True, null=True) + zone_fax = models.TextField(blank=True, null=True) + zone_fax_ext = models.TextField(blank=True, null=True) + zone_raw_text = models.TextField(blank=True, null=True) + + class Meta: + """Set DnsRecords model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "dns_records" + +# Possibly shodan +class DomainAlerts(models.Model): + """Define DomainAlerts model.""" + + domain_alert_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + sub_domain = models.ForeignKey( + "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid" + ) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + organization_uid = models.UUIDField() + alert_type = models.TextField(blank=True, null=True) + message = models.TextField(blank=True, null=True) + previous_value = models.TextField(blank=True, null=True) + new_value = models.TextField(blank=True, null=True) + date = models.DateField(blank=True, null=True) + + class Meta: + """Set DomainAlerts model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "domain_alerts" + unique_together = (("alert_type", "sub_domain", "date", "new_value"),) + +class DomainPermutations(models.Model): + """Define DomainPermutations model.""" + + suspected_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + organization = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization_uid" + ) + domain_permutation = models.TextField(blank=True, null=True) + ipv4 = models.TextField(blank=True, null=True) + ipv6 = models.TextField(blank=True, null=True) + mail_server = models.TextField(blank=True, null=True) + name_server = models.TextField(blank=True, null=True) + fuzzer = models.TextField(blank=True, null=True) + date_observed = models.DateField(blank=True, null=True) + ssdeep_score = models.TextField(blank=True, null=True) + malicious = models.BooleanField(blank=True, null=True) + blocklist_attack_count = models.IntegerField(blank=True, null=True) + blocklist_report_count = models.IntegerField(blank=True, null=True) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + sub_domain = models.ForeignKey( + "SubDomains", + on_delete=models.CASCADE, + db_column="sub_domain_uid", + blank=True, + null=True, + ) + dshield_record_count = models.IntegerField(blank=True, null=True) + dshield_attack_count = models.IntegerField(blank=True, null=True) + date_active = models.DateField(blank=True, null=True) + + class Meta: + """Set DomainPermutations model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "domain_permutations" + unique_together = (("domain_permutation", "organization"),) + +class DotgovDomains(models.Model): + """Define DotgovDomains model.""" + + dotgov_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + domain_name = models.TextField(unique=True) + domain_type = models.TextField(blank=True, null=True) + agency = models.TextField(blank=True, null=True) + organization = models.TextField(blank=True, null=True) + city = models.TextField(blank=True, null=True) + state = models.TextField(blank=True, null=True) + security_contact_email = models.TextField(blank=True, null=True) + + class Meta: + """Set DotgovDomains model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "dotgov_domains" + +class Executives(models.Model): + """Define Executives model.""" + + executives_uid = models.UUIDField(primary_key=True) + organization = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization" + ) + executives = models.TextField() + + class Meta: + """Set Executives model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "executives" + +# merged with vs's IP table +# class Ips(models.Model): +# """Define Ips model.""" + +# ip_hash = models.TextField(primary_key=True) +# ip = models.GenericIPAddressField(unique=True) +# origin_cidr = models.ForeignKey( +# Cidr, on_delete=models.CASCADE, db_column="origin_cidr", blank=True, null=True +# ) +# shodan_results = models.BooleanField(blank=True, null=True) +# live = models.BooleanField(blank=True, null=True) +# date_last_live = models.DateTimeField(blank=True, null=True) +# last_reverse_lookup = models.DateTimeField(blank=True, null=True) +# first_seen = models.DateField(blank=True, null=True) +# last_seen = models.DateField(blank=True, null=True) +# current = models.BooleanField(blank=True, null=True) +# from_cidr = models.BooleanField(blank=True, null=True) # varchar type in db??? +# organization_uid = models.UUIDField(blank=True, null=True) + +# class Meta: +# """Set Ips model metadata.""" + +# managed = False +# db_table = "ips" + +# replaced with manny to many field in ip table +# class IpsSubs(models.Model): +# """Define IpsSubs model.""" + +# ips_subs_uid = models.UUIDField(primary_key=True) +# ip_hash = models.ForeignKey(Ips, on_delete=models.CASCADE, db_column="ip_hash") +# sub_domain_uid = models.ForeignKey( +# "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid" +# ) + +# class Meta: +# """Set IpsSubs model metadata.""" + +# managed = False +# # db_table = 'ips_subs' +# unique_together = (("ip_hash", "sub_domain_uid"),) + +class Mentions(models.Model): + """Define Mentions model.""" + + mentions_uid = models.UUIDField(primary_key=True) + category = models.TextField(blank=True, null=True) + collection_date = models.TextField(blank=True, null=True) + content = models.TextField(blank=True, null=True) + creator = models.TextField(blank=True, null=True) + date = models.DateField(blank=True, null=True) + sixgill_mention_id = models.TextField(unique=True, blank=True, null=True) + post_id = models.TextField(blank=True, null=True) + lang = models.TextField(blank=True, null=True) + rep_grade = models.TextField(blank=True, null=True) + site = models.TextField(blank=True, null=True) + site_grade = models.TextField(blank=True, null=True) + title = models.TextField(blank=True, null=True) + type = models.TextField(blank=True, null=True) + url = models.TextField(blank=True, null=True) + comments_count = models.TextField(blank=True, null=True) + sub_category = models.TextField(blank=True, null=True) + tags = models.TextField(blank=True, null=True) + organization_uid = models.UUIDField() + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + title_translated = models.TextField(blank=True, null=True) + content_translated = models.TextField(blank=True, null=True) + detected_lang = models.TextField(blank=True, null=True) + + class Meta: + """Set Mentions model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "mentions" + +# Likely can be removed +class OrgIdMap(models.Model): + """Define OrgIdMap model.""" + + cyhy_id = models.TextField(blank=True, null=True) + pe_org_id = models.TextField(blank=True, null=True) + merge_orgs = models.BooleanField(blank=True, null=True) + + class Meta: + """Set OrgIdMap model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "org_id_map" + unique_together = (("cyhy_id", "pe_org_id"),) + + +class OrgType(models.Model): + """Define OrgType model.""" + + org_type_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + org_type = models.TextField(blank=True, null=True) + + class Meta: + """Set OrgType model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "org_type" + + +# needs to be merged merged +# class Organizations(models.Model): +# """Define Organizations model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# name = models.TextField() +# cyhy_db_name = models.TextField(unique=True, blank=True, null=True) +# org_type_uid = models.ForeignKey( +# OrgType, +# on_delete=models.CASCADE, +# db_column="org_type_uid", +# blank=True, +# null=True, +# ) +# report_on = models.BooleanField(blank=True, null=True) +# password = models.TextField(blank=True, null=True) +# date_first_reported = models.DateTimeField(blank=True, null=True) +# parent_org_uid = models.ForeignKey( +# "self", +# on_delete=models.CASCADE, +# db_column="parent_org_uid", +# blank=True, +# null=True, +# ) +# premium_report = models.BooleanField(blank=True, null=True) +# agency_type = models.TextField(blank=True, null=True) +# demo = models.BooleanField(blank=True, null=True) +# scorecard = models.BooleanField(blank=True, null=True) +# fceb = models.BooleanField(blank=True, null=True) +# receives_cyhy_report = models.BooleanField(blank=True, null=True) +# receives_bod_report = models.BooleanField(blank=True, null=True) +# receives_cybex_report = models.BooleanField(blank=True, null=True) +# run_scans = models.BooleanField(blank=True, null=True) +# is_parent = models.BooleanField(blank=True, null=True) +# ignore_roll_up = models.BooleanField(blank=True, null=True) +# retired = models.BooleanField(blank=True, null=True) +# cyhy_period_start = models.DateField(blank=True, null=True) +# fceb_child = models.BooleanField(blank=True, null=True) +# election = models.BooleanField(blank=True, null=True) +# scorecard_child = models.BooleanField(blank=True, null=True) +# location_name = models.TextField(blank=True, null=True) +# county = models.TextField(blank=True, null=True) +# county_fips = models.IntegerField(blank=True, null=True) +# state_abbreviation = models.TextField(blank=True, null=True) +# state_fips = models.IntegerField(blank=True, null=True) +# state_name = models.TextField(blank=True, null=True) +# country = models.TextField(blank=True, null=True) +# country_name = models.TextField(blank=True, null=True) + +# class Meta: +# """Set Organizations model metadata.""" + +# managed = False +# db_table = "organizations" + + +class PshttResults(models.Model): + """Define PshttResults model.""" + + pshtt_results_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + organization = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organization_uid" + ) + sub_domain = models.ForeignKey( + "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid" + ) + data_source = models.ForeignKey( + "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + ) + sub_domain = models.TextField() + date_scanned = models.DateField(blank=True, null=True) + base_domain = models.TextField(blank=True, null=True) + base_domain_hsts_preloaded = models.BooleanField(blank=True, null=True) + canonical_url = models.TextField(blank=True, null=True) + defaults_to_https = models.BooleanField(blank=True, null=True) + domain = models.TextField(blank=True, null=True) + domain_enforces_https = models.BooleanField(blank=True, null=True) + domain_supports_https = models.BooleanField(blank=True, null=True) + domain_uses_strong_hsts = models.BooleanField(blank=True, null=True) + downgrades_https = models.BooleanField(blank=True, null=True) + htss = models.BooleanField(blank=True, null=True) + hsts_entire_domain = models.BooleanField(blank=True, null=True) + hsts_header = models.TextField(blank=True, null=True) + hsts_max_age = models.DecimalField( + max_digits=1000, decimal_places=1000, blank=True, null=True + ) + hsts_preload_pending = models.BooleanField(blank=True, null=True) + hsts_preload_ready = models.BooleanField(blank=True, null=True) + hsts_preloaded = models.BooleanField(blank=True, null=True) + https_bad_chain = models.BooleanField(blank=True, null=True) + https_bad_hostname = models.BooleanField(blank=True, null=True) + https_cert_chain_length = models.IntegerField(blank=True, null=True) + https_client_auth_required = models.BooleanField(blank=True, null=True) + https_custom_truststore_trusted = models.BooleanField(blank=True, null=True) + https_expired_cert = models.BooleanField(blank=True, null=True) + https_full_connection = models.BooleanField(blank=True, null=True) + https_live = models.BooleanField(blank=True, null=True) + https_probably_missing_intermediate_cert = models.BooleanField( + blank=True, null=True + ) + https_publicly_trusted = models.BooleanField(blank=True, null=True) + https_self_signed_cert = models.BooleanField(blank=True, null=True) + https_leaf_cert_expiration_date = models.DateField(blank=True, null=True) + https_leaf_cert_issuer = models.TextField(blank=True, null=True) + https_leaf_cert_subject = models.TextField(blank=True, null=True) + https_root_cert_issuer = models.TextField(blank=True, null=True) + ip = models.GenericIPAddressField(blank=True, null=True) + live = models.BooleanField(blank=True, null=True) + notes = models.TextField(blank=True, null=True) + redirect = models.BooleanField(blank=True, null=True) + redirect_to = models.TextField(blank=True, null=True) + server_header = models.TextField(blank=True, null=True) + server_version = models.TextField(blank=True, null=True) + strictly_forces_https = models.BooleanField(blank=True, null=True) + unknown_error = models.BooleanField(blank=True, null=True) + valid_https = models.BooleanField(blank=True, null=True) + ep_http_headers = models.TextField( + blank=True, null=True + ) # This field type is a guess. + ep_http_server_header = models.TextField(blank=True, null=True) + ep_http_server_version = models.TextField(blank=True, null=True) + ep_https_headers = models.TextField( + blank=True, null=True + ) # This field type is a guess. + ep_https_hsts_header = models.TextField(blank=True, null=True) + ep_https_server_header = models.TextField(blank=True, null=True) + ep_https_server_version = models.TextField(blank=True, null=True) + ep_httpswww_headers = models.TextField( + blank=True, null=True + ) # This field type is a guess. + ep_httpswww_hsts_header = models.TextField(blank=True, null=True) + ep_httpswww_server_header = models.TextField(blank=True, null=True) + ep_httpswww_server_version = models.TextField(blank=True, null=True) + ep_httpwww_headers = models.TextField( + blank=True, null=True + ) # This field type is a guess. + ep_httpwww_server_header = models.TextField(blank=True, null=True) + ep_httpwww_server_version = models.TextField(blank=True, null=True) + + class Meta: + """Set PshttResults model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "pshtt_results" + unique_together = (("organization", "sub_domain"),) + + + +class PeReportSummaryStats(models.Model): + """Define ReportSummaryStats model.""" + + report_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + organization = models.ForeignKey( + Organization, on_delete=models.CASCADE, db_column="organization_uid" + ) + start_date = models.DateField() + end_date = models.DateField(blank=True, null=True) + ip_count = models.IntegerField(blank=True, null=True) + root_count = models.IntegerField(blank=True, null=True) + sub_count = models.IntegerField(blank=True, null=True) + ports_count = models.IntegerField(blank=True, null=True) + creds_count = models.IntegerField(blank=True, null=True) + breach_count = models.IntegerField(blank=True, null=True) + cred_password_count = models.IntegerField(blank=True, null=True) + domain_alert_count = models.IntegerField(blank=True, null=True) + suspected_domain_count = models.IntegerField(blank=True, null=True) + insecure_port_count = models.IntegerField(blank=True, null=True) + verified_vuln_count = models.IntegerField(blank=True, null=True) + suspected_vuln_count = models.IntegerField(blank=True, null=True) + suspected_vuln_addrs_count = models.IntegerField(blank=True, null=True) + threat_actor_count = models.IntegerField(blank=True, null=True) + dark_web_alerts_count = models.IntegerField(blank=True, null=True) + dark_web_mentions_count = models.IntegerField(blank=True, null=True) + dark_web_executive_alerts_count = models.IntegerField(blank=True, null=True) + dark_web_asset_alerts_count = models.IntegerField(blank=True, null=True) + pe_number_score = models.TextField(blank=True, null=True) + pe_letter_grade = models.TextField(blank=True, null=True) + pe_percent_score = models.DecimalField( + max_digits=1000, decimal_places=1000, blank=True, null=True + ) + cidr_count = models.IntegerField(blank=True, null=True) + port_protocol_count = models.IntegerField(blank=True, null=True) + software_count = models.IntegerField(blank=True, null=True) + foreign_ips_count = models.IntegerField(blank=True, null=True) + + class Meta: + """Set ReportSummaryStats model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "pe_report_summary_stats" + unique_together = (("organization", "start_date"),) + +class RootDomains(models.Model): + """Define RootDomains model.""" + + root_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + organization = models.ForeignKey( + Organization, on_delete=models.CASCADE, db_column="organization_uid" + ) + root_domain = models.TextField() + ip_address = models.TextField(blank=True, null=True) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + enumerate_subs = models.BooleanField(blank=True, null=True) + + class Meta: + """Set RootDomains model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "root_domains" + unique_together = (("root_domain", "organization"),) + + +class PeTeamMembers(models.Model): + """Define TeamMembers model.""" + + team_member_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + team_member_fname = models.TextField(blank=False, null=False) + team_member_lname = models.TextField(blank=False, null=False) + team_member_email = models.TextField(blank=False, null=False) + team_member_ghID = models.TextField(blank=False, null=False) + team_member_phone = models.TextField(blank=True, null=True) + team_member_role = models.TextField(blank=True, null=True) + team_member_notes = models.TextField(blank=True, null=True) + + class Meta: + """Set TeamMembers model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "pe_team_members" + +class ShodanAssets(models.Model): + """Define ShodanAssets model.""" + + shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + organization = models.ForeignKey( + Organization, on_delete=models.CASCADE, db_column="organization_uid" + ) + organization = models.TextField(blank=True, null=True) + ip = models.TextField(blank=True, null=True) + port = models.IntegerField(blank=True, null=True) + protocol = models.TextField(blank=True, null=True) + timestamp = models.DateTimeField(blank=True, null=True) + product = models.TextField(blank=True, null=True) + server = models.TextField(blank=True, null=True) + tags = models.TextField(blank=True, null=True) # This field type is a guess. + domains = models.TextField(blank=True, null=True) # This field type is a guess. + hostnames = models.TextField(blank=True, null=True) # This field type is a guess. + isn = models.TextField(blank=True, null=True) + asn = models.IntegerField(blank=True, null=True) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + country_code = models.TextField(blank=True, null=True) + location = models.TextField(blank=True, null=True) + + class Meta: + """Set ShodanAssets model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "shodan_assets" + unique_together = ( + ("organization", "ip", "port", "protocol", "timestamp"), + ) + +# class ShodanInsecureProtocolsUnverifiedVulns(models.Model): +# """Define ShodanInsecureProtocolsUnverifiedVulns model.""" + +# insecure_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) +# organization_uid = models.ForeignKey( +# Organization, on_delete=models.CASCADE, db_column="organization_uid" +# ) +# organization = models.TextField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# port = models.IntegerField(blank=True, null=True) +# protocol = models.TextField(blank=True, null=True) +# type = models.TextField(blank=True, null=True) +# name = models.TextField(blank=True, null=True) +# potential_vulns = models.TextField( +# blank=True, null=True +# ) # This field type is a guess. +# mitigation = models.TextField(blank=True, null=True) +# timestamp = models.DateTimeField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# server = models.TextField(blank=True, null=True) +# tags = models.TextField(blank=True, null=True) # This field type is a guess. +# domains = models.TextField(blank=True, null=True) # This field type is a guess. +# hostnames = models.TextField(blank=True, null=True) # This field type is a guess. +# isn = models.TextField(blank=True, null=True) +# asn = models.IntegerField(blank=True, null=True) +# data_source_uid = models.ForeignKey( +# DataSource, on_delete=models.CASCADE, db_column="data_source_uid" +# ) + +# class Meta: +# """Set ShodanInsecureProtocolsUnverifiedVulns model metadata.""" + +# managed = False +# db_table = "shodan_insecure_protocols_unverified_vulns" +# unique_together = ( +# ("organization_uid", "ip", "port", "protocol", "timestamp"), +# ) + + +class ShodanVulns(models.Model): + """Define ShodanVulns model.""" + + shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + organization = models.ForeignKey( + Organization, on_delete=models.CASCADE, db_column="organization_uid" + ) + organization = models.TextField(blank=True, null=True) + ip = models.TextField(blank=True, null=True) + port = models.TextField(blank=True, null=True) + protocol = models.TextField(blank=True, null=True) + timestamp = models.DateTimeField(blank=True, null=True) + cve = models.TextField(blank=True, null=True) + severity = models.TextField(blank=True, null=True) + cvss = models.DecimalField( + max_digits=1000, decimal_places=1000, blank=True, null=True + ) + summary = models.TextField(blank=True, null=True) + product = models.TextField(blank=True, null=True) + attack_vector = models.TextField(blank=True, null=True) + av_description = models.TextField(blank=True, null=True) + attack_complexity = models.TextField(blank=True, null=True) + ac_description = models.TextField(blank=True, null=True) + confidentiality_impact = models.TextField(blank=True, null=True) + ci_description = models.TextField(blank=True, null=True) + integrity_impact = models.TextField(blank=True, null=True) + ii_description = models.TextField(blank=True, null=True) + availability_impact = models.TextField(blank=True, null=True) + ai_description = models.TextField(blank=True, null=True) + tags = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) # This field type is a guess. + domains = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) # This field type is a guess. + hostnames = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) # This field type is a guess. + isn = models.TextField(blank=True, null=True) + asn = models.IntegerField(blank=True, null=True) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + type = models.TextField(blank=True, null=True) + name = models.TextField(blank=True, null=True) + potential_vulns = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) # This field type is a guess. + mitigation = models.TextField(blank=True, null=True) + server = models.TextField(blank=True, null=True) + is_verified = models.BooleanField(blank=True, null=True) + banner = models.TextField(blank=True, null=True) + version = models.TextField(blank=True, null=True) + mitigation = models.TextField(blank=True, null=True) + cpe = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) + + class Meta: + """Set ShodanVulns model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "shodan_vulns" + unique_together = ( + ("organization", "ip", "port", "protocol", "timestamp"), + ) + +class SubDomains(models.Model): + """Define SubDomains model.""" + + sub_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + sub_domain = models.TextField()# Crossfeed Domains name field + root_domain = models.ForeignKey( + RootDomains, on_delete=models.CASCADE, db_column="root_domain_uid" + ) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + dns_record = models.ForeignKey( + DnsRecords, + on_delete=models.CASCADE, + db_column="dns_record_uid", + blank=True, + null=True, + ) + status = models.BooleanField(blank=True, null=True) + first_seen = models.DateField(blank=True, null=True) + last_seen = models.DateField(blank=True, null=True) + created_at = models.DateTimeField(db_column="created_at") + updated_at = models.DateTimeField(db_column="updated_at") + current = models.BooleanField(blank=True, null=True) + identified = models.BooleanField(blank=True, null=True) + ip_address = models.TextField(blank=True, null=True)# XFD column + synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True)# XFD column + from_root_domain = models.TextField(db_column="from_root_domain", blank=True, null=True)# XFD column + subdomain_source = models.TextField( + db_column="subdomain_source", max_length=255, blank=True, null=True + )# XFD column + organization = models.ForeignKey( + Organization, on_delete=models.CASCADE, db_column="organization_uid" + ) + ip_only = models.BooleanField(db_column="ip_only", default=False)# XFD column + reverse_name = models.CharField(db_column="reverse_name", max_length=512)# XFD column + screenshot = models.CharField(max_length=512, blank=True, null=True)# XFD Crossfeed Domains screenshot field + country = models.CharField(max_length=255, blank=True, null=True)# XFD column + asn = models.CharField(max_length=255, blank=True, null=True)# XFD column + cloud_hosted = models.BooleanField(db_column="cloud_hosted", default=False)# XFD column + ssl = models.JSONField(blank=True, null=True)# XFD columnv + censys_certificates_results = models.JSONField( + db_column="censys_certificates_results", default=dict + )# XFD column + trustymail_results = models.JSONField(db_column="trustymail_results", default=dict)# XFD column + class Meta: + """Set SubDomains model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "sub_domains" + unique_together = (("sub_domain", "root_domain"),) + +class TopCves(models.Model): + """Define TopCves model.""" + + top_cves_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + cve_id = models.TextField(blank=True, null=True) + dynamic_rating = models.TextField(blank=True, null=True) + nvd_base_score = models.TextField(blank=True, null=True) + date = models.DateField(blank=True, null=True) + summary = models.TextField(blank=True, null=True) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + + class Meta: + """Set TopCves model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "top_cves" + unique_together = (("cve_id", "date"),) + +# Not sure if this is still used +class TopicTotals(models.Model): + """Define TopicTotals model.""" + + count_uuid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + organization_uid = models.UUIDField() + content_count = models.IntegerField() + count_date = models.TextField(blank=True, null=True) + + class Meta: + """Set TopicTotals model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "topic_totals" + +# Not sure if this is still used +class UniqueSoftware(models.Model): + """Define UniqueSoftware model.""" + + field_id = models.UUIDField( + db_column="_id", primary_key=True, default=uuid.uuid1() + ) # Field renamed because it started with '_'. + software_name = models.TextField(blank=False, null=False) + + class Meta: + """Set UniqueSoftware model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "unique_software" + +class WebAssets(models.Model): + """Define WebAssets model.""" + + asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + asset_type = models.TextField() + asset = models.TextField() + ip_type = models.TextField(blank=True, null=True) + verified = models.BooleanField(blank=True, null=True) + organization = models.ForeignKey( + Organization, on_delete=models.CASCADE, db_column="organization_uid" + ) + asset_origin = models.TextField(blank=True, null=True) + report_on = models.BooleanField(blank=True, null=True) + last_scanned = models.DateTimeField(blank=True, null=True) + report_status_reason = models.TextField(blank=True, null=True) + data_source = models.ForeignKey( + DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + ) + + class Meta: + """Set WebAssets model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "web_assets" + unique_together = (("asset", "organization"),) + +class WeeklyStatusesMdl(models.Model): + """Define WeeklyStatusesMdl model.""" + + weekly_status_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + user_status = models.TextField(blank=True) + key_accomplishments = models.TextField(blank=True, null=True) + ongoing_task = models.TextField() + upcoming_task = models.TextField() + obstacles = models.TextField(blank=True, null=True) + non_standard_meeting = models.TextField(blank=True, null=True) + deliverables = models.TextField(blank=True, null=True) + pto = models.TextField(blank=True, null=True) + week_ending = models.DateField() + notes = models.TextField(blank=True, null=True) + statusComplete = models.IntegerField(blank=True, null=True) + + class Meta: + """Set WeeklyStatusesMdl model metadata.""" + + # unique_together = (('week_ending', 'user_status'),) + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "weekly_statuses_mdl" + + +# cyhy_kevs table model (needed for kev_list endpoint) +class CyhyKevs(models.Model): + """Define CyhyKevs model.""" + + cyhy_kevs_uid = models.UUIDField(primary_key=True) + kev = models.CharField(blank=True, null=True, max_length=255) + first_seen = models.DateField(blank=True, null=True) + last_seen = models.DateField(blank=True, null=True) + + class Meta: + """Set CyhyKevs model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cyhy_kevs" + + +class XpanseBusinessUnits(models.Model): + """Define XpanseBusinessUnits model.""" + + xpanse_business_unit_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + entity_name = models.TextField(unique=True, blank=True, null=True) + cyhy_db_name = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="cyhy_db_name", to_field="acronym" + ) + state = models.TextField(blank=True, null=True) + county = models.TextField(blank=True, null=True) + city = models.TextField(blank=True, null=True) + sector = models.TextField(blank=True, null=True) + entity_type = models.TextField(blank=True, null=True) + region = models.TextField(blank=True, null=True) + rating = models.IntegerField(blank=True, null=True) + + class Meta: + """Set XpanseBusinessUnits metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "xpanse_business_units" + + +class XpanseAssetsMdl(models.Model): + """Define XpanseAssetsMdl model.""" + + xpanse_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + asm_id = models.TextField(unique=True, blank=False, null=False) + asset_name = models.TextField(blank=True, null=True) + asset_type = models.TextField(blank=True, null=True) + last_observed = models.DateTimeField(blank=True, null=True) + first_observed = models.DateTimeField(blank=True, null=True) + externally_detected_providers = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + created = models.DateTimeField(blank=True, null=True) + ips = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) + active_external_services_types = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + domain = models.TextField(blank=True, null=True) + certificate_issuer = models.TextField(blank=True, null=True) + certificate_algorithm = models.TextField(blank=True, null=True) + certificate_classifications = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + resolves = models.BooleanField(blank=True, null=True) + # details + top_level_asset_mapper_domain = models.TextField(blank=True, null=True) + domain_asset_type = models.JSONField(blank=True, null=True) + is_paid_level_domain = models.BooleanField(blank=True, null=True) + domain_details = models.JSONField(blank=True, null=True) + dns_zone = models.TextField(blank=True, null=True) + latest_sampled_ip = models.IntegerField(blank=True, null=True) + + recent_ips = models.JSONField(blank=True, null=True) + external_services = models.JSONField(blank=True, null=True) + externally_inferred_vulnerability_score = models.DecimalField( + max_digits=5, decimal_places=2, blank=True, null=True + ) + externally_inferred_cves = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + explainers = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + tags = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) + + class Meta: + """Set XpanseAssetsMdl metdata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "xpanse_assets_mdl" + + +class XpanseCvesMdl(models.Model): + """Define XpanseCvesMdl model.""" + + xpanse_cve_uid = models.UUIDField(unique=True, primary_key=True, default=uuid.uuid1) + cve_id = models.TextField(unique=True, blank=True, null=True) + cvss_score_v2 = models.DecimalField( + max_digits=5, decimal_places=2, blank=True, null=True + ) + cve_severity_v2 = models.TextField(blank=True, null=True) + cvss_score_v3 = models.DecimalField( + max_digits=5, decimal_places=2, blank=True, null=True + ) + cve_severity_v3 = models.TextField(blank=True, null=True) + + class Meta: + """Set XpanseCvesMdl metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "xpanse_cves_mdl" + + +class XpanseServicesMdl(models.Model): + """Define XpanseServicesMdl model.""" + + xpanse_service_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + service_id = models.TextField(unique=True, blank=True, null=True) + service_name = models.TextField(blank=True, null=True) + service_type = models.TextField(blank=True, null=True) + ip_address = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + domain = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) + externally_detected_providers = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + is_active = models.TextField(blank=True, null=True) + first_observed = models.DateTimeField(blank=True, null=True) + last_observed = models.DateTimeField(blank=True, null=True) + port = models.IntegerField(blank=True, null=True) + protocol = models.TextField(blank=True, null=True) + active_classifications = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + inactive_classifications = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + discovery_type = models.TextField(blank=True, null=True) + externally_inferred_vulnerability_score = models.DecimalField( + max_digits=5, decimal_places=2, blank=True, null=True + ) + externally_inferred_cves = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + service_key = models.TextField(blank=True, null=True) + service_key_type = models.TextField(blank=True, null=True) + + cves = models.ManyToManyField(XpanseCvesMdl, through="XpanseCveServiceMdl") + + class Meta: + """Set XpanseServicesMdl metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "xpanse_services_mdl" + + +class XpanseCveServiceMdl(models.Model): + """Define XpanseCves-Service linking table model.""" + + xpanse_inferred_cve = models.ForeignKey(XpanseCvesMdl, on_delete=models.CASCADE) + xpanse_service = models.ForeignKey(XpanseServicesMdl, on_delete=models.CASCADE) + inferred_cve_match_type = models.TextField(blank=True, null=True) + product = models.TextField(blank=True, null=True) + confidence = models.TextField(blank=True, null=True) + vendor = models.TextField(blank=True, null=True) + version_number = models.TextField(blank=True, null=True) + activity_status = models.TextField(blank=True, null=True) + first_observed = models.DateTimeField(blank=True, null=True) + last_observed = models.DateTimeField(blank=True, null=True) + + class Meta: + """Set XpanseCveServiceMdl metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "xpanse_cve_services_mdl" + unique_together = (("xpanse_inferred_cve", "xpanse_service"),) + + +class XpanseAlerts(models.Model): + """Define XpanseAlerts model.""" + + xpanse_alert_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + time_pulled_from_xpanse = models.DateTimeField(blank=True, null=True) + alert_id = models.TextField(unique=True, blank=False, null=False) + detection_timestamp = models.DateTimeField(blank=True, null=True) + alert_name = models.TextField(blank=True, null=True) + # endpoint_id ???, + description = models.TextField(blank=True, null=True) + host_name = models.TextField(blank=True, null=True) + alert_action = models.TextField(blank=True, null=True) + # user_name ??? null, + # mac_addresses ??? null, + # source ??? null, + action_pretty = models.TextField(blank=True, null=True) + # category ??? null, + # project ??? null, + # cloud_provider ??? null, + # resource_sub_type ??? null, + # resource_type ??? null, + action_country = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + # event_type ??? null, + # is_whitelisted ??? null, + # image_name ??? null, + # action_local_ip ??? null, + # action_local_port ??? null, + # action_external_hostname ??? null, + # action_remote_ip ??? null, + action_remote_port = ArrayField( + models.IntegerField(blank=True, null=False), blank=True, null=True + ) + # "matching_service_rule_id ??? null, + starred = models.BooleanField(blank=True, null=True) + external_id = models.TextField(blank=True, null=True) + related_external_id = models.TextField(blank=True, null=True) + alert_occurrence = models.IntegerField(blank=True, null=True) + severity = models.TextField(blank=True, null=True) + matching_status = models.TextField(blank=True, null=True) + # end_match_attempt_ts ??? null, + local_insert_ts = models.DateTimeField(blank=True, null=True) + last_modified_ts = models.DateTimeField(blank=True, null=True) + case_id = models.IntegerField(blank=True, null=True) + # deduplicate_tokens ??? null, + # filter_rule_id ??? null, + # event_id ??? null, + event_timestamp = ArrayField( + models.DateTimeField(blank=True, null=False), blank=True, null=True + ) + # action_local_ip_v6 ??? null, + # action_remote_ip_v6 ??? null, + alert_type = models.TextField(blank=True, null=True) + resolution_status = models.TextField(blank=True, null=True) + resolution_comment = models.TextField(blank=True, null=True) + # dynamic_fields ??? null, + tags = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) + # malicious_urls ??? null, + last_observed = models.DateTimeField(blank=True, null=True) + country_codes = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + cloud_providers = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + ipv4_addresses = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + # ipv6_addresses ??? null, + domain_names = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + service_ids = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + website_ids = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + asset_ids = ArrayField( + models.TextField(blank=True, null=False), blank=True, null=True + ) + certificate = models.JSONField(blank=True, null=True) + # { + # issuerName": "IOS-Self-Signed-Certificate-782645061", + # subjectName": "IOS-Self-Signed-Certificate-782645061", + # validNotBefore": 1398850008000, + # validNotAfter": 1577836800000, + # serialNumber": "1" + # }, + port_protocol = models.TextField(blank=True, null=True) + # business_unit_hierarchies + attack_surface_rule_name = models.TextField(blank=True, null=True) + remediation_guidance = models.TextField(blank=True, null=True) + asset_identifiers = models.JSONField(blank=True, null=True) + + business_units = models.ManyToManyField(XpanseBusinessUnits, related_name="alerts") + services = models.ManyToManyField(XpanseServicesMdl, related_name="alerts") + assets = models.ManyToManyField(XpanseAssetsMdl, related_name="alerts") + + class Meta: + """Set XpanseAlerts model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "xpanse_alerts_mdl" + + +class CpeVender(models.Model): + """Define CpeVender model.""" + + cpe_vender_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + vender_name = models.TextField(unique=True, blank=True, null=True) + + class Meta: + """Set CpeVender model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cpe_vender" + + +class CpeProduct(models.Model): + """Define CpeProduct model.""" + + cpe_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + cpe_product_name = models.TextField(blank=True, null=True) + version_number = models.TextField(blank=True, null=True) + cpe_vender = models.ForeignKey( + "CpeVender", on_delete=models.CASCADE, db_column="cpe_vender_uid", default=None + ) + + # Create linking table for many to many relationship + cves = models.ManyToManyField(Cve, related_name="products") + + class Meta: + """Set CpeProduct model metadata.""" + + app_label = 'dmz_mini_dl' + managed = manage_db + db_table = "cpe_product_mdl" + unique_together = (("cpe_product_name", "version_number"),) + + +# # THese are all views, so they shouldn't be generated via the ORM + +# # This should be a view not a table +# class VwPshttDomainsToRun(models.Model): +# """Define VwPshttDomainsToRun model.""" + +# sub_domain_uid = models.UUIDField(primary_key=True) +# sub_domain = models.TextField(blank=True, null=True) +# organization_uid = models.UUIDField() +# name = models.TextField(blank=True, null=True) + +# class Meta: +# """Set VwPshttDomainsToRun model metadata.""" + +# managed = False +# db_table = "vw_pshtt_domains_to_run" + + +# class VwBreachcompCredsbydate(models.Model): +# """Define VwBreachcompCredsbydate model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# mod_date = models.DateField(blank=True, null=True) +# no_password = models.BigIntegerField(blank=True, null=True) +# password_included = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwBreachcompCredsbydate model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_breachcomp_credsbydate" + + +# class VwDarkwebMentionsbydate(models.Model): +# """Define VwDarkwebMentionsbydate model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# count = models.BigIntegerField( +# db_column="Count", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebMentionsbydate model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_mentionsbydate" + + +# class VwShodanvulnsSuspected(models.Model): +# """Define VwShodanvulnsSuspected model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# organization = models.TextField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# port = models.TextField(blank=True, null=True) +# protocol = models.TextField(blank=True, null=True) +# type = models.TextField(blank=True, null=True) +# name = models.TextField(blank=True, null=True) +# potential_vulns = models.TextField( +# blank=True, null=True +# ) # This field type is a guess. +# mitigation = models.TextField(blank=True, null=True) +# timestamp = models.DateTimeField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# server = models.TextField(blank=True, null=True) +# tags = models.TextField(blank=True, null=True) # This field type is a guess. +# domains = models.TextField(blank=True, null=True) # This field type is a guess. +# hostnames = models.TextField(blank=True, null=True) # This field type is a guess. +# isn = models.TextField(blank=True, null=True) +# asn = models.IntegerField(blank=True, null=True) +# data_source = models.TextField(blank=True, null=True) + +# class Meta: +# """Set VwShodanvulnsSuspected model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_shodanvulns_suspected" + + +# class VwShodanvulnsVerified(models.Model): +# """Define VwShodanvulnsVerified model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# organization = models.TextField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# port = models.TextField(blank=True, null=True) +# protocol = models.TextField(blank=True, null=True) +# timestamp = models.DateTimeField(blank=True, null=True) +# cve = models.TextField(blank=True, null=True) +# severity = models.TextField(blank=True, null=True) +# cvss = models.DecimalField( +# max_digits=1000, decimal_places=1000, blank=True, null=True +# ) +# summary = models.TextField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# attack_vector = models.TextField(blank=True, null=True) +# av_description = models.TextField(blank=True, null=True) +# attack_complexity = models.TextField(blank=True, null=True) +# ac_description = models.TextField(blank=True, null=True) +# confidentiality_impact = models.TextField(blank=True, null=True) +# ci_description = models.TextField(blank=True, null=True) +# integrity_impact = models.TextField(blank=True, null=True) +# ii_description = models.TextField(blank=True, null=True) +# availability_impact = models.TextField(blank=True, null=True) +# ai_description = models.TextField(blank=True, null=True) +# tags = models.TextField(blank=True, null=True) # This field type is a guess. +# domains = models.TextField(blank=True, null=True) # This field type is a guess. +# hostnames = models.TextField(blank=True, null=True) # This field type is a guess. +# isn = models.TextField(blank=True, null=True) +# asn = models.IntegerField(blank=True, null=True) +# data_source = models.TextField(blank=True, null=True) +# banner = models.TextField(blank=True, null=True) +# version = models.TextField(blank=True, null=True) +# cpe = ArrayField( +# models.TextField(blank=True, null=True), blank=True, null=True +# ) + +# class Meta: +# """Set VwShodanvulnsVerified model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_shodanvulns_verified" + + +# class VwBreachcompBreachdetails(models.Model): +# """Define VwBreachcompBreachdetails model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# breach_name = models.TextField(blank=True, null=True) +# mod_date = models.DateField(blank=True, null=True) +# description = models.TextField(blank=True, null=True) +# breach_date = models.DateField(blank=True, null=True) +# password_included = models.BooleanField(blank=True, null=True) +# number_of_creds = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwBreachcompBreachdetails model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_breachcomp_breachdetails" + + +# class VwDarkwebSocmediaMostactposts(models.Model): +# """Define VwDarkwebSocmediaMostactposts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# comments_count = models.IntegerField( +# db_column="Comments Count", blank=True, null=True +# ) # Field name made lowercase. Field renamed to remove unsuitable characters. + +# class Meta: +# """Set VwDarkwebSocmediaMostactposts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_socmedia_mostactposts" + + +# class VwDarkwebMostactposts(models.Model): +# """Define VwDarkwebMostactposts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# comments_count = models.IntegerField( +# db_column="Comments Count", blank=True, null=True +# ) # Field name made lowercase. Field renamed to remove unsuitable characters. + +# class Meta: +# """Set VwDarkwebMostactposts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_mostactposts" + + +# class VwDarkwebAssetalerts(models.Model): +# """Define VwDarkwebAssetalerts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# events = models.BigIntegerField( +# db_column="Events", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebAssetalerts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_assetalerts" + + +# class VwDarkwebExecalerts(models.Model): +# """Define VwDarkwebExecalerts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# events = models.BigIntegerField( +# db_column="Events", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebExecalerts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_execalerts" + + +# class VwDarkwebThreatactors(models.Model): +# """Define VwDarkwebThreatactors model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# creator = models.TextField( +# db_column="Creator", blank=True, null=True +# ) # Field name made lowercase. +# grade = models.DecimalField( +# db_column="Grade", max_digits=1000, decimal_places=1000, blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebThreatactors model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_threatactors" + + +# class VwDarkwebPotentialthreats(models.Model): +# """Define VwDarkwebPotentialthreats model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. +# threats = models.TextField( +# db_column="Threats", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebPotentialthreats model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_potentialthreats" + + +# class VwDarkwebSites(models.Model): +# """Define VwDarkwebSites model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebSites model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_sites" + + +# class VwDarkwebInviteonlymarkets(models.Model): +# """Define VwDarkwebInviteonlymarkets model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebInviteonlymarkets model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_inviteonlymarkets" + + +# class VwDarkwebTopcves(models.Model): +# """Define VwDarkwebTopcves model.""" + +# top_cves_uid = models.UUIDField(primary_key=True) +# cve_id = models.TextField(blank=True, null=True) +# dynamic_rating = models.TextField(blank=True, null=True) +# nvd_base_score = models.TextField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# summary = models.TextField(blank=True, null=True) +# data_source_uid = models.UUIDField(blank=True, null=True) + +# class Meta: +# """Set VwDarkwebTopcves model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_topcves" + + +# class VwCidrs(models.Model): +# """Define VwCidrs model.""" + +# cidr_uid = models.UUIDField(primary_key=True) +# network = models.TextField(blank=True, null=True) # This field type is a guess. +# organization_uid = models.UUIDField(blank=True, null=True) +# data_source_uid = models.UUIDField(blank=True, null=True) +# insert_alert = models.TextField(blank=True, null=True) + +# class Meta: +# """Set VwCidrs model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_cidrs" + + +# class VwBreachcomp(models.Model): +# """Define VwBreachcomp model.""" + +# credential_exposures_uid = models.UUIDField(primary_key=True) +# email = models.TextField(blank=True, null=True) +# breach_name = models.TextField(blank=True, null=True) +# organization_uid = models.UUIDField(blank=True, null=True) +# root_domain = models.TextField(blank=True, null=True) +# sub_domain = models.TextField(blank=True, null=True) +# hash_type = models.TextField(blank=True, null=True) +# name = models.TextField(blank=True, null=True) +# login_id = models.TextField(blank=True, null=True) +# password = models.TextField(blank=True, null=True) +# phone = models.TextField(blank=True, null=True) +# data_source_uid = models.UUIDField(blank=True, null=True) +# description = models.TextField(blank=True, null=True) +# breach_date = models.DateField(blank=True, null=True) +# added_date = models.DateTimeField(blank=True, null=True) +# modified_date = models.DateTimeField(blank=True, null=True) +# data_classes = models.TextField( +# blank=True, null=True +# ) # This field type is a guess. +# password_included = models.BooleanField(blank=True, null=True) +# is_verified = models.BooleanField(blank=True, null=True) +# is_fabricated = models.BooleanField(blank=True, null=True) +# is_sensitive = models.BooleanField(blank=True, null=True) +# is_retired = models.BooleanField(blank=True, null=True) +# is_spam_list = models.BooleanField(blank=True, null=True) + +# class Meta: +# """Set VwBreachcomp model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_breachcomp" + + +# class VwOrgsTotalDomains(models.Model): +# """Define VwOrgsTotalDomains model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_root_domain = models.BigIntegerField(blank=True, null=True) +# num_sub_domain = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsTotalDomains model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_total_domains" + + +# class VwOrgsContactInfo(models.Model): +# """Define VwOrgsContactInfo model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# agency_name = models.TextField(blank=True, null=True) +# contact_type = models.TextField(blank=True, null=True) +# contact_name = models.TextField(blank=True, null=True) +# email = models.TextField(blank=True, null=True) +# phone = models.TextField(blank=True, null=True) +# date_pulled = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsContactInfo model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_contact_info" + + +# class VwOrgsTotalIps(models.Model): +# """Define VwOrgsTotalIps model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_ips = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsTotalIps model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_total_ips" + + +# class MatVwOrgsAllIps(models.Model): +# """Define MatVwOrgsAllIps model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# ip_addresses = ArrayField( +# models.GenericIPAddressField(blank=True, null=True), blank=True, null=True +# ) + +# class Meta: +# """Set MatVwOrgsAllIps model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "mat_vw_orgs_all_ips" + + +# class VwOrgsAttacksurface(models.Model): +# """Define VwOrgsAttacksurface model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_ports = models.BigIntegerField(blank=True, null=True) +# num_root_domain = models.BigIntegerField(blank=True, null=True) +# num_sub_domain = models.BigIntegerField(blank=True, null=True) +# num_ips = models.BigIntegerField(blank=True, null=True) +# num_cidrs = models.BigIntegerField(blank=True, null=True) +# num_ports_protocols = models.BigIntegerField(blank=True, null=True) +# num_software = models.BigIntegerField(blank=True, null=True) +# num_foreign_ips = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsAttacksurface model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_attacksurface" + + +# class VwOrgsTotalPorts(models.Model): +# """Define VwOrgsTotalPorts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_ports = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsTotalPorts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_total_ports" + + +# class VwIpsSubRootOrgInfo(models.Model): +# """VwIpsSubRootOrgInfo model class.""" + +# ip_hash = models.CharField(blank=True, null=True, max_length=255) +# ip = models.CharField(blank=True, null=True, max_length=255) +# origin_cidr = models.UUIDField(blank=True, null=True) +# organization_uid = models.UUIDField(blank=True, null=True) +# i_current = models.BooleanField(blank=True, null=True) +# sd_current = models.BooleanField(blank=True, null=True) + +# class Meta: +# """VwIpsSubRootOrgInfo model meta class.""" + +# managed = False +# db_table = "vw_ips_sub_root_org_info" + + +# class VwIpsCidrOrgInfo(models.Model): +# """VwIpsCidrOrgInfo model class.""" + +# ip_hash = models.CharField(blank=True, null=True, max_length=255) +# ip = models.CharField(blank=True, null=True, max_length=255) +# origin_cidr = models.UUIDField(blank=True, null=True) +# network = models.CharField(blank=True, null=True, max_length=255) +# organization_uid = models.UUIDField(blank=True, null=True) + +# class Meta: +# """VwIpsCidrOrgInfo model meta class.""" + +# managed = False +# db_table = "vw_ips_cidr_org_info" + + +# class VwPEScoreCheckNewCVE(models.Model): +# """VwPEScoreCheckNewCVE model class.""" + +# cve_name = models.CharField(blank=True, null=True, max_length=255) + +# class Meta: +# """VwPEScoreCheckNewCVE model meta class.""" + +# managed = False +# db_table = "vw_pescore_check_new_cve" + + +# # ---------- D-Score View Models ---------- +# # D-Score VS Cert View +# class VwDscoreVSCert(models.Model): +# """Define VwDscoreVSCert model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_cert = models.BigIntegerField(blank=True, null=True) +# num_monitor_cert = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscoreVSCert model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_vs_cert" + + +# # D-Score VS Mail View +# class VwDscoreVSMail(models.Model): +# """Define VwDscoreVSMail model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_valid_dmarc = models.BigIntegerField(blank=True, null=True) +# num_valid_spf = models.BigIntegerField(blank=True, null=True) +# num_valid_dmarc_or_spf = models.BigIntegerField(blank=True, null=True) +# total_mail_domains = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscoreVSMail model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_vs_mail" + + +# # D-Score PE IP View +# class VwDscorePEIp(models.Model): +# """Define VwDscorePEIp model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_ip = models.BigIntegerField(blank=True, null=True) +# num_monitor_ip = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscorePEIp model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_pe_ip" + + +# # D-Score PE Domain View +# class VwDscorePEDomain(models.Model): +# """Define VwDscorePEDomain model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_domain = models.BigIntegerField(blank=True, null=True) +# num_monitor_domain = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscorePEDomain model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_pe_domain" + + +# # D-Score WAS Webapp View +# class VwDscoreWASWebapp(models.Model): +# """Define VwDscoreWASWebapp model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_webapp = models.BigIntegerField(blank=True, null=True) +# num_monitor_webapp = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscoreWASWebapp model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_was_webapp" + + +# # ---------- I-Score View Models ---------- +# # I-Score VS Vuln View +# class VwIscoreVSVuln(models.Model): +# """Define VwIscoreVSVuln model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreVSVuln model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_vs_vuln" + + +# # I-Score VS Vuln Previous View +# class VwIscoreVSVulnPrev(models.Model): +# """Define VwIscoreVSVulnPrev model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) +# time_closed = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreVSVulnPrev model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_vs_vuln_prev" + + +# # I-Score PE Vuln View +# class VwIscorePEVuln(models.Model): +# """Define VwIscorePEVuln model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEVuln model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_vuln" + + +# # I-Score PE Cred View +# class VwIscorePECred(models.Model): +# """Define VwIscorePECred model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# password_creds = models.BigIntegerField(blank=True, null=True) +# total_creds = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePECred model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_cred" + + +# # I-Score PE Breach View +# class VwIscorePEBreach(models.Model): +# """Define VwIscorePEBreach model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# breach_count = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEBreach model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_breach" + + +# # I-Score PE Darkweb View +# class VwIscorePEDarkweb(models.Model): +# """Define VwIscorePEDarkweb model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# alert_type = models.CharField(blank=True, null=True, max_length=255) +# date = models.DateField(blank=True, null=True) +# Count = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEDarkweb model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_darkweb" + + +# # I-Score PE Protocol View +# class VwIscorePEProtocol(models.Model): +# """Define VwIscorePEProtocol model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# port = models.CharField(blank=True, null=True, max_length=255) +# ip = models.CharField(blank=True, null=True, max_length=255) +# protocol = models.CharField(blank=True, null=True, max_length=255) +# protocol_type = models.CharField(blank=True, null=True, max_length=255) +# date = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEProtocol model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_protocol" + + +# # I-Score WAS Vuln View +# class VwIscoreWASVuln(models.Model): +# """Define VwIscoreWASVuln model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) +# owasp_category = models.CharField(blank=True, null=True, max_length=255) + +# class Meta: +# """Set VwIscoreWASVuln model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_was_vuln" + + +# # I-Score WAS Vuln Previous View +# class VwIscoreWASVulnPrev(models.Model): +# """Define VwIscoreWASVulnPrev model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# was_total_vulns_prev = models.BigIntegerField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreWASVulnPrev model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_was_vuln_prev" + + +# # ---------- Misc. Score Related Models ---------- +# # vw_iscore_orgs_ip_counts view model (used for XS/S/M/L/XL orgs endpoints) +# class VwIscoreOrgsIpCounts(models.Model): +# """Define VwIscoreOrgsIpCounts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.CharField(blank=True, null=True, max_length=255) +# ip_count = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreOrgsIpCounts model metadata.""" + +# managed = False +# db_table = "vw_iscore_orgs_ip_counts"""" Django ORM models """ + diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py index cedc5a7f..a1575f7e 100644 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py @@ -57,6 +57,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "dataAPI.apps.DataapiConfig", + "dmz_mini_dl.apps.DmzMiniDlConfig", "bulkupload.apps.BulkuploadConfig", "home.apps.HomeConfig", "manage_login.apps.ManageLoginConfig", @@ -178,9 +179,19 @@ "PASSWORD": config("password"), "HOST": config("host"), "PORT": config("port"), + }, + "mini_data_lake": { + 'ENGINE': "django.db.backends.postgresql_psycopg2", # Replace with your database engine + 'NAME': config("mdl_database"), + 'USER': config('mdl_user'), + 'PASSWORD': config('mdl_password'), + 'HOST': config('mdl_host'), + 'PORT': config('mdl_port'), } } +DATABASE_ROUTERS = ['pe_reports_django.db_routers.MyAppRouter'] + # Celery settings CELERY_BROKER_URL = ( f"amqp://{config('RABBITMQ_USER')}:{config('RABBITMQ_PASS')}@localhost:5672/" From e8699b9befd841040a4620f74db17b6232d09088 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:54:32 -0400 Subject: [PATCH 2/7] add sync_dmz_mdl script add script that creates the empty datalake so that models can be migrated into it --- src/pe_reports/sync_dmz_mdl.py | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/pe_reports/sync_dmz_mdl.py diff --git a/src/pe_reports/sync_dmz_mdl.py b/src/pe_reports/sync_dmz_mdl.py new file mode 100644 index 00000000..54f7ee44 --- /dev/null +++ b/src/pe_reports/sync_dmz_mdl.py @@ -0,0 +1,62 @@ +import psycopg2 +from psycopg2 import sql +from data.config import staging_config + + +def create_database_and_user(db_name, new_user, new_password, admin_user, admin_password,admin_db, host='localhost', port='5432'): + # Connect to the PostgreSQL server using admin credentials + conn = psycopg2.connect(dbname=admin_db, user=admin_user, password=admin_password, host=host, port=port) + conn.autocommit = True # Enable autocommit to create databases and users + + try: + # Create a cursor object + cursor = conn.cursor() + + # Check if the new user exists + cursor.execute(sql.SQL("SELECT 1 FROM pg_roles WHERE rolname = %s"), [new_user]) + user_exists = cursor.fetchone() + + if not user_exists: + # Create the new user if it does not exist + cursor.execute(sql.SQL("CREATE USER {} WITH PASSWORD %s").format(sql.Identifier(new_user)), [new_password]) + print(f"User '{new_user}' created successfully.") + else: + print(f"User '{new_user}' already exists.") + + # Check if the database exists + cursor.execute(sql.SQL("SELECT 1 FROM pg_catalog.pg_database WHERE datname = %s"), [db_name]) + db_exists = cursor.fetchone() + + if not db_exists: + # Create the database if it does not exist + cursor.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(db_name))) + print(f"Database '{db_name}' created successfully.") + else: + print(f"Database '{db_name}' already exists.") + + # Grant all privileges on the new database to the new user + cursor.execute(sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {}").format(sql.Identifier(db_name), sql.Identifier(new_user))) + print(f"Granted all privileges on '{db_name}' to user '{new_user}'.") + + except Exception as e: + print(f"An error occurred: {e}") + + finally: + # Close the cursor and connection + cursor.close() + conn.close() + +if __name__ == "__main__": + DMZ_DIC = staging_config(section="mdl_dmz") + PE_DIC = staging_config(section="cd_crossfeed") + # Database connection parameters + DB_NAME = str(DMZ_DIC.get("mdl_database")) # Name of the database to create + NEW_USER = str(DMZ_DIC.get("mdl_user")) # New PostgreSQL username + NEW_PASSWORD = str(DMZ_DIC.get("mdl_password")) # New PostgreSQL password + ADMIN_USER = str(PE_DIC.get("user")) # Admin PostgreSQL username + ADMIN_PASSWORD = str(PE_DIC.get("password")) # Admin PostgreSQL password + ADMIN_DB = str(PE_DIC.get("database")) + HOST = str(DMZ_DIC.get("mdl_host")) # Host where PostgreSQL is running + PORT = str(DMZ_DIC.get("mdl_port")) # Port on which PostgreSQL is listening + + create_database_and_user(DB_NAME, NEW_USER, NEW_PASSWORD, ADMIN_USER, ADMIN_PASSWORD, ADMIN_DB, HOST, PORT) From 2de72eb53bb4c566e9fe23ad9a37ba76aa004b89 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Mon, 18 Nov 2024 08:30:26 -0700 Subject: [PATCH 3/7] Update shodan api calls to save to mdl Update shodan api calls to save to mdl as well as pe database --- .../dataAPI/schemas.py | 63 +- .../dataAPI/views.py | 319 +++++++--- .../dmz_mini_dl/models.py | 4 +- src/pe_source/data/pe_db/db_query_source.py | 43 +- src/pe_source/data/shodan_db/shodan_search.py | 30 +- src/pe_source/xpanse_alert_pull.py | 555 ++++++++++++++++++ src/pe_source/xpanse_org_sync.py | 147 +++++ 7 files changed, 1047 insertions(+), 114 deletions(-) create mode 100644 src/pe_source/xpanse_alert_pull.py create mode 100644 src/pe_source/xpanse_org_sync.py diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py index dc0ec933..f4f14f0a 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py @@ -3039,6 +3039,7 @@ class XpanseBusinessUnitsInsert(BaseModel): entity_type: Optional[str] = None region: Optional[str] = None rating: Optional[int] = None + cyhy_db_name: Optional[str] = None # --- xpanse endpoint, Issue 682 --- @@ -3929,15 +3930,21 @@ class Config: class ShodanAssetsInsert(BaseModel): """ShodanAssetsInsert schema class.""" - email: Optional[str] = None + asn: Optional[int] = None + domains: Optional[List[str]] = None + hostnames: Optional[List[str]] = None + ip: Optional[str] = None + isn: Optional[str] = None + organization: Optional[str] = None organizations_uid: Optional[str] = None - root_domain: Optional[str] = None - sub_domain: Optional[str] = None - modified_date: Optional[str] = None - breach_name: Optional[str] = None - credential_breaches_uid: Optional[str] = None + port: Optional[int] = None + product: Optional[str] = None + protocol: Optional[str] = None + tags: Optional[List[str]] = None + timestamp: Optional[str] = None + country_code: Optional[str] = None + location: Optional[str] = None data_source_uid: Optional[str] = None - name: Optional[str] = None class Config: """ShodanAssetsInsert schema config class.""" @@ -3963,15 +3970,43 @@ class Config: class ShodanVulnsInsert(BaseModel): """ShodanVulnsInsert schema class.""" - email: Optional[str] = None - organizations_uid: Optional[str] = None - root_domain: Optional[str] = None - sub_domain: Optional[str] = None - modified_date: Optional[str] = None - breach_name: Optional[str] = None - credential_breaches_uid: Optional[str] = None + + organizations_uid:Optional[str] = None + organization:Optional[str] = None + ip:Optional[str] = None + port:Optional[str] = None + protocol:Optional[str] = None + timestamp:Optional[str] = None + cve:Optional[str] = None + severity:Optional[str] = None + cvss:Optional[float] = None + summary:Optional[str] = None + product:Optional[str] = None + attack_vector:Optional[str] = None + av_description:Optional[str] = None + attack_complexity:Optional[str] = None + ac_description:Optional[str] = None + confidentiality_impact:Optional[str] = None + ci_description:Optional[str] = None + integrity_impact:Optional[str] = None + ii_description: Optional[str] = None + availability_impact: Optional[str] = None + ai_description: Optional[str] = None + tags: Optional[List[str]] = None + domains: Optional[List[str]] = None + hostnames: Optional[List[str]] = None + isn: Optional[str] = None + asn: Optional[int] = None data_source_uid: Optional[str] = None + type: Optional[str] = None name: Optional[str] = None + potential_vulns: Optional[List[str]] = None + mitigation: Optional[str] = None + server: Optional[str] = None + is_verified: Optional[bool] = None + banner: Optional[str] = None + version: Optional[str] = None + cpe: Optional[List[str]] = None class Config: """ShodanVulnsInsert schema config class.""" diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/views.py b/src/pe_reports/pe_reports_django_project/dataAPI/views.py index 2ee07eb1..0ee1aa79 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/views.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/views.py @@ -134,6 +134,19 @@ XpanseCveService, XpanseServices, ) +from dmz_mini_dl.models import( + Organization as MDL_Organization, + XpanseBusinessUnits as MDL_XpanseBusinessUnits, + XpanseAlerts as MDL_XpanseAlerts, + XpanseAssetsMdl as MDL_XpanseAssets, + XpanseServicesMdl as MDL_XpanseServices, + XpanseCvesMdl as MDL_XpanseCves, + XpanseCveServiceMdl as MDL_XpanseCveService, + ShodanAssets as MDL_ShodanAssets, + ShodanVulns as MDL_ShodanVulns, + DataSource as MDL_DataSource, + +) from jose import exceptions, jwt from redis import asyncio as aioredis from slowapi import Limiter @@ -5705,7 +5718,7 @@ def cred_exp_intelx_insert( "/xpanse_business_unit_insert_or_update", dependencies=[Depends(get_api_key)], # response_model=Dict[schemas.PshttDataBase], - tags=["Update or insert CVE data from NIST"], + tags=["Update or insert Xpanse Business Unit"], ) # @transaction.atomic def xpanse_business_unit_insert_or_update( @@ -5718,21 +5731,58 @@ def xpanse_business_unit_insert_or_update( try: userapiTokenverify(theapiKey=tokens) LOGGER.info(f"The api key submitted {tokens}") + + mapped_org = None + mdl_mapped_org = None + #TODO: only for fceb for now, need to update for when we add more xpanse stakeholder + # match = re.search(r'\((.*?)\)', data.entity_name) + if data.cyhy_db_name is not None: + try: + mapped_org = Organizations.objects.get(cyhy_db_name=data.cyhy_db_name) + except Organizations.DoesNotExist: + mapped_org = None + try: + mdl_mapped_org = MDL_Organization.objects.get(acronym=data.cyhy_db_name) + except MDL_Organization.DoesNotExist: + mdl_mapped_org = None + defaults={ + "state": data.state, + "county": data.county, + "city": data.city, + "sector": data.sector, + "entity_type": data.entity_type, + "region": data.region, + "rating": data.rating, + "cyhy_db_name": mapped_org + } + + mdl_defaults={ + "state": data.state, + "county": data.county, + "city": data.city, + "sector": data.sector, + "entity_type": data.entity_type, + "region": data.region, + "rating": data.rating, + "cyhy_db_name": mdl_mapped_org + } + # if mapped_org is not None: + # defaults["cyhy_db_name"] = mapped_org + + ( + mdl_business_unit_object, + mdl_created, + ) = MDL_XpanseBusinessUnits.objects.update_or_create( + entity_name=data.entity_name, + defaults=mdl_defaults, + ) ( business_unit_object, created, ) = XpanseBusinessUnits.objects.update_or_create( entity_name=data.entity_name, - defaults={ - "state": data.state, - "county": data.county, - "city": data.city, - "sector": data.sector, - "entity_type": data.entity_type, - "region": data.region, - "rating": data.rating, - }, + defaults=defaults ) if created: LOGGER.info( @@ -5753,13 +5803,43 @@ def xpanse_business_unit_insert_or_update( else: return {"message": "No api key was submitted"} +@api_router.get( + "/linked_xpanse_business_units", + dependencies=[ + Depends(get_api_key) + ], # Depends(RateLimiter(times=200, seconds=60))], + # response_model=List[schemas.OrganizationsFullTable], + tags=["Retrieve all Xpanse business units that link to an organization."], +) +def linked_xpanse_business_units(tokens: dict = Depends(get_api_key)): + """Call API endpoint to get Xpanse business unit that link to an organization.""" + # Check for API key + LOGGER.info(f"The api key submitted {tokens}") + if tokens: + try: + userapiTokenverify(theapiKey=tokens) + # If API key valid, make query + xpanse_business_units = list( + XpanseBusinessUnits.objects.filter(cyhy_db_name__isnull=False).values() + ) + # Convert data types to match response model + for row in xpanse_business_units: + row["xpanse_business_unit_uid"] = convert_uuid_to_string( + row["xpanse_business_unit_uid"] + ) + return xpanse_business_units + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} + # --- xpanse endpoint, Issue 682 --- @api_router.put( "/xpanse_alert_insert_or_update", dependencies=[Depends(get_api_key)], # response_model=Dict[schemas.PshttDataBase], - tags=["Update or insert CVE data from NIST"], + tags=["Update or insert alert data from Xpanse"], ) # @transaction.atomic def xpanse_alert_insert_or_update( @@ -5775,9 +5855,7 @@ def xpanse_alert_insert_or_update( LOGGER.info("Got into Xpanse Alert insert") # vender_prod_dict = data.vender_product - alert_object, created = XpanseAlerts.objects.update_or_create( - alert_id=data.alert_id, - defaults={ + alert_defaults = { "time_pulled_from_xpanse": data.time_pulled_from_xpanse, # "alert_id": data.alert_id, "detection_timestamp": data.detection_timestamp, @@ -5818,25 +5896,42 @@ def xpanse_alert_insert_or_update( # business_units: Optional[List[str]] = None # services: Optional[List[XpanseService]] = None # assets : Optional[List[XpanseAsset]] = None - }, + } + + # Create alerts in both databases + alert_object, created = XpanseAlerts.objects.update_or_create( + alert_id=data.alert_id, + defaults=alert_defaults, ) if created: LOGGER.info("new Xpanse alert record created for %s", data.alert_name) + mdl_alert_object, mdl_alert_created = MDL_XpanseAlerts.objects.update_or_create( + alert_id=data.alert_id, + defaults=alert_defaults, + ) + if mdl_alert_created: + LOGGER.info("new Xpanse alert record created in MDL for %s", data.alert_name) + + business_unit_list = [] + mdl_business_unit_list = [] for b_u in data.business_units: business_unit_list.append( XpanseBusinessUnits.objects.get(entity_name=b_u) ) - + mdl_business_unit_list.append( + MDL_XpanseBusinessUnits.objects.get(entity_name=b_u) + ) alert_object.business_units.set(business_unit_list) + mdl_alert_object.business_units.set(mdl_business_unit_list) + asset_list = [] + mdl_asset_list = [] for asset_data in data.assets: - asset_object, created = XpanseAssets.objects.update_or_create( - asm_id=asset_data.asm_id, - defaults={ + asset_defaults = { "asset_name": asset_data.asset_name, "asset_type": asset_data.asset_type, "last_observed": asset_data.last_observed, @@ -5863,17 +5958,25 @@ def xpanse_alert_insert_or_update( "externally_inferred_cves": asset_data.externally_inferred_cves, "explainers": asset_data.explainers, "tags": asset_data.tags, - }, + } + asset_object, created = XpanseAssets.objects.update_or_create( + asm_id=asset_data.asm_id, + defaults=asset_defaults, ) asset_list.append(asset_object) + mdl_asset_object, mdl_asset_created = MDL_XpanseAssets.objects.update_or_create( + asm_id=asset_data.asm_id, + defaults=asset_defaults, + ) + mdl_asset_list.append(mdl_asset_object) alert_object.assets.set(asset_list) + mdl_alert_object.assets.set(mdl_asset_list) services_list = [] + mdl_services_list = [] for service_data in data.services: - service_object, created = XpanseServices.objects.update_or_create( - service_id=service_data.service_id, - defaults={ + service_defaults={ "service_name": service_data.service_name, "service_type": service_data.service_type, "ip_address": service_data.ip_address, @@ -5891,30 +5994,35 @@ def xpanse_alert_insert_or_update( "externally_inferred_cves": service_data.externally_inferred_cves, "service_key": service_data.service_key, "service_key_type": service_data.service_key_type, - }, + } + service_object, created = XpanseServices.objects.update_or_create( + service_id=service_data.service_id, + defaults=service_defaults, + ) + mdl_service_object, mdl_service_created = MDL_XpanseServices.objects.update_or_create( + service_id=service_data.service_id, + defaults=service_defaults, ) - LOGGER.info(service_data) + # LOGGER.info(service_data) if service_data.cves is not None: for cve_data, cve_match_data in service_data.cves: LOGGER.info(cve_data) LOGGER.info(cve_match_data) - cve_object, created = XpanseCves.objects.update_or_create( - cve_id=cve_data.cve_id, - defaults={ + cve_defaults = { "cvss_score_v2": cve_data.cvss_score_v2, "cve_severity_v2": cve_data.cve_severity_v2, "cvss_score_v3": cve_data.cvss_score_v3, "cve_severity_v3": cve_data.cve_severity_v3, - }, + } + cve_object, created = XpanseCves.objects.update_or_create( + cve_id=cve_data.cve_id, + defaults=cve_defaults, ) - - ( - cve_match_object, - created, - ) = XpanseCveService.objects.update_or_create( - xpanse_inferred_cve=cve_object, - xpanse_service=service_object, - defaults={ + mdl_cve_object, mdl_cve_created = MDL_XpanseCves.objects.update_or_create( + cve_id=cve_data.cve_id, + defaults=cve_defaults, + ) + cve_service_default = { "inferred_cve_match_type": cve_match_data.inferred_cve_match_type, "product": cve_match_data.product, "confidence": cve_match_data.confidence, @@ -5923,14 +6031,33 @@ def xpanse_alert_insert_or_update( "activity_status": cve_match_data.activity_status, "first_observed": cve_match_data.first_observed, "last_observed": cve_match_data.last_observed, - }, + } + ( + cve_match_object, + created, + ) = XpanseCveService.objects.update_or_create( + xpanse_inferred_cve=cve_object, + xpanse_service=service_object, + defaults=cve_service_default, + ) + ( + mdl_cve_match_object, + mdl_cve_service_created, + ) = MDL_XpanseCveService.objects.update_or_create( + xpanse_inferred_cve=mdl_cve_object, + xpanse_service=mdl_service_object, + defaults=cve_service_default, ) services_list.append(service_object) + mdl_services_list.append(mdl_service_object) alert_object.services.set(services_list) - alert_object.save() + + mdl_alert_object.services.set(mdl_services_list) + mdl_alert_object.save() + # for vender, product_list in vender_prod_dict.items(): # vender_obj, vender_created = CpeVender.objects.update_or_create( @@ -6984,55 +7111,95 @@ def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = tags=["Insert Shodan data into the shodan_assets table."], ) def shodan_assets_insert( - data: schemas.ShodanAssetsInsertInput, tokens: dict = Depends(get_api_key) -): + data: schemas.ShodanAssetsInsertInput, tokens: dict = Depends(get_api_key)): """Insert Shodan data into the shodan_assets table using the API endpoint.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: userapiTokenverify(theapiKey=tokens) - # If API key valid, insert intelx data - create_cnt = 0 - for row in data.exp_data: + # If API key is valid, proceed with the operation + update_create_count = 0 + for row in data.asset_data: row_dict = row.__dict__ + try: - # Check if record already exists - ShodanAssets.objects.get( - organizations_uid=row_dict["organizations_uid"], - ip=row_dict["ip"], - port=row_dict["port"], - protocol=row_dict["protocol"], + org_instance = Organizations.objects.get(organizations_uid=row_dict["organizations_uid"]) + acronym = org_instance.acronym # Assuming 'acronym' is a field in Organizations model + # Assuming ExternalOrganizations is a model that matches organization acronym to an organization_id + mdl_org = MDL_Organization.objects.get(acronym=acronym) + try: + mdl_data_source = MDL_DataSource.objects.get(name="Shodan") + except DataSource.DoesNotExist: + LOGGER.warning(f"DataSource with UID {row_dict['data_source_uid_id']} not found.") + mdl_data_source = None # Set to None if DataSource is not found + + mdl_asset_fields = { + "asn": row_dict.get("asn"), + "domains": row_dict.get("domains", []), + "hostnames": row_dict.get("hostnames", []), + "isn": row_dict.get("isn"), + "organization_name": row_dict.get("organization"), + "product": row_dict.get("product"), + "tags": row_dict.get("tags", []), + "country_code": row_dict.get("country_code"), + "location": row_dict.get("location"), + "data_source": mdl_data_source, + } + + mdl_obj, created = MDL_ShodanAssets.objects.update_or_create( + organization=mdl_org, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], timestamp=row_dict["timestamp"], + defaults=mdl_asset_fields ) - # If record already exists, do nothing - except CredentialExposures.DoesNotExist: - # If record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( - organizations_uid=row_dict["organizations_uid"] - ) - ShodanAssets.objects.create( - # Need to fill this out - # credential_exposures_uid=uuid.uuid1(), - email=row_dict["email"], - organizations_uid=curr_org_inst, - root_domain=row_dict["root_domain"], - sub_domain=row_dict["sub_domain"], - modified_date=row_dict["modified_date"], - breach_name=row_dict["breach_name"], - name=row_dict["name"], + except: + LOGGER.warning(f"Shodan Asset failed to save to MDL.") + + try: + # Prepare the fields that will be used for create or update (only non-unique fields here) + asset_fields = { + "asn": row_dict.get("asn"), + "domains": row_dict.get("domains", []), + "hostnames": row_dict.get("hostnames", []), + "isn": row_dict.get("isn"), + "organization": row_dict.get("organization"), + "product": row_dict.get("product"), + "tags": row_dict.get("tags", []), + "country_code": row_dict.get("country_code"), + "location": row_dict.get("location"), + "data_source_uid": row_dict.get("data_source_uid"), + } + + # Use 'update_or_create' to either create or update the record + obj, created = ShodanAssets.objects.update_or_create( + organizations_uid=row_dict["organizations_uid"], # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=row_dict["timestamp"], + defaults=asset_fields ) - create_cnt += 1 - # Return success message - return ( - str(create_cnt) - + " records created in the credential_exposures table" - ) + + if created: + update_create_count += 1 + except: + LOGGER.warning(f"Shodan Asset failed to save to PE DB.") + continue + + # Return the success message with the count of created/updated records + return {"message": f"{update_create_count} records created/updated in the shodan_assets table."} + except ObjectDoesNotExist: - LOGGER.info("API key expired please try again") + LOGGER.info("API key expired or invalid. Please try again.") + return {"message": "Invalid API key or expired. Please try again."} + except Exception as e: + LOGGER.error(f"Error: {str(e)}") + return {"message": "An error occurred while processing the request."} else: - return {"message": "No api key was submitted"} - + return {"message": "No API key was submitted."} # --- insert_shodan_vulns(), Issue 017 atc-framework --- @api_router.put( @@ -7053,7 +7220,7 @@ def shodan_vulns_insert( userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx data create_cnt = 0 - for row in data.exp_data: + for row in data.vuln_data: row_dict = row.__dict__ try: CredentialExposures.objects.get( diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py index 8c67758d..79047a5c 100644 --- a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py @@ -2667,7 +2667,9 @@ class XpanseBusinessUnits(models.Model): xpanse_business_unit_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) entity_name = models.TextField(unique=True, blank=True, null=True) cyhy_db_name = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="cyhy_db_name", to_field="acronym" + "Organization", on_delete=models.CASCADE, db_column="cyhy_db_name", to_field="acronym", + null=True, # Allow NULL values + blank=True ) state = models.TextField(blank=True, null=True) county = models.TextField(blank=True, null=True) diff --git a/src/pe_source/data/pe_db/db_query_source.py b/src/pe_source/data/pe_db/db_query_source.py index 14192d6a..413011ae 100644 --- a/src/pe_source/data/pe_db/db_query_source.py +++ b/src/pe_source/data/pe_db/db_query_source.py @@ -610,7 +610,36 @@ def insert_or_update_business_unit(business_unit_dict): except json.decoder.JSONDecodeError as err: LOGGER.error(err) +# --- Issue 699 pe-reports --- +def get_linked_xpanse_business_units(): + """ + Query API to retrieve data for all business units that link to an org. + Return: + All linked xpanse business units + """ + # Endpoint info + endpoint_url = pe_api_url + "linked_xpanse_business_units" + headers = { + "Content-Type": "application/json", + "access_token": pe_api_key, + } + try: + result = requests.get(endpoint_url, headers=headers).json() + # Process data and return + return result + except requests.exceptions.HTTPError as errh: + LOGGER.error(errh) + except requests.exceptions.ConnectionError as errc: + LOGGER.error(errc) + except requests.exceptions.Timeout as errt: + LOGGER.error(errt) + except requests.exceptions.RequestException as err: + LOGGER.error(err) + except json.decoder.JSONDecodeError as err: + LOGGER.error(err) + + # --- Issue 682 --- def api_xpanse_alert_insert(xpanse_alert_dict): """ @@ -1414,7 +1443,7 @@ def query_PE_subs(org_uid): # --- Issue 016 atc-framework --- -def insert_shodan_assets(dataframe, table, thread, org_name, failed): +def insert_shodan_assets(asset_data): """ Query API to insert Shodan data into the shodan_assets table. @@ -1422,15 +1451,15 @@ def insert_shodan_assets(dataframe, table, thread, org_name, failed): data: Dataframe of the shodan data to be inserted into shodan_assets. """ # Endpoint info - endpoint_url = pe_api_url + "shodan_assets_inserts" + endpoint_url = pe_api_url + "shodan_assets_insert" headers = { "Content-Type": "application/json", "access_token": pe_api_key, } - data = json.dumps({"asset_data": dataframe}) + data = json.dumps({"asset_data": asset_data}) try: # Call endpoint - result = requests.post(endpoint_url, headers=headers, data=data).json() + result = requests.put(endpoint_url, headers=headers, data=data).json() # Process data and return LOGGER.info(result) except requests.exceptions.HTTPError as errh: @@ -1446,7 +1475,7 @@ def insert_shodan_assets(dataframe, table, thread, org_name, failed): # --- Issue 017 atc-framework --- -def insert_shodan_vulns(dataframe, table, thread, org_name, failed): +def insert_shodan_vulns(vuln_data): """ Query API to insert Shodan data into the shodan_vulns table. @@ -1459,10 +1488,10 @@ def insert_shodan_vulns(dataframe, table, thread, org_name, failed): "Content-Type": "application/json", "access_token": pe_api_key, } - data = json.dumps({"vuln_data": dataframe}) + data = json.dumps({"vuln_data": vuln_data}) try: # Call endpoint - result = requests.post(endpoint_url, headers=headers, data=data).json() + result = requests.put(endpoint_url, headers=headers, data=data).json() # Process data and return LOGGER.info(result) except requests.exceptions.HTTPError as errh: diff --git a/src/pe_source/data/shodan_db/shodan_search.py b/src/pe_source/data/shodan_db/shodan_search.py index 286f9446..5341cff6 100644 --- a/src/pe_source/data/shodan_db/shodan_search.py +++ b/src/pe_source/data/shodan_db/shodan_search.py @@ -14,7 +14,8 @@ from pe_source.data.pe_db.db_query_source import ( # get_ips_dhs,; get_ips_hhs,; get_ips_nasa, get_data_source_uid, get_ips, - insert_shodan_data, + insert_shodan_assets, + insert_shodan_vulns ) LOGGER = logging.getLogger(__name__) @@ -103,6 +104,7 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): ) # Loop through chunks and query Shodan + source_uid = get_data_source_uid("Shodan") for i, ip_chunk in enumerate(ip_chunks): count = i + 1 try_count = 1 @@ -167,7 +169,8 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "is_verified": False, "cpe": d.get("cpe", None), "banner": d.get("data", None), - "version": d.get("version", None) + "version": d.get("version", None), + "data_source_uid": source_uid } ) elif d["_shodan"]["module"] in risky_ports: @@ -211,7 +214,8 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "is_verified": False, "cpe": d.get("cpe", None), "banner": d.get("data", None), - "version": d.get("version", None) + "version": d.get("version", None), + "data_source_uid": source_uid } ) @@ -231,7 +235,8 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "tags": r["tags"], "timestamp": d["timestamp"], "country_code": location["country_code"], - "location": str(location) + "location": str(location), + "data_source_uid": source_uid } ) @@ -269,21 +274,14 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): LOGGER.info("{} {}/{} complete - {}".format(thread_name, count, tot, org_name)) - df = pd.DataFrame(data) - risk_df = pd.DataFrame(risk_data) - vuln_df = pd.DataFrame(vuln_data) - all_vuln_df = vuln_df.append(risk_df, ignore_index=True) + all_vulns = vuln_data + risk_data # Grab the data source uid and add to each dataframe - source_uid = get_data_source_uid("Shodan") - df["data_source_uid"] = source_uid - risk_df["data_source_uid"] = source_uid - vuln_df["data_source_uid"] = source_uid - all_vuln_df["data_source_uid"] = source_uid + # Insert data into the PE database - failed = insert_shodan_data(df, "shodan_assets", thread_name, org_name, failed) - failed = insert_shodan_data( - all_vuln_df, "shodan_vulns", thread_name, org_name, failed + failed = insert_shodan_assets(data, "shodan_assets", thread_name, org_name, failed) + failed = insert_shodan_vulns( + all_vulns, "shodan_vulns", thread_name, org_name, failed ) return failed diff --git a/src/pe_source/xpanse_alert_pull.py b/src/pe_source/xpanse_alert_pull.py new file mode 100644 index 00000000..b3780622 --- /dev/null +++ b/src/pe_source/xpanse_alert_pull.py @@ -0,0 +1,555 @@ +"""Scan to track Xpanse alerts and incidents in the P&E database. + +Usage: + xpanse_alert_pull.py [--orgs=ORG_LIST] [--last_modified=MOD_TIME] [--log-level=LEVEL] + +Options: + -h --help Show this message. + -o --orgs=ORG_LIST A semicolon-separated list of Xpanse business_units. + If not specified, data will be gathered for all business_units. + Orgs in the list must match the names in Xpanse. E.g. Culberson County, Texas; DHS - Citizenship and Immigration Services (CIS) - CISA + [default: all] + -m --last_modified=MOD_TIME An integer in timestamp epoch milliseconds. + Scan will pull all alerts and assets updated since the provided time. + If not specified, data will be gathered for all assets and alerts. E.g. 1696996800000 + [default: all_time] + -l --log-level=LEVEL If specified, then the log level will be set to + the specified value. Valid values are "debug", "info", + "warning", "error", and "critical". [default: info] +""" + +# Standard Python Libraries +import csv +import datetime +import json +import logging +import sys +from typing import Any, Dict + +# Third-Party Libraries +from _version import __version__ +from data.pe_db.db_query_source import ( # api_pull_xpanse_vulns, + api_xpanse_alert_insert, + get_linked_xpanse_business_units, +) +import docopt +import pytz +import requests +from schema import And, Or, Schema, SchemaError, Use + +# cisagov Libraries +import pe_reports +from pe_reports.data.config import staging_config + +API_DIC = staging_config(section="xpanse") +xpanse_url = "https://api-cisa-xpanse.crtx.gv.paloaltonetworks.com/public_api/" +# "https://api-cisa.crtx.federal.paloaltonetworks.com/public_api/" +api_key = API_DIC.get("api_key") +auth_id = API_DIC.get("auth_id") + +LOGGER = logging.getLogger(__name__) + + +# def pull_asset_data(xpanse_asset_id_list=[]): +# """Pull asset data from the Xpanse API.""" +# assets = [] + +# url = xpanse_url + "v1/assets/get_asset_internet_exposure" +# request_data = {"asm_id_list": xpanse_asset_id_list} + +# payload = json.dumps({"request_data": request_data}) + +# headers = { +# "x-xdr-auth-id": auth_id, +# "Authorization": api_key, +# "Content-Type": "application/json", +# } + +# response = requests.request("POST", url, headers=headers, data=payload) + +# resp_dict = response.json() + +# for asset in resp_dict["reply"]["details"]: +# asset_dict = format_asset(asset) +# assets.append(asset_dict) + +# return assets +# # save_asset(asset_dict) + + +def pull_alerts_data(linked_org_list, business_units_list=[]): + """Pull alerts data from the Xpanse API.""" + url = xpanse_url + "v2/alerts/get_alerts_multi_events" + + if len(business_units_list) == 0: + business_units_list = list(map(lambda d: d['entity_name'], linked_org_list)) + + for org in business_units_list: + request_data = { + "use_page_token": True, + "search_from":0, + "search_to": 5000, + } + filters = [] + LOGGER.info("Running Xpanse alert pull on %s", org) + filters.append( + {"field": "business_units_list", "operator": "in", "value": [org]} + ) + + # TODO maybe change this to be creation time + # if last_modified != "all_time": + # filters.append({ + # "field": "" + # }) + + if len(filters) > 0: + request_data["filters"] = filters + + payload = json.dumps({"request_data": request_data}) + + headers = { + "x-xdr-auth-id": auth_id, + "Authorization": api_key, + "Content-Type": "application/json", + } + try: + response = requests.request("POST", url, headers=headers, data=payload) + resp_dict = response.json() + + page_token = resp_dict["reply"]["next_page_token"] + LOGGER.info( + "The current org has %s alerts", resp_dict["reply"]["total_count"] + ) + + formatted_alerts = format_alerts(resp_dict["reply"]["alerts"]) + + for alert in formatted_alerts: + api_xpanse_alert_insert(alert) + + while page_token is not None: + request_data = {"next_page_token": page_token} + + payload = json.dumps({"request_data": request_data}) + + response = requests.request("POST", url, headers=headers, data=payload) + resp_dict = response.json() + + page_token = resp_dict["reply"]["next_page_token"] + + formatted_alerts = format_alerts(resp_dict["reply"]["alerts"]) + for alert in formatted_alerts: + api_xpanse_alert_insert(alert) + + except Exception as e: + LOGGER.error("Error querying assets for %s: %s.", org, e) + + +# def format_asset(asset): +# """Format Xpanse asset to match db tables.""" +# asset_dict = { +# "asm_id": asset.get("asm_ids", None), +# "asset_name": asset.get("name", None), +# "asset_type": asset.get("type", None), +# "last_observed": asset.get("last_observed", None), +# "first_observed": asset.get("first_observed", None), +# "externally_detected_providers": asset.get( +# "externally_detected_providers", None +# ), +# "created": asset.get("created", None), +# "ips": asset.get("ips", None), +# "active_external_services_types": asset.get( +# "active_external_services_types", None +# ), +# "domain": asset.get("domain", None), +# "certificate_issuer": asset.get("certificate_issuer", None), +# "certificate_algorithm": asset.get("certificate_algorithm", None), +# "certificate_classifications": asset.get("certificate_classifications", None), +# "resolves": asset.get("resolves", None), +# "top_level_asset_mapper_domain": asset["details"].get( +# "topLevelAssetMapperDomain", None +# ), +# "domain_asset_type": asset["details"].get("domainAssetType", None), +# "is_paid_level_domain": asset["details"].get("isPaidLevelDomain", None), +# "domain_details": asset["details"].get("domainDetails", None), +# "dns_zone": asset.get("dnsZone", None), +# "latest_sampled_ip": asset.get("latestSampledIp", None), +# "recent_ips": asset.get("recentIps", None), +# "external_services": asset.get("external_services", None), +# "externally_inferred_vulnerability_score": asset.get( +# "externally_inferred_vulnerability_score", None +# ), +# "externally_inferred_cves": asset.get("externally_inferred_cves", None), +# "explainers": asset.get("explainers", None), +# "tags": asset.get("tags", None), +# } + + # return asset_dict + + +def format_alerts(alerts): + """Format Xpanse alerts to match db tables.""" + alert_services_dict = {} + service_ids = [] + for alert in alerts: + try: + service_ids += alert.get('service_ids', []) + alert_services_dict[alert['alert_id']] = alert.get('service_ids', []) + except: + continue + + services = [] + max_n = 5000 + if service_ids is not None: + service_id_chunks = [ + service_ids[i : i + max_n] for i in range(0, len(service_ids), max_n) + ] + + for service_chunk in service_id_chunks: + max_retries = 3 + retry_delay = 5 + for retry_count in range(max_retries): + try: + service_response = pull_service_data(service_chunk) + if service_response is not None: + break + except Exception as e: + # Log the error message + LOGGER.error(f"Error querying services: {e}") + + # If it's not the last retry, wait for the retry_delay before retrying + if retry_count < max_retries - 1: + LOGGER.info("Retrying...") + time.sleep(retry_delay) + else: + # If it's the last retry, set it to None and it will skip to the next chunk + service_response = None + + + if service_response is None: + continue + for service_obj in service_response: + cves = [] + if service_obj["details"].get("inferredCvesObserved", None) is not None: + for cve in service_obj["details"].get("inferredCvesObserved", None): + cves.append( + ( + { + "cve_id": cve["inferredCve"]["cveId"], + "cvss_score_v2": cve["inferredCve"].get( + "cvssScoreV2", None + ), + "cve_severity_v2": cve["inferredCve"].get( + "cveSeverityV2", None + ), + "cvss_score_v3": cve["inferredCve"].get( + "cvssScoreV3", None + ), + "cve_severity_v3": cve["inferredCve"].get( + "cveSeverityV3", None + ), + }, + { + "inferred_cve_match_type": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("inferredCveMatchType", None), + "product": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("product", None), + "confidence": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("confidence", None), + "vendor": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("vendor", None), + "version_number": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("version", None), + "activity_status": cve.get( + "activityStatus", None + ), + "first_observed": cve.get( + "firstObserved", None + ), + "last_observed": cve.get( + "lastObserved", None + ), + }, + ) + ) + services.append( + { + "service_id": service_obj.get("service_id", None), + "service_name": service_obj.get("service_name", None), + "service_type": service_obj.get("service_type", None), + "ip_address": service_obj.get( + "ip_address", None + ), # list of ip strings + "domain": service_obj.get("domain", None), # list of ? + "externally_detected_providers": service_obj.get( + "externally_detected_providers", None + ), + "is_active": service_obj.get("is_active", None), + "first_observed": service_obj.get("first_observed", None), + "last_observed": service_obj.get("last_observed", None), + "port": service_obj.get("port", None), + "protocol": service_obj.get("protocol", None), + "active_classifications": service_obj.get( + "active_classifications", None + ), # list of strings + "inactive_classifications": service_obj.get( + "inactive_classifications", None + ), + "discovery_type": service_obj.get("discovery_type", None), + "externally_inferred_vulnerability_score": service_obj.get( + "externally_inferred_vulnerability_score", None + ), + "externally_inferred_cves": service_obj.get( + "externally_inferred_cves", None + ), + "service_key": service_obj["details"].get("serviceKey", None), + "service_key_type": service_obj["details"].get( + "serviceKeyType", None + ), + # providerDetails + # certificates + # domains + # ips + # classifications + # tlsVersions + "cves": cves + # enrichedObservationSource + # ip_ranges + } + ) + + alert_list = [] + for alert in alerts: + tags = (alert.get("tags", None),) + business_units_list = [] + try: + for tag in tags[0]: + if tag.startswith("BU:"): + business_units_list.append(tag[3:].strip()) + # print(business_units_list) + except: + business_units_list = [] + + assets = [] + ##Uncomment to track assets + # asset_ids = alert["asset_ids"] + # if asset_ids is not None: + # asset_id_chunks = [ + # asset_ids[i : i + max_n] for i in range(0, len(asset_ids), max_n) + # ] + + # for asset_chunk in asset_id_chunks: + # asset_response = pull_asset_data(asset_chunk) + # assets += asset_response + current_services = [] + try: + for service in alert.get("service_ids", []): + service_identified = next((d for d in services if d.get('service_id') == service), None) + if service_identified: + current_services.append(service_identified) + except: + pass + + + alert_dict = { + "time_pulled_from_xpanse": datetime.datetime.utcnow().replace(tzinfo=pytz.utc), + "alert_id": alert.get("alert_id", None), + "detection_timestamp": alert.get("detection_timestamp", None), + "alert_name": alert.get("name", None), + # endpoint_id ???, + "description": alert.get("description", None), + # "endpoint_id": alert.get('endpoint_id', None), + # "host_ip": alert.get('host_ip', None), + "host_name": alert.get("host_name", None), + "alert_action": alert.get("action", None), + # user_name ??? null, + # mac_addresses ??? null, + # source ??? null, + "action_pretty": alert.get("action_pretty", None), + # category ??? null, + # project ??? null, + # cloud_provider ??? null, + # resource_sub_type ??? null, + # resource_type ??? null, + "action_country": alert.get("action_country", None), # list type + # event_type ??? null, + # is_whitelisted ??? null, + # image_name ??? null, + # action_local_ip ??? null, + # action_local_port ??? null, + # action_external_hostname ??? null, + # action_remote_ip ??? null, + "action_remote_port": alert.get("action_remote_port", None), # list type + # "matching_service_rule_id ??? null, + "starred": alert.get("starred", None), + "external_id": alert.get("external_id", None), + "related_external_id": None, + "alert_occurrence": None, + "severity": alert.get("severity", None), + "matching_status": alert.get("matching_status", None), + # end_match_attempt_ts ??? null, + "local_insert_ts": alert.get("local_insert_ts", None), + "last_modified_ts": alert.get("last_modified_ts") if alert.get("last_modified_ts") is not None else alert.get("local_insert_ts", None), + "case_id": alert.get("case_id", None), + # deduplicate_tokens ??? null, + # filter_rule_id ??? null, + # event_id ??? null, + "event_timestamp": alert.get("event_timestamp", None), # list type + # action_local_ip_v6 ??? null, + # action_remote_ip_v6 ??? null, + "alert_type": alert.get("alert_type", None), + "resolution_status": alert.get("resolution_status", None), + "resolution_comment": alert.get("resolution_comment", None), + # dynamic_fields ??? null, + "tags": alert.get("tags", None), + # malicious_urls ??? null, + "last_observed": alert.get("last_observed", None), + "country_codes": alert.get("country_codes", None), # list type + "cloud_providers": alert.get("cloud_providers", None), # list type + "ipv4_addresses": alert.get("ipv4_addresses", None), # list type + # ipv6_addresses ??? null, + "domain_names": alert.get("domain_names", None), # list type + "service_ids": alert.get("service_ids", None), # already addressed above + # "website_ids": alert.get('website_ids', None), + "asset_ids": alert.get("asset_ids", None), # list type + "certificate": alert.get("certificate", None), + # { + # issuerName": "IOS-Self-Signed-Certificate-782645061", + # subjectName": "IOS-Self-Signed-Certificate-782645061", + # validNotBefore": 1398850008000, + # validNotAfter": 1577836800000, + # serialNumber": "1" + # }, + "port_protocol": alert.get("port_protocol", None), + # business_unit_hierarchies + # "business_unit_hierarchies": alert.get('business_unit_hierarchies', None), #list of BUs + # attack_surface_rule_name ??? null, + # remediation_guidance ??? null, + "attack_surface_rule_name": alert.get("attack_surface_rule_name", None), + "remediation_guidance": alert.get("remediation_guidance", None), + "asset_identifiers": alert.get( + "asset_identifiers", None + ), # messy list of objects + "business_units": business_units_list, + "services": current_services, + "assets": assets, + } + + if alert_dict["external_id"] is not None: + alert_dict["related_external_id"] = "-".join( + alert_dict["external_id"].split("-")[:-1] + ) + alert_dict["alert_occurrence"] = ( + int(alert_dict["external_id"].split("-")[-1]) / 2 + ) + else: + alert_dict["related_external_id"] = None + alert_dict["alert_occurrence"] = None + + alert_list.append(alert_dict) + return alert_list + +def pull_service_data(service_id_list): + """Pull service info from the Xpanse API using a service_id.""" + url = xpanse_url + "v1/assets/get_external_service" + request_data = {"service_id_list": service_id_list} + + payload = json.dumps({"request_data": request_data}) + + headers = { + "x-xdr-auth-id": auth_id, + "Authorization": api_key, + "Content-Type": "application/json", + } + + response = requests.request("POST", url, headers=headers, data=payload) + + resp_dict = response.json() + + return resp_dict.get("reply", {}).get("details", None) + + +def run_xpanse_scans(last_modified, orgs_list): + """Run Xpanse scans.""" + if orgs_list != "all": + orgs_list = orgs_list.split(";") + else: + orgs_list = [] + + linked_org_list = get_linked_xpanse_business_units() + + pull_alerts_data(linked_org_list, orgs_list) + # api_pull_xpanse_vulns(orgs_list[0], datetime.datetime(2023, 10, 10, 1, 00)) + + return 1 + + +def main(): + """Launch Xpanse scans.""" + args: Dict[str, str] = docopt.docopt(__doc__, version=__version__) + + schema: Schema = Schema( + { + "--log-level": And( + str, + Use(str.lower), + lambda n: n in ("debug", "info", "warning", "error", "critical"), + error="Possible values for --log-level are " + + "debug, info, warning, error, and critical.", + ), + str: object, # Don't care about other keys, if any + } + ) + + try: + validated_args: Dict[str, Any] = schema.validate(args) + except SchemaError as err: + # Exit because one or more of the arguments were invalid + print(err, file=sys.stderr) + sys.exit(1) + + # Assign validated arguments to variables + log_level: str = validated_args["--log-level"] + + # Set up logging + logging.basicConfig( + filename=pe_reports.CENTRAL_LOGGING_FILE, + filemode="a", + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%m/%d/%Y %I:%M:%S", + level=log_level.upper(), + ) + + run_xpanse_scans( + validated_args["--last_modified"], + validated_args["--orgs"], + ) + +def print_start_time(): + global start_time + start_time = datetime.datetime.now() + print(f"Script started at: {start_time}") + +# Function to print the end time and calculate duration +def print_end_time(): + end_time = datetime.datetime.now() + print(f"Script ended at: {end_time}") + + # Calculate duration + duration = end_time - start_time + + # Convert duration to hours, minutes, seconds + hours, remainder = divmod(duration.seconds, 3600) + minutes, seconds = divmod(remainder, 60) + + print(f"Script took {hours} hours, {minutes} minutes, and {seconds} seconds to run.") + + +if __name__ == "__main__": + print_start_time() + main() + print_end_time() \ No newline at end of file diff --git a/src/pe_source/xpanse_org_sync.py b/src/pe_source/xpanse_org_sync.py new file mode 100644 index 00000000..ca091f5a --- /dev/null +++ b/src/pe_source/xpanse_org_sync.py @@ -0,0 +1,147 @@ +"""Script to save Xpanse Business Units and link to cyhy orgs. + +Usage: + xpanse_org_sync.py XPANSE_ORG_CSV_PATH [--log-level=LEVEL] + +Options: + -h --help Show this message. + XPANSE_ORG_CSV_PATH The path to the XPANSE Business_unit CSV. + -l --log-level=LEVEL If specified, then the log level will be set to + the specified value. Valid values are "debug", "info", + "warning", "error", and "critical". [default: info] +""" +# Standard Python Libraries +import csv +import datetime +import json +import logging +import sys +from typing import Any, Dict +import re + +# Third-Party Libraries +from _version import __version__ +from data.pe_db.db_query_source import ( # api_pull_xpanse_vulns, + insert_or_update_business_unit, +) +import docopt +import pytz +import requests +from schema import And, Or, Schema, SchemaError, Use + +# cisagov Libraries +import pe_reports +from pe_reports.data.config import staging_config + +LOGGER = logging.getLogger(__name__) + +def extract_last_substring_in_square_brackets(input_string): + # Define the regular expression pattern + pattern = r'\[([^\]]+)\]' # Matches [ followed by any characters that are not ], followed by ] + + # Find all matches of the pattern in the input_string + matches = re.findall(pattern, input_string) + # Return the last match or None if no matches are found + return matches[-1] if matches else None + +def sync_orgs(orgs_csv): + """Sync orgs to the database.""" + try: + print(orgs_csv) + orgs_reader = csv.DictReader(orgs_csv) + except FileNotFoundError: + LOGGER.error("No file found at provided filepath.") + except Exception as e: + LOGGER.error("Unknown error reading csv: %s", e) + + # map_dict = {} + # try: + # file_path = 'xpanse_org_map.csv' + # with open(file_path, 'r', encoding='utf-8-sig') as csvfile: + # map_reader = csv.DictReader(csvfile) + + # for d in map_reader: + # print(d) + # # Add key-value pair to the result dictionary + # print(d['Xpanse Business Unit Name']) + # print(d['CISA Short Code']) + # map_dict[d['Xpanse Business Unit Name']] = d['CISA Short Code'] # Print each row as a dictionary + # except FileNotFoundError: + # print(f"Error: File '{file_path}' not found.") + # except Exception as e: + # print(f"Error reading '{file_path}': {e}") + # # Initialize an empty dictionary to store the result + # print(map_dict) + + for org in orgs_reader: + try: + # if map_dict.get(org["Entity Name"].strip(), None) is not None: + # print(map_dict.get(org["Entity Name"].strip())) + cyhy_db_name = extract_last_substring_in_square_brackets(org["Entity Name"].strip()) + if cyhy_db_name: + print(cyhy_db_name) + business_unit_dict = { + "entity_name": org["Entity Name"].strip(), + "state": org["State"].strip(), + "county": org["County"].strip(), + "city": org["City"].strip(), + "sector": org["Sector"].strip(), + "entity_type": org["Entity Type"].strip(), + "region": org["Region"].strip(), + "rating": int(org["Rating"].strip()), + "cyhy_db_name": cyhy_db_name + } + + response = insert_or_update_business_unit(business_unit_dict) + except Exception as e: + LOGGER.error('Failure saving %s', org["Entity Name"]) + LOGGER.error("Unknown error saving: %s", e) + continue + + +def main(): + """Launch Xpanse scans.""" + args: Dict[str, str] = docopt.docopt(__doc__, version=__version__) + + + schema: Schema = Schema( + { + "--log-level": And( + str, + Use(str.lower), + lambda n: n in ("debug", "info", "warning", "error", "critical"), + error="Possible values for --log-level are " + + "debug, info, warning, error, and critical.", + ), + "XPANSE_ORG_CSV_PATH": Or( + None, + Use(open, error="XPANSE_ORG_CSV_PATH should point to a readable CSV"), + ) + } + ) + + + try: + + validated_args: Dict[str, Any] = schema.validate(args) + + except SchemaError as err: + # Exit because one or more of the arguments were invalid + print(err, file=sys.stderr) + sys.exit(1) + + + log_level: str = validated_args["--log-level"] + + logging.basicConfig( + filename=pe_reports.CENTRAL_LOGGING_FILE, + filemode="a", + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%m/%d/%Y %I:%M:%S", + level=log_level.upper(), + ) + + sync_orgs(validated_args["XPANSE_ORG_CSV_PATH"]) + +if __name__ == "__main__": + main() \ No newline at end of file From 9f21b3fa0d3db1991a63c1c460a7a8b589d9aa88 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 26 Nov 2024 11:40:12 -0500 Subject: [PATCH 4/7] Fixes django project, resolves numerous config issues and allows the app to run --- setup.py | 8 +- src/pe_reports/Dockerfile | 49 ++ src/pe_reports/__init__.py | 105 ---- src/pe_reports/docker-compose.yaml | 82 ++- .../config/nginx_config_conf.d/nginx.conf | 12 + .../config/rabbitmq.conf | 4 + .../data/rabbit@atc_rabbitmq-feature_flags | 2 + .../rabbitmq/data/rabbit@atc_rabbitmq.pid | 1 + .../rabbit@atc_rabbitmq/cluster_nodes.config | 1 + .../vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost | 1 + .../msg_store_persistent/0.rdq | 0 .../msg_store_transient/0.rdq | 0 .../628WB79CIFDYO9LJI6DKMI09L/recovery.dets | Bin 0 -> 5464 bytes .../nodes_running_at_shutdown | 1 + .../quorum/rabbit@atc_rabbitmq/meta.dets | Bin 0 -> 5464 bytes .../quorum/rabbit@atc_rabbitmq/names.dets | Bin 0 -> 5464 bytes .../rabbit_durable_exchange.DCD | Bin 0 -> 1329 bytes .../rabbit_durable_queue.DCD | Bin 0 -> 94 bytes .../rabbit_durable_route.DCD | 1 + .../rabbit_runtime_parameters.DCD | Bin 0 -> 195 bytes .../data/rabbit@atc_rabbitmq/rabbit_serial | 1 + .../rabbit_topic_permission.DCD | 1 + .../data/rabbit@atc_rabbitmq/rabbit_user.DCD | Bin 0 -> 232 bytes .../rabbit_user_permission.DCD | Bin 0 -> 190 bytes .../data/rabbit@atc_rabbitmq/rabbit_vhost.DCD | Bin 0 -> 172 bytes .../data/rabbit@atc_rabbitmq/schema.DAT | Bin 0 -> 33650 bytes .../data/rabbit@atc_rabbitmq/schema_version | 1 + .../dataAPI/tasks.py | 2 +- .../dataAPI/views.py | 135 +++-- .../dmz_mini_dl/models.py | 19 +- .../pe_reports_django/asgi.py | 11 +- .../pe_reports_django/db_routers.py | 32 ++ .../pe_reports_django/requirements.txt | 483 +++++++++--------- .../pe_reports_django/settings.py | 4 +- .../static/CISAImage.png | Bin .../static/admin/css/autocomplete.css | 0 .../static/admin/css/base.css | 0 .../static/admin/css/changelists.css | 0 .../static/admin/css/dark_mode.css | 0 .../static/admin/css/dashboard.css | 0 .../static/admin/css/fonts.css | 0 .../static/admin/css/forms.css | 0 .../static/admin/css/login.css | 0 .../static/admin/css/nav_sidebar.css | 0 .../static/admin/css/responsive.css | 0 .../static/admin/css/responsive_rtl.css | 0 .../static/admin/css/rtl.css | 0 .../css/vendor/select2/LICENSE-SELECT2.md | 0 .../admin/css/vendor/select2/select2.css | 0 .../admin/css/vendor/select2/select2.min.css | 0 .../static/admin/css/widgets.css | 0 .../static/admin/fonts/LICENSE.txt | 0 .../static/admin/fonts/README.txt | 0 .../admin/fonts/Roboto-Bold-webfont.woff | Bin .../admin/fonts/Roboto-Light-webfont.woff | Bin .../admin/fonts/Roboto-Regular-webfont.woff | Bin .../static/admin/img/LICENSE | 0 .../static/admin/img/README.txt | 0 .../static/admin/img/calendar-icons.svg | 0 .../static/admin/img/gis/move_vertex_off.svg | 0 .../static/admin/img/gis/move_vertex_on.svg | 0 .../static/admin/img/icon-addlink.svg | 0 .../static/admin/img/icon-alert.svg | 0 .../static/admin/img/icon-calendar.svg | 0 .../static/admin/img/icon-changelink.svg | 0 .../static/admin/img/icon-clock.svg | 0 .../static/admin/img/icon-deletelink.svg | 0 .../static/admin/img/icon-no.svg | 0 .../static/admin/img/icon-unknown-alt.svg | 0 .../static/admin/img/icon-unknown.svg | 0 .../static/admin/img/icon-viewlink.svg | 0 .../static/admin/img/icon-yes.svg | 0 .../static/admin/img/inline-delete.svg | 0 .../static/admin/img/search.svg | 0 .../static/admin/img/selector-icons.svg | 0 .../static/admin/img/sorting-icons.svg | 0 .../static/admin/img/tooltag-add.svg | 0 .../static/admin/img/tooltag-arrowright.svg | 0 .../static/admin/js/SelectBox.js | 0 .../static/admin/js/SelectFilter2.js | 0 .../static/admin/js/actions.js | 0 .../admin/js/admin/DateTimeShortcuts.js | 0 .../admin/js/admin/RelatedObjectLookups.js | 0 .../static/admin/js/autocomplete.js | 0 .../static/admin/js/calendar.js | 0 .../static/admin/js/cancel.js | 0 .../static/admin/js/change_form.js | 0 .../static/admin/js/collapse.js | 0 .../static/admin/js/core.js | 0 .../static/admin/js/filters.js | 0 .../static/admin/js/inlines.js | 0 .../static/admin/js/jquery.init.js | 0 .../static/admin/js/nav_sidebar.js | 0 .../static/admin/js/popup_response.js | 0 .../static/admin/js/prepopulate.js | 0 .../static/admin/js/prepopulate_init.js | 0 .../static/admin/js/urlify.js | 0 .../static/admin/js/vendor/jquery/LICENSE.txt | 0 .../static/admin/js/vendor/jquery/jquery.js | 0 .../admin/js/vendor/jquery/jquery.min.js | 0 .../static/admin/js/vendor/select2/LICENSE.md | 0 .../static/admin/js/vendor/select2/i18n/af.js | 0 .../static/admin/js/vendor/select2/i18n/ar.js | 0 .../static/admin/js/vendor/select2/i18n/az.js | 0 .../static/admin/js/vendor/select2/i18n/bg.js | 0 .../static/admin/js/vendor/select2/i18n/bn.js | 0 .../static/admin/js/vendor/select2/i18n/bs.js | 0 .../static/admin/js/vendor/select2/i18n/ca.js | 0 .../static/admin/js/vendor/select2/i18n/cs.js | 0 .../static/admin/js/vendor/select2/i18n/da.js | 0 .../static/admin/js/vendor/select2/i18n/de.js | 0 .../admin/js/vendor/select2/i18n/dsb.js | 0 .../static/admin/js/vendor/select2/i18n/el.js | 0 .../static/admin/js/vendor/select2/i18n/en.js | 0 .../static/admin/js/vendor/select2/i18n/es.js | 0 .../static/admin/js/vendor/select2/i18n/et.js | 0 .../static/admin/js/vendor/select2/i18n/eu.js | 0 .../static/admin/js/vendor/select2/i18n/fa.js | 0 .../static/admin/js/vendor/select2/i18n/fi.js | 0 .../static/admin/js/vendor/select2/i18n/fr.js | 0 .../static/admin/js/vendor/select2/i18n/gl.js | 0 .../static/admin/js/vendor/select2/i18n/he.js | 0 .../static/admin/js/vendor/select2/i18n/hi.js | 0 .../static/admin/js/vendor/select2/i18n/hr.js | 0 .../admin/js/vendor/select2/i18n/hsb.js | 0 .../static/admin/js/vendor/select2/i18n/hu.js | 0 .../static/admin/js/vendor/select2/i18n/hy.js | 0 .../static/admin/js/vendor/select2/i18n/id.js | 0 .../static/admin/js/vendor/select2/i18n/is.js | 0 .../static/admin/js/vendor/select2/i18n/it.js | 0 .../static/admin/js/vendor/select2/i18n/ja.js | 0 .../static/admin/js/vendor/select2/i18n/ka.js | 0 .../static/admin/js/vendor/select2/i18n/km.js | 0 .../static/admin/js/vendor/select2/i18n/ko.js | 0 .../static/admin/js/vendor/select2/i18n/lt.js | 0 .../static/admin/js/vendor/select2/i18n/lv.js | 0 .../static/admin/js/vendor/select2/i18n/mk.js | 0 .../static/admin/js/vendor/select2/i18n/ms.js | 0 .../static/admin/js/vendor/select2/i18n/nb.js | 0 .../static/admin/js/vendor/select2/i18n/ne.js | 0 .../static/admin/js/vendor/select2/i18n/nl.js | 0 .../static/admin/js/vendor/select2/i18n/pl.js | 0 .../static/admin/js/vendor/select2/i18n/ps.js | 0 .../admin/js/vendor/select2/i18n/pt-BR.js | 0 .../static/admin/js/vendor/select2/i18n/pt.js | 0 .../static/admin/js/vendor/select2/i18n/ro.js | 0 .../static/admin/js/vendor/select2/i18n/ru.js | 0 .../static/admin/js/vendor/select2/i18n/sk.js | 0 .../static/admin/js/vendor/select2/i18n/sl.js | 0 .../static/admin/js/vendor/select2/i18n/sq.js | 0 .../admin/js/vendor/select2/i18n/sr-Cyrl.js | 0 .../static/admin/js/vendor/select2/i18n/sr.js | 0 .../static/admin/js/vendor/select2/i18n/sv.js | 0 .../static/admin/js/vendor/select2/i18n/th.js | 0 .../static/admin/js/vendor/select2/i18n/tk.js | 0 .../static/admin/js/vendor/select2/i18n/tr.js | 0 .../static/admin/js/vendor/select2/i18n/uk.js | 0 .../static/admin/js/vendor/select2/i18n/vi.js | 0 .../admin/js/vendor/select2/i18n/zh-CN.js | 0 .../admin/js/vendor/select2/i18n/zh-TW.js | 0 .../admin/js/vendor/select2/select2.full.js | 0 .../js/vendor/select2/select2.full.min.js | 0 .../admin/js/vendor/xregexp/LICENSE.txt | 0 .../static/admin/js/vendor/xregexp/xregexp.js | 0 .../admin/js/vendor/xregexp/xregexp.min.js | 0 .../static/css/custom.css | 0 .../static/js/htmx.js | 0 .../static/power.svg | 0 .../templates/base.html | 134 +++-- .../templates/home.html | 0 170 files changed, 585 insertions(+), 505 deletions(-) create mode 100644 src/pe_reports/Dockerfile create mode 100755 src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf create mode 100755 src/pe_reports/pe_reports_django_project/config/rabbitmq.conf create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags create mode 100644 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid create mode 100644 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent/0.rdq create mode 100644 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient/0.rdq create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/recovery.dets create mode 100644 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/nodes_running_at_shutdown create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/meta.dets create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/names.dets create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_exchange.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_queue.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_runtime_parameters.DCD create mode 100644 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_serial create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_topic_permission.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user_permission.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_vhost.DCD create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema.DAT create mode 100755 src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/dataAPI/tasks.py mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/dataAPI/views.py mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py create mode 100755 src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/CISAImage.png mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/autocomplete.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/base.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/changelists.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/dark_mode.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/dashboard.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/fonts.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/forms.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/login.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/nav_sidebar.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/responsive.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/responsive_rtl.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/rtl.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/LICENSE-SELECT2.md mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.min.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/css/widgets.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/fonts/LICENSE.txt mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/fonts/README.txt mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Bold-webfont.woff mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Light-webfont.woff mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Regular-webfont.woff mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/LICENSE mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/README.txt mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/calendar-icons.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_off.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_on.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-addlink.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-alert.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-calendar.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-changelink.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-clock.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-deletelink.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-no.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown-alt.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-viewlink.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/icon-yes.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/inline-delete.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/search.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/selector-icons.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/sorting-icons.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-add.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-arrowright.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/SelectBox.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/SelectFilter2.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/actions.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/admin/DateTimeShortcuts.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/admin/RelatedObjectLookups.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/autocomplete.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/calendar.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/cancel.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/change_form.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/collapse.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/core.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/filters.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/inlines.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/jquery.init.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/nav_sidebar.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/popup_response.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate_init.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/urlify.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/LICENSE.txt mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.min.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/LICENSE.md mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/af.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ar.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/az.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bg.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bn.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bs.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ca.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/cs.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/da.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/de.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/dsb.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/el.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/en.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/es.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/et.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/eu.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fa.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fi.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fr.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/gl.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/he.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hi.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hr.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hsb.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hu.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hy.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/id.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/is.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/it.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ja.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ka.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/km.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ko.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lt.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lv.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/mk.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ms.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nb.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ne.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nl.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pl.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ps.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt-BR.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ro.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ru.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sk.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sl.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sq.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr-Cyrl.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sv.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/th.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tk.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tr.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/uk.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/vi.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-CN.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-TW.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.min.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/LICENSE.txt mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.min.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/css/custom.css mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/js/htmx.js mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/static/power.svg mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/templates/base.html mode change 100644 => 100755 src/pe_reports/pe_reports_django_project/templates/home.html diff --git a/setup.py b/setup.py index 997067cc..2e227cf0 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,7 @@ def get_version(version_file): "matplotlib == 3.3.4", "nested-lookup", "openpyxl", - "pandas == 1.1.5", + "pandas", "pdfkit", "psutil", "psycopg2-binary", @@ -152,7 +152,11 @@ def get_version(version_file): "shodan == 1.27.0", "sshtunnel", "sslyze>=5.0.0", - # "spacy", + "spacy", + "spacy-loogers", + "spacy-legacy", + "spacy-transformers", + "spacy-alignments" "nltk", "beautifulsoup4", "sublist3r", diff --git a/src/pe_reports/Dockerfile b/src/pe_reports/Dockerfile new file mode 100644 index 00000000..10587de3 --- /dev/null +++ b/src/pe_reports/Dockerfile @@ -0,0 +1,49 @@ +# Use the Python 3.10.2 image as the base image + +FROM python:3.10.2 + +# Install required tools +#RUN apt-get update && apt-get install -y curl build-essential + +# Install Rust and Cargo +#RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + +# Add Rust to PATH +#ENV PATH="/root/.cargo/bin:${PATH}" + +# Install required tools +RUN apt-get update && apt-get install -y bash g++ gcc make redis redis-tools + +# Create non-root user +RUN useradd -m -u 1001 atc_api + +# Upgrade pip and certifi +RUN python3 -m pip install --upgrade pip && pip install --upgrade certifi + + + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# Create working directory with correct ownership +RUN mkdir /code && chown atc_api:atc_api /code +WORKDIR /code + +# Install dependencies +COPY --chown=atc_api:atc_api ./pe_reports_django_project/pe_reports_django/requirements.txt /code/ +RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --upgrade numpy spacy +# Copy the project code +COPY --chown=atc_api:atc_api pe_reports_django_project /code + +# Switch to non-root user +USER atc_api + +# Set Django environment variable +ENV DJANGO_SETTINGS_MODULE=pe_reports_django.settings + +# Run the application +CMD uvicorn --workers 4 pe_reports_django.asgi:app1 --host 0.0.0.0 --port 8000 --reload + + diff --git a/src/pe_reports/__init__.py b/src/pe_reports/__init__.py index c9fbe7ac..e69de29b 100644 --- a/src/pe_reports/__init__.py +++ b/src/pe_reports/__init__.py @@ -1,105 +0,0 @@ -"""The pe_reports library.""" -# We disable a Flake8 check for "Module imported but unused (F401)" here because -# although this import is not directly used, it populates the value -# package_name.__version__, which is used to get version information about this -# Python package. - -# Standard Python Libraries -import logging -from logging.handlers import RotatingFileHandler -import os - -# Third-Party Libraries -# from celery import Celery -from flask import Flask, render_template -from flask_login import LoginManager -from flask_migrate import Migrate -from flask_sqlalchemy import SQLAlchemy - -# cisagov Libraries -from pe_reports.data.config import config - -from ._version import __version__ # noqa: F401 - -# Stakeholder views -# from pe_reports.home.views import home_blueprint -# from pe_reports.report_gen.views import report_gen_blueprint -# from pe_reports.stakeholder.views import stakeholder_blueprint -# from pe_reports.stakeholder_bulk_upload.views import stakeholder_bulk_upload_blueprint -# from pe_reports.stakeholder_full.views import stakeholder_full_blueprint - - -params = config() -login_manager = LoginManager() -# Flask implementation -app = Flask(__name__) -app.config["SECRET_KEY"] = os.getenv("FLASK_SECRET_KEY", "dev") -app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False -app.config[ - "SQLALCHEMY_DATABASE_URI" -] = f'postgresql+psycopg2://{params["user"]}:{params["password"]}@{params["host"]}:{params["port"]}/{params["database"]}' - - -# Configure the redis server -# app.config["CELERY_BROKER_URL"] = "redis://localhost:6379/0" -# app.config["CELERY_RESULT_BACKEND"] = "redis://localhost:6379/0" -app.config["UPLOAD_FOLDER"] = "src/pe_reports/uploads/" -app.config["ALLOWED_EXTENSIONS"] = {"txt", "csv"} - -CENTRAL_LOGGING_FILE = "pe_reports_logging.log" -DEBUG = False -# Setup Logging -"""Set up logging and call the run_pe_script function.""" -if DEBUG is True: - level = "DEBUG" -else: - level = "INFO" - -# Logging will rotate at 2GB -logging.basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - datefmt="%m/%d/%Y %I:%M:%S", - level=level, - handlers=[ - RotatingFileHandler(CENTRAL_LOGGING_FILE, maxBytes=2000000, backupCount=10) - ], -) - -app.config["LOGGER"] = logging.getLogger(__name__) - -# with open('username.txt', 'w') as file: -# file.write(pwd.getpwuid(os.getuid())[0]) - -# Creates a Celery object -# celery = Celery(app.name, broker=app.config["CELERY_BROKER_URL"]) -# celery.conf.update(app.config) - -# Config DB -db = SQLAlchemy(app) -Migrate(app, db) - -# TODO: Add a login page in the future. Issue #207 contains details -# login_manager.init_app(app) -# login_manager.login_view = "login" - -__all__ = ["app", "pages", "report_generator", "stylesheet"] - - -# Register the flask apps -# app.register_blueprint(stakeholder_blueprint) -# app.register_blueprint(stakeholder_full_blueprint) -# app.register_blueprint(stakeholder_bulk_upload_blueprint) -# app.register_blueprint(report_gen_blueprint) -# TODO: Add login blueprint. Issue #207 contains details -# app.register_blueprint(manage_login_blueprint) -# app.register_blueprint(home_blueprint) - - -@app.errorhandler(404) -def page_not_found(e): - return render_template("404.html") - - -if __name__ == "__main__": - logging.info("The program has started...") - app.run(host="127.0.0.1", debug=DEBUG, port=8000) diff --git a/src/pe_reports/docker-compose.yaml b/src/pe_reports/docker-compose.yaml index a3b5de8c..5cf56c83 100644 --- a/src/pe_reports/docker-compose.yaml +++ b/src/pe_reports/docker-compose.yaml @@ -1,16 +1,15 @@ ---- -version: "3.8" +version: "3.9" services: - pe_reports_rabbitmq: - container_name: pe_reports_rabbitmq - hostname: pe_reports_rabbitmq + atc_rabbitmq: + container_name: atc_rabbitmq + hostname: atc_rabbitmq image: rabbitmq:3.8.14-management restart: always ports: - - 15672:15672 - - 5672:5672 + - 15674:15672 + - 5674:5672 env_file: - - ../../src/pe_reports/pe_reports_django_project/.env + - ./pe_reports_django_project/.env environment: - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER} - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASS} @@ -24,37 +23,70 @@ services: - LANGUAGE=C.UTF-8 - LC_ALL=C.UTF-8 volumes: - - /home/ubuntu/pe-reports/src/pe_reports/data/rabbitmq/data:/var/lib/rabbitmq/mnesia:rw - - /var/www/pe-reports/src/pe_reports/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - ./pe_reports_django_project/data/rabbitmq/data:/var/lib/rabbitmq/mnesia:rw + - ./pe_reports_django_project/config:/etc/rabbitmq:rw networks: - - pe_reports_rabbitmq_network + - atc_network - pe_reports_redis: - container_name: pe_reports_redis - hostname: pe_reports_redis - image: redis:latest + atc_redis: + container_name: atc_redis + hostname: atc_redis + image: redis:7.4.1 restart: always ports: - - 6379:6379 + - 6378:6379 volumes: - - redis_data:/data + - ./pe_reports_django_project/redis_data:/data networks: - - pe_reports_redis_network + - atc_network + + web: + build: . + container_name: atc_web + volumes: + - ./pe_reports_django_project:/code + - ./pe_reports_django_project/config:/code/config + ports: + - "8002:8000" + env_file: + - pe_reports_django_project/.env + environment: + - DJANGO_SETTINGS_MODULE:pe_reports_django.settings + networks: + - atc_network + + nginx: image: nginx:1.25.0 + container_name: atc_nginx ports: - - "8089:8089" + - "8091:8091" volumes: - - ./config/nginx_config_conf.d:/etc/nginx/conf.d - - ./pe_reports_django_project/static:/var/www/pe-reports/static - - ./pe-reports:/var/www/pe-reports + - ./pe_reports_django_project/config/nginx_config_conf.d:/etc/nginx/conf.d networks: - - pe_reports_nginx_network + - atc_network + depends_on: + - web + + # database: + # image: postgres + # restart: always + # env_file: + # - ./pe_reports_django_project/.env + # networks: + # - atc_network + # volumes: + # - ./pe_reports_django_project/postgres_data:/var/lib/postgresql/data + # ports: + # - 5437:5432 + # container_name: atc_database networks: - pe_reports_network: + atc_network: driver: bridge + + volumes: - redis_data: {} + redis_data: {} \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf b/src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf new file mode 100755 index 00000000..6008eb2c --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf @@ -0,0 +1,12 @@ +server { + listen 8091; + server_name localhost; + + location / { + proxy_pass http://web:8000; # Assuming 'web' is the service name and '8000' is the port where Gunicorn runs + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/config/rabbitmq.conf b/src/pe_reports/pe_reports_django_project/config/rabbitmq.conf new file mode 100755 index 00000000..9c184353 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/config/rabbitmq.conf @@ -0,0 +1,4 @@ +loopback_users.guest = false +listeners.tcp.default = 5672 +default_pass = guest1 +default_user = admin diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags new file mode 100755 index 00000000..a9a883dd --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags @@ -0,0 +1,2 @@ +[implicit_default_bindings,maintenance_mode_status,quorum_queue,user_limits, + virtual_host_metadata]. diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid new file mode 100644 index 00000000..88101bcc --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid @@ -0,0 +1 @@ +465 \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config new file mode 100644 index 00000000..48a2fbb4 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config @@ -0,0 +1 @@ +{[rabbit@atc_rabbitmq],[rabbit@atc_rabbitmq]}. diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost new file mode 100755 index 00000000..35ec3b9d --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost @@ -0,0 +1 @@ +/ \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent/0.rdq b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent/0.rdq new file mode 100755 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient/0.rdq b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient/0.rdq new file mode 100644 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/recovery.dets b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/recovery.dets new file mode 100755 index 0000000000000000000000000000000000000000..c1ae47e3b05f8128913ddbd12be673bebf7cc413 GIT binary patch literal 5464 zcmZQz5Vc@fefB*A0|O%zb0U~vF`zIAFfuT~7!aDSiJR2M?+wQhXC(r{vY8fVAT7gAut*OqaiRF0;3@?8UmvsFd71*Aut*O dqaiRF0wW>>1ehV?rV7xpNFE5^z!KiJR2M?+wQhXC(r{vY8fVAT7gAut*OqaiRF0;3@?8UmvsFd71*Aut*O dqaiRF0wW>>1ehV?rV7xpNFE5^z!KiJR2M?+wQhXC(r{vY8fVAT7gAut*OqaiRF0;3@?8UmvsFd71*Aut*O dqaiRF0wW>>1ehV?rV7xpNFE5^z!KQgAqL)niC5r#YA2*Z zsw$zVikN(Nw(dUv^xbI3^Tqx36aa4c-PPr@3=k$+F^?rNB#p3#0%KcPfH#Xy7vP^o zvvY()%{WiZ1v4UU!@QIp2ME46c(+F!!tYPp^{wq1BJ9(XkW9@<`Zm#26}1+W+c%?= zU$?F>VR;#0s!14xpbLDXYeLvfSXQmh2ZV90kdjmp{v|eUAc9|4q}{Ag=jCy#3~9wO zoA(ktc7k)uLRw%gxysJ?zk(aW2~$Pw2zS9Sf@a1M;VDL){_i=2_U`t`(%E?cU&qz&7Ni9lYU{6U-i3bW~GcX(K8DulC snCKZ=rZ5N>B_<_hmN+DqB*#OTxrG_bNem2ZN0Jzr{dJNUcx8S70C;&Bm;e9( literal 0 HcmV?d00001 diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD new file mode 100755 index 00000000..f8dd237a --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD @@ -0,0 +1 @@ +cXM \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_runtime_parameters.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_runtime_parameters.DCD new file mode 100755 index 0000000000000000000000000000000000000000..15854308204bea62ed1c4d5a76fef7d0ce052dfc GIT binary patch literal 195 zcmXxcK?;IE7>41$MhvP|-6B#FTGTEf0x6AFwuxb8{y_&F%TX7tyGlptI^CgxYWMQN z_w@S25$AXW00~bI;UjkuhUGP*xnw8}!T<$QI}|_ye^G$5^n*3R)G*FfyJ4-M-_&(Z zIDq@&K#p?`@a1M;VDL){_i=2_U`t`(%E?cU&qz&7Ni9lYU{6U-i3bW~GcX(K8DulC znCKZ=rZ5N>B_<_hmN+DqB*#OTxrG_bNem2ZN0JzrecO^4*ck(XrZ&M%<;~11NiE7t z%!w~8PA$p>%CV-GrWTigSSp+;zilku`lkLiTA;_c&E9Wv`H}-OZ8?s$NSOE3D=C*V z%xL+Q160VE!oZuDlAD>ASzJ<-Sdw3qmBJtgu|K{bvADQAzbGX>Be6ImGcP^9I3v-> K)T{)kjR63OYDe+_ literal 0 HcmV?d00001 diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user_permission.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user_permission.DCD new file mode 100755 index 0000000000000000000000000000000000000000..a58a4453679006d3d2cb5d3ce6bd115b74d4f68b GIT binary patch literal 190 zcmZQ%VrEH>@a1M;VDL){_i=2_U`t`(%E?cU&qz&7Ni9lYU{6U-i3bW~GcX(K8DulC znCKZ=rZ5N>B_<_hmN+DqB*#OTxrG_bNem2ZN0Jzr{dJNUcz3Y@O^tw<%ACT$Us{}6 y6km{9l$%*xoSC1O0Tkl`iM7nkG$6|<(7rb1YZ`WY-hX@qK!G?ShdDg^*ZPcno6 literal 0 HcmV?d00001 diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_vhost.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_vhost.DCD new file mode 100755 index 0000000000000000000000000000000000000000..21746957b7234eae86df04b4ec4f553316db6345 GIT binary patch literal 172 zcmZQ%VrEH>@a1M;VDL){_i=2_U`t`(%E?cU&qz&7Ni9lYU{6U-i3bW~GcX(K8DulC znCKZ=rZ5N>B_<_hmN+DqB*#OTxrG_bNem2ZN0JzrecO^4*iIe%n_4A}?un^Sdz_v6!>15>Jj3T>4Iw+j{yUAU zSL*T82$2Ni$Fm3>P0)U|{YpJs#c6=Wj#k!4#g4_aSsw3jM5=j+=cShPdXPq4DRgRv!8M z1>o;1&(`t=8uIj-X_KWdb$H*&8a7a&L2PrWn$`Fhn^2SYOrI9ZF0nP;XyA$9&}qXr8$L8TP3j(NKI)n}bgi`wt4w^~ zwW=-O^yq8To_$iS!vlwP&v#q8&w8F{d8Vs1U8iZfz8LewX_w4LD~EK)Icvc~YxzbA zeeXxVs(gK_yt7ft-TvQu^lzASAPpVW@Pg@jmeYW{F~J0#Ig>I2<=)D(xOwjH^8Akl z6QnKF9M7kOsL zWm|p#g4J_L#q%9t9%9RDLS{%1n?*FeYF zmPhqaR1|b*HV{&Q3-=}3#K;AzFPNdg)CUxnDVflUSe<@&``v4IamXJ}3i&Loi*B5! zuJr;1@IH?J85#dOIR1UG==ClFpi{enkN}fW>coqH>1Z?UtO=%R*23Vm-z^;A!r-Nw z3H84c2E72z4R{AU!|{KIjQ)h@GM6ZYOUM2Gk2+I)xYBqH6a+w|s2Mbt5 z8WDl?y}**7Mhwn}sCUc)6D5cW_>S(>p(bz%%T`k(hT%f+0_vKM3vOb;X=dakK5v}8 zhvWWOI!@vS@S%Fd=1?GaGr)iDx{Uv6j(?*CU+SjjIuH?yTK}<)!jhmzVuR^Rf+C|w zj2NIXyDEet!Ow@_(*mit%uFo(BPaiLkc0g4jddvZ+;u+x|B&Zjg!p3TUAh>W?@<=4 zH4UwO-upxYZ2G)U{_yl6G4KCb$Gi(^gU7l4FUs@3i{sxjZA}E(KnCS4n|>hHffj!*_H1@0al}0{cPe3Rc8Tje56H51>P{fsz3e5gO6KN2CB` zR*Wps5U!dAtn}(mi{>PZbFmPVGjSGPL~{>vp>X@(N+_fPoIdF!x5)j!zsdM7&D+{M zIDgG3;+HOjFdmZw)F=L)G}?Ah^L!YQr31ddZQ4~Zm@GFHR8$~zZW)pg{T~^sv{Nd^5CFi9F+*S@e1RA5 zd-Gu~FD}2M)PA2LczwgdHpjmy<6o@ni_o8WL9da$hJBL=6^ZThsPR?`P|6(?L$NgUr*^l4<1%|zC`othU$s`!Ko~>Z3-rY0u&^OiY zp1u|W`P~8j^A#EYV*OwE|L0m}tIhe3ZPZ=&_lxIG>~oAHF-m`WN$GcT`KjelZ+X^& zsX6B%xmx!8^G(ZuHA1gW;5vK<$9}@Q*wD3bO@;1XhSUs}oXIY9j}r{TdQQ-n6fr1z zKA}N~JA!zE#ogOoH}?3{ zQEu$KmvsB|>ZINqRaE}L)q#0)=h_nPO=-Us&5S7@}19xY%R zj8T)H8BkIpfNpo+a1rpt&whGb#3A2G7?&ulYFjgifYPZFhyXk;gF<+_p`{+0$@1!bvfdMlpI4oZ}!VhF%9oqWMLd6~Hm=gA=~}&X*n%PO{2?>1&Ht4K4`< zI_Ce`mg_p89W}4j)byHFH@v2)w~4y=W&x6s2tn6W>yMBq%U z=I5{cJ+r37p(Gro+HNHZ;HP=~?=!NgbFc3b{Vc$rF8epky5=-k0x&>H@?{aY17Fjo(h!8Wz7#cPZrfxsqLo6I@zIqUZ1VTp)mElOUhN;ul z{!ApsZ=Ug9;S%GP+Dx>S0G;q%3x_!VBl-XP`2Ig;l-q>ns5D4J=dm4jG+y{bxD}cY z1j=XPX%z!@STx``AEKeM7ERq@XB1|%L|D4~=I1z?gP>XRLkIDYtRF8Kf@BUB<%93RDPy6K!59x2Kum{ZlZp6- z|I1f}o1so0O^(IQxC854|9?@&|KWCKWy+r=ek73uYwlq3W560*V5P0@+ZYaEH2+(yVN3@R4^&2I2mBJ+ zN9eDiiMmmQ&O!Mit()ZS;FmYQ7A{XB^!;#o3gy@@cZ&8Egr)K}RFKNks3euQqe-be zgZ4}19q5Eq-if{}m3QTWUk(+|-;L$+o`PK7TawFnPRiwd`{nZNiJ&|d(BpwQgr0T( E4;zmc%K!iX literal 0 HcmV?d00001 diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version new file mode 100755 index 00000000..203a50c1 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version @@ -0,0 +1 @@ +[store_msg,persistent_bytes,multiple_routing_keys,exchange_options,queue_options,topic_permission,vhost_limits,user_password_hashing,cluster_name,policy_apply_to,topic_trie_node,mirrored_supervisor,gm,user_admin_to_tags,exchange_event_serial,semi_durable_route,topic_trie,add_opts_to_listener,remove_user_scope,move_messages_to_vhost_store]. diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py b/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py old mode 100644 new mode 100755 index b4bf133a..5ba2dd66 --- a/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py @@ -69,7 +69,7 @@ # Import schemas from . import schemas -LOGGER = logginng.getLogger(__name__) +LOGGER = logging.getLogger(__name__) # ---------- Task Helper Functions ---------- def convert_uuid_to_string(uuid): diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/views.py b/src/pe_reports/pe_reports_django_project/dataAPI/views.py old mode 100644 new mode 100755 index 0ee1aa79..76e0c5d3 --- a/src/pe_reports/pe_reports_django_project/dataAPI/views.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/views.py @@ -105,6 +105,7 @@ ReportSummaryStats, RootDomains, ShodanAssets, + ShodanVulns, SubDomains, VwBreachcomp, VwBreachcompBreachdetails, @@ -7170,7 +7171,7 @@ def shodan_assets_insert( "tags": row_dict.get("tags", []), "country_code": row_dict.get("country_code"), "location": row_dict.get("location"), - "data_source_uid": row_dict.get("data_source_uid"), + "data_source_uid_id": row_dict.get("data_source_uid"), } # Use 'update_or_create' to either create or update the record @@ -7223,39 +7224,111 @@ def shodan_vulns_insert( for row in data.vuln_data: row_dict = row.__dict__ try: - CredentialExposures.objects.get( - breach_name=row_dict["breach_name"], - email=row_dict["email"], - ) - # If record already exists, do nothing - except CredentialExposures.DoesNotExist: - # If record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( - organizations_uid=row_dict["organizations_uid"] - ) - curr_source_inst = DataSource.objects.get( - data_source_uid=row_dict["data_source_uid"] - ) - curr_breach_inst = CredentialBreaches.objects.get( - breach_name=row_dict["breach_name"], - ) - CredentialExposures.objects.create( - # credential_exposures_uid=uuid.uuid1(), - email=row_dict["email"], - organizations_uid=curr_org_inst, - root_domain=row_dict["root_domain"], - sub_domain=row_dict["sub_domain"], - modified_date=row_dict["modified_date"], - breach_name=row_dict["breach_name"], - credential_breaches_uid=curr_breach_inst, - data_source_uid=curr_source_inst, - name=row_dict["name"], - ) - create_cnt += 1 + org_instance = Organizations.objects.get(organizations_uid=row_dict["organizations_uid"]) + acronym = org_instance.acronym # Assuming 'acronym' is a field in Organizations model + # Assuming ExternalOrganizations is a model that matches organization acronym to an organization_id + mdl_org = MDL_Organization.objects.get(acronym=acronym) + try: + mdl_data_source = MDL_DataSource.objects.get(name="Shodan") + except DataSource.DoesNotExist: + LOGGER.warning(f"DataSource with UID {row_dict['data_source_uid_id']} not found.") + mdl_data_source = None # Set to None if DataSource is not found + + mdl_vuln_data = { + "organization_name": row_dict.get("organization"), + "cve": row_dict.get("cve"), + "severity": row_dict.get("severity"), + "cvss": row_dict.get("cvss"), + "summary": row_dict.get("summary"), + "product": row_dict.get("product"), + "attack_vector": row_dict.get("attack_vector"), + "av_description": row_dict.get("av_description"), + "attack_complexity": row_dict.get("attack_complexity"), + "ac_description": row_dict.get("ac_description"), + "confidentiality_impact": row_dict.get("confidentiality_impact"), + "ci_description": row_dict.get("ci_description"), + "integrity_impact": row_dict.get("integrity_impact"), + "ii_description": row_dict.get("ii_description"), + "availability_impact": row_dict.get("availability_impact"), + "ai_description": row_dict.get("ai_description"), + "tags": row_dict.get("tags"), + "domains": row_dict.get("domains"), + "hostnames": row_dict.get("hostnames"), + "isn": row_dict.get("isn"), + "asn": row_dict.get("asn"), + "data_source": mdl_data_source, + "type": row_dict.get("type"), + "name": row_dict.get("name"), + "potential_vulns": row_dict.get("potential_vulns"), + "mitigation": row_dict.get("mitigation"), + "server": row_dict.get("server"), + "is_verified": row_dict.get("is_verified"), + "banner": row_dict.get("banner"), + "version": row_dict.get("version"), + "cpe": row_dict.get("cpe") + } + + mdl_obj, created = MDL_ShodanVulns.objects.update_or_create( + organization=mdl_org, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=row_dict["timestamp"], + defaults=mdl_vuln_data) + except: + LOGGER.warning(f"Shodan Vuln failed to save to MDL.") + + try: + vuln_data = { + "organization": row_dict.get("organization"), + "cve": row_dict.get("cve"), + "severity": row_dict.get("severity"), + "cvss": row_dict.get("cvss"), + "summary": row_dict.get("summary"), + "product": row_dict.get("product"), + "attack_vector": row_dict.get("attack_vector"), + "av_description": row_dict.get("av_description"), + "attack_complexity": row_dict.get("attack_complexity"), + "ac_description": row_dict.get("ac_description"), + "confidentiality_impact": row_dict.get("confidentiality_impact"), + "ci_description": row_dict.get("ci_description"), + "integrity_impact": row_dict.get("integrity_impact"), + "ii_description": row_dict.get("ii_description"), + "availability_impact": row_dict.get("availability_impact"), + "ai_description": row_dict.get("ai_description"), + "tags": row_dict.get("tags"), + "domains": row_dict.get("domains"), + "hostnames": row_dict.get("hostnames"), + "isn": row_dict.get("isn"), + "asn": row_dict.get("asn"), + "data_source_uid_id": row_dict.get("data_source_uid"), + "type": row_dict.get("type"), + "name": row_dict.get("name"), + "potential_vulns": row_dict.get("potential_vulns"), + "mitigation": row_dict.get("mitigation"), + "server": row_dict.get("server"), + "is_verified": row_dict.get("is_verified"), + "banner": row_dict.get("banner"), + "version": row_dict.get("version"), + "cpe": row_dict.get("cpe") + } + + obj, created = MDL_ShodanVulns.objects.update_or_create( + organizations_uid=org_instance, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=row_dict["timestamp"], + defaults=vuln_data) + if created: + create_cnt+=1 + except: + LOGGER.warning(f"Shodan Vuln failed to save to PE DB.") + continue # Return success message return ( str(create_cnt) - + " records created in the credential_exposures table" + + " records created in the shodan vulns table" ) except ObjectDoesNotExist: LOGGER.info("API key expired please try again") diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py old mode 100644 new mode 100755 index 79047a5c..953759d4 --- a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py @@ -2346,24 +2346,25 @@ class Meta: class ShodanAssets(models.Model): """Define ShodanAssets model.""" - shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid" + Organization, on_delete=models.CASCADE, db_column="organization_uid", blank=True, null=True ) - organization = models.TextField(blank=True, null=True) + # If you still need to store the organization name or acronym, use a separate field for that + organization_name = models.TextField(blank=True, null=True) # New field to store the name or acronym ip = models.TextField(blank=True, null=True) port = models.IntegerField(blank=True, null=True) protocol = models.TextField(blank=True, null=True) timestamp = models.DateTimeField(blank=True, null=True) product = models.TextField(blank=True, null=True) server = models.TextField(blank=True, null=True) - tags = models.TextField(blank=True, null=True) # This field type is a guess. - domains = models.TextField(blank=True, null=True) # This field type is a guess. - hostnames = models.TextField(blank=True, null=True) # This field type is a guess. + tags = models.JSONField(blank=True, null=True) # Store tags as a list (JSON format) + domains = models.JSONField(blank=True, null=True) # Store domains as a list (JSON format) + hostnames = models.JSONField(blank=True, null=True) # Store hostnames as a list (JSON format) isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, on_delete=models.CASCADE, db_column="data_source_uid", blank=True, null=True ) country_code = models.TextField(blank=True, null=True) location = models.TextField(blank=True, null=True) @@ -2420,11 +2421,11 @@ class Meta: class ShodanVulns(models.Model): """Define ShodanVulns model.""" - shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organization = models.ForeignKey( Organization, on_delete=models.CASCADE, db_column="organization_uid" ) - organization = models.TextField(blank=True, null=True) + organization_name = models.TextField(blank=True, null=True) ip = models.TextField(blank=True, null=True) port = models.TextField(blank=True, null=True) protocol = models.TextField(blank=True, null=True) diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py old mode 100644 new mode 100755 index 0ca7fdf9..5e57a42a --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py @@ -8,9 +8,9 @@ """ # Standard Python Libraries import os - -# Third-Party Libraries -from dataAPI.views import api_router +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pe_reports_django.settings") +import django +django.setup() # Following 2 lines custom code from django.apps import apps @@ -21,9 +21,10 @@ from fastapi.staticfiles import StaticFiles from starlette.middleware.cors import CORSMiddleware -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pe_reports_django.settings") +# Third-Party Libraries +from dataAPI.views import api_router -application = get_wsgi_application() +# application = get_wsgi_application() # Below this comment is custom code apps.populate(settings.INSTALLED_APPS) diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py new file mode 100755 index 00000000..e0a9bc66 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py @@ -0,0 +1,32 @@ +class MyAppRouter: + def db_for_read(self, model, **hints): + # Specify the app you want to route to the mini_data_lake database + if model._meta.app_label == 'dmz_mini_dl': + return 'mini_data_lake' + return 'default' # All other models go to the default database + + def db_for_write(self, model, **hints): + if model._meta.app_label == 'dmz_mini_dl': + return 'mini_data_lake' + return 'default' # All other models go to the default database + + def allow_relation(self, obj1, obj2, **hints): + # Check the app labels of both objects + app_label1 = obj1._meta.app_label + app_label2 = obj2._meta.app_label + + # If both objects are from the specific app, allow the relation + if app_label1 == 'dmz_mini_dl' and app_label2 == 'dmz_mini_dl': + return True + + # If only one of them is from the specific app, disallow the relation + if app_label1 == 'dmz_mini_dl' or app_label2 == 'dmz_mini_dl': + return False + + # Allow relations between all other models + return True + + def allow_migrate(self, db, app_label, model_name=None, **hints): + if app_label == 'dmz_mini_dl': + return db == 'mini_data_lake' # Migrate the specific app to the mini_data_lake database + return db == 'default' # All other apps migrate to the default database \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt old mode 100644 new mode 100755 index 0671ea45..35cb5f54 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt @@ -1,254 +1,229 @@ -alembic==1.8.1 -amqp==5.1.1 -anyio==3.6.1 -arabic-reshaper==2.1.3 -asgiref==3.6.0 -asn1crypto==1.5.1 -asttokens==2.0.5 -async-timeout==4.0.2 -attrs==21.4.0 -Babel==2.12.1 -backcall==0.2.0 -bcrypt==4.0.1 -beautifulsoup4==4.10.0 -billiard==3.6.4.0 -blis==0.7.8 -boto3==1.21.10 -botocore==1.24.10 -build==0.7.0 -catalogue==2.0.6 -celery==5.2.7 -certifi==2021.10.8 -cffi==1.15.0 -cfgv==3.3.1 -channels==4.0.0 -chardet==3.0.4 -charset-normalizer==2.0.12 -check-manifest==0.48 -chevron==0.14.0 -circlify==0.15.0 -click==8.1.3 -click-didyoumean==0.3.0 -click-plugins==1.1.1 -click-repl==0.2.0 -colorama==0.4.4 -contextlib2==21.6.0 -coverage==6.3.2 -coveralls==3.3.1 -crispy-bootstrap5==0.7 -cryptography==36.0.2 -cssselect2==0.7.0 -cycler==0.11.0 -cymem==2.0.6 -DataProperty==0.55.0 -dateparser==1.1.0 -decorator==5.1.1 -demoji==1.1.0 -Deprecated==1.2.13 -distlib==0.3.4 -Django==4.1.5 -django-crispy-forms==1.14.0 -django-database-view==0.3.0 -dnspython==2.2.1 -dnstwist==20220815 -docopt==0.6.2 -docx==0.2.4 -docxcompose==1.4.0 -docxtpl==0.16.6 -dshield==0.2.1 -ecdsa==0.18.0 -email-validator==1.3.0 -en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.4.0/en_core_web_lg-3.4.0-py3-none-any.whl -en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.4.0/en_core_web_trf-3.4.0-py3-none-any.whl -et-xmlfile==1.1.0 -executing==0.8.3 -Faker==13.3.2 -fastapi==0.88.0 -filelock==3.6.0 -Flask==2.1.2 -Flask-Login==0.6.1 -Flask-Migrate==3.1.0 -Flask-SQLAlchemy==2.5.1 -Flask-WTF==1.0.1 -flower==1.2.0 -future==0.18.2 -glob2==0.7 -googletrans==4.0.0rc1 -greenlet==2.0.0a2 -gunicorn==20.1.0 -h11==0.9.0 -h2==3.2.0 -hpack==3.0.0 -hstspreload==2021.12.1 -html5lib==1.1 -httpcore==0.9.1 -httptools==0.5.0 -httpx==0.13.3 -huggingface-hub==0.8.1 -humanize==4.6.0 -hyperframe==5.2.0 -identify==2.5.0 -idna==2.5 -importlib-resources==5.4.0 -iniconfig==1.1.1 -ipython==8.4.0 -itsdangerous==2.1.2 -jedi==0.18.1 -Jinja2==3.1.2 -jmespath==0.10.0 -joblib==1.1.0 -jsonschema==4.4.0 -kiwisolver==1.3.2 -kombu==5.2.4 -langcodes==3.3.0 -limits==3.4.0 -lxml==4.8.0 -Mako==1.2.1 -MarkupSafe==2.1.1 -matplotlib==3.3.4 -matplotlib-inline==0.1.3 -mbstrdecoder==1.1.0 -mongo-db-from-config @ http://github.com/cisagov/mongo-db-from-config/tarball/develop -murmurhash==1.0.7 -mypy==0.961 -mypy-extensions==0.4.3 -nassl==4.0.2 -nltk==3.7 -nodeenv==1.6.0 -numpy==1.22.3 -openpyxl==3.0.9 -oscrypto==1.3.0 -packaging==21.3 -pandas==1.1.5 -paramiko==2.12.0 -parso==0.8.3 -path==16.4.0 -pathvalidate==2.5.0 -pathy==0.6.2 -pdfkit==1.0.0 --e git+https://github.com/cisagov/pe-reports.git@eda9a332132581c12b8620ed925e4b82d0d62316#egg=pe_reports -pep517==0.12.0 -pexpect==4.8.0 -phonenumbers==8.12.45 -pickleshare==0.7.5 -Pillow==9.0.1 -platformdirs==2.5.1 -pluggy==1.0.0 -pre-commit==2.18.1 -preshed==3.0.6 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 --e git+https://github.com/cisagov/pshtt.git@65596ff08fa7bf0357b5af63da73e2dead91c304#egg=pshtt -psutil==5.9.1 -psycopg2-binary==2.9.3 -ptyprocess==0.7.0 -publicsuffix==1.1.1 -pure-eval==0.2.2 -py==1.11.0 -pyasn1==0.4.8 -pycparser==2.21 -pydantic==1.9.0 -Pygments==2.12.0 -pygtail==0.12.0 -pyHanko==0.16.0 -pyhanko-certvalidator==0.19.8 -pymongo==4.0.1 -PyMuPDF==1.19.0 -PyNaCl==1.5.0 -pyOpenSSL==22.0.0 -pyparsing==3.0.7 -PyPDF2==1.26.0 -PyPDF3==1.0.6 -pyrsistent==0.18.1 -pytablereader==0.31.3 -pytablewriter==0.64.2 -pytest==7.0.1 -pytest-cov==3.0.0 -python-bidi==0.4.2 -python-dateutil==2.8.2 -python-decouple==3.6 -python-docx==0.8.11 -python-dotenv==0.21.0 -python-jose==3.3.0 -python-multipart==0.0.5 -python-pptx==0.6.21 -python-stdnum==1.17 -pytz==2023.3 -pytz-deprecation-shim==0.1.0.post0 -PyYAML==6.0 -qrcode==7.3.1 -rabbitmq==0.2.0 -regex==2022.3.15 -reportlab==3.6.6 -requests==2.27.1 -retry==0.9.2 -rfc3986==1.5.0 -rsa==4.9 -s3transfer==0.5.2 -schema==0.7.5 -scikit-learn==1.0.2 -scipy==1.8.0 -scrubadub==2.0.0 -semver==2.13.0 -shodan==1.27.0 -six==1.16.0 -sklearn==0.0 -slowapi==0.1.8 -smart-open==5.2.1 -sniffio==1.2.0 -soupsieve==2.3.1 -spacy==3.4.0 -spacy-alignments==0.8.5 -spacy-legacy==3.0.9 -spacy-loggers==1.0.3 -spacy-transformers==1.1.7 -SQLAlchemy==1.4.39 -sqlparse==0.4.3 -srsly==2.4.4 -sshtunnel==0.4.0 -sslyze==5.0.3 -stack-data==0.3.0 -starlette==0.22.0 -Sublist3r==1.0 -svglib==1.5.1 -tabledata==1.3.0 -tcolorpy==0.1.2 -textblob==0.15.3 -thinc==8.1.0 -threadpoolctl==3.1.0 -tinycss2==1.2.1 -tls-parser==2.0.0 -tokenizers==0.12.1 -toml==0.10.2 -tomli==2.0.1 -torch==1.12.0 -tornado==6.3.1 -tox==3.24.5 -tqdm==4.63.0 -traitlets==5.3.0 -transformers==4.20.1 -typepy==1.3.0 -typer==0.4.2 -types-PyYAML==6.0.4 -typing_extensions==4.1.1 -tzdata==2021.5 -tzlocal==4.1 -unicorn==2.0.1.post1 -uritools==4.0.1 -urllib3==1.26.0 -uvicorn==0.18.3 -uvloop==0.17.0 -vine==5.0.0 -virtualenv==20.14.0 -wasabi==0.9.1 -watchfiles==0.17.0 -wcwidth==0.2.6 -webencodings==0.5.1 -websockets==10.3 -Werkzeug==2.0.0 -wget==3.2 -whitenoise==6.3.0 -wrapt==1.15.0 -WTForms==3.0.1 -xhtml2pdf==0.2.5 -XlsxWriter==3.0.3 +aioredis +alembic +amqp +anyio +arabic-reshaper +argcomplete +asgiref +async-timeout +attrs +Babel +bcrypt +beautifulsoup4 +billiard +blinker +blis +boto3 +botocore +build +CacheControl +catalogue +celery +certifi +cffi +chardet +charset-normalizer +chevron +circlify +cleo +click +click-didyoumean +click-plugins +click-repl +colorama +confection +contextlib2 +coverage +crashtest +crispy-bootstrap5 +cron-descriptor +cryptography +cycler +cymem +Cython +decorator +demoji +Deprecated +distlib +Django +django-celery-beat +django-crispy-forms +django-netfields +django-timezone-field +dnspython +dnstwist +docopt +docxcompose +docxtpl +dshield +dulwich +ecdsa +ecs-logging +elastic-apm +email-validator +en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.5.0/en_core_web_lg-3.5.0-py3-none-any.whl#sha256 +et-xmlfile +exceptiongroup +fastapi +fastapi-limiter +filelock +Flask +Flask-Login +Flask-Migrate +Flask-SQLAlchemy +Flask-WTF +flower +freetype-py +glob2 +googletrans +greenlet +gunicorn +h11 +h2 +hpack +hstspreload +html5lib +httpcore +httpx +humanize +hyperframe +idna +importlib-metadata +importlib-resources +iniconfig +installer +itsdangerous +jaraco.classes +jeepney +Jinja2 +jmespath +joblib +jsonschema +keyring +kiwisolver +kombu +langcodes +limits +lockfile +lxml +Mako +MarkupSafe +matplotlib +mongo-db-from-config @ http://github.com/cisagov/mongo-db-from-config/tarball/develop#sha256 +more-itertools +msgpack +murmurhash +nassl +nested-lookup +netaddr +nltk +numpy +openpyxl +packaging +pandas +paramiko +pathy +pdfkit +pdfrw +-e git+https://github.com/cisagov/ATC-Framework.git@CD-add-CODEOWNERS#egg=pe_reports +pexpect +pika +Pillow +pipx +pkginfo +platformdirs +pluggy +poetry +poetry-core +poetry-plugin-export +preshed +prometheus-client +prompt-toolkit +psutil +psycopg2-binary +ptyprocess +publicsuffixlist +py +pyasn1 +pycairo +pycparser +pydantic +pymongo +PyMuPDF +PyNaCl +pyOpenSSL +pyparsing +PyPDF2 +pyproject_hooks +pyrsistent +pytest +pytest-cov +python-bidi +python-crontab +python-dateutil +python-decouple +python-docx +python-jose +python-multipart +python-pptx +pytz +PyYAML +rapidfuzz +redis +regex +reportlab +requests +requests-toolbelt +retry +rfc3986 +rlPyCairo +rsa +s3transfer +schema +scikit-learn +scipy +SecretStorage +shellingham +shodan +six +slowapi +smart-open +sniffio +soupsieve +spacy +spacy-legacy +spacy-loggers +SQLAlchemy +sqlparse +srsly +sshtunnel +sslyze +starlette +Sublist3r +thinc +threadpoolctl +tls-parser +tomli +tomlkit +tornado +tqdm +trove-classifiers +typer +types-PyYAML +typing_extensions +tzdata +urllib3 +userpath +uvicorn +vine +virtualenv +wasabi +wcwidth +webencodings +Werkzeug +whitenoise +wrapt +WTForms +xhtml2pdf +XlsxWriter +zipp \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py old mode 100644 new mode 100755 index a1575f7e..348d4e49 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py @@ -100,7 +100,7 @@ "class": "logging.handlers.RotatingFileHandler", "maxBytes": 1024 * 1024 * 15, "backupCount": 10, - "filename": "./pe_reportsLogFile.log", + "filename": os.path.join(BASE_DIR, "pe_reportsLogFile.log"), "formatter": "verbose", }, }, @@ -196,7 +196,7 @@ CELERY_BROKER_URL = ( f"amqp://{config('RABBITMQ_USER')}:{config('RABBITMQ_PASS')}@localhost:5672/" ) -CELERY_RESULT_BACKEND = "redis://localhost:6379" +CELERY_RESULT_BACKEND = "redis://atc_redis:6379" CELERY_RESULT_EXPIRES = 86400 CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, "celerybeat-schedule.db") diff --git a/src/pe_reports/pe_reports_django_project/static/CISAImage.png b/src/pe_reports/pe_reports_django_project/static/CISAImage.png old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/autocomplete.css b/src/pe_reports/pe_reports_django_project/static/admin/css/autocomplete.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/base.css b/src/pe_reports/pe_reports_django_project/static/admin/css/base.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/changelists.css b/src/pe_reports/pe_reports_django_project/static/admin/css/changelists.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/dark_mode.css b/src/pe_reports/pe_reports_django_project/static/admin/css/dark_mode.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/dashboard.css b/src/pe_reports/pe_reports_django_project/static/admin/css/dashboard.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/fonts.css b/src/pe_reports/pe_reports_django_project/static/admin/css/fonts.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/forms.css b/src/pe_reports/pe_reports_django_project/static/admin/css/forms.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/login.css b/src/pe_reports/pe_reports_django_project/static/admin/css/login.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/nav_sidebar.css b/src/pe_reports/pe_reports_django_project/static/admin/css/nav_sidebar.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/responsive.css b/src/pe_reports/pe_reports_django_project/static/admin/css/responsive.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/responsive_rtl.css b/src/pe_reports/pe_reports_django_project/static/admin/css/responsive_rtl.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/rtl.css b/src/pe_reports/pe_reports_django_project/static/admin/css/rtl.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/LICENSE-SELECT2.md b/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/LICENSE-SELECT2.md old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.css b/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.min.css b/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.min.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/widgets.css b/src/pe_reports/pe_reports_django_project/static/admin/css/widgets.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/LICENSE.txt b/src/pe_reports/pe_reports_django_project/static/admin/fonts/LICENSE.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/README.txt b/src/pe_reports/pe_reports_django_project/static/admin/fonts/README.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Bold-webfont.woff b/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Bold-webfont.woff old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Light-webfont.woff b/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Light-webfont.woff old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Regular-webfont.woff b/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Regular-webfont.woff old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/LICENSE b/src/pe_reports/pe_reports_django_project/static/admin/img/LICENSE old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/README.txt b/src/pe_reports/pe_reports_django_project/static/admin/img/README.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/calendar-icons.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/calendar-icons.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_off.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_off.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_on.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_on.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-addlink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-addlink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-alert.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-alert.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-calendar.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-calendar.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-changelink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-changelink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-clock.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-clock.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-deletelink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-deletelink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-no.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-no.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown-alt.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown-alt.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-viewlink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-viewlink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-yes.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-yes.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/inline-delete.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/inline-delete.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/search.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/search.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/selector-icons.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/selector-icons.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/sorting-icons.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/sorting-icons.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-add.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-add.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-arrowright.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-arrowright.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/SelectBox.js b/src/pe_reports/pe_reports_django_project/static/admin/js/SelectBox.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/SelectFilter2.js b/src/pe_reports/pe_reports_django_project/static/admin/js/SelectFilter2.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/actions.js b/src/pe_reports/pe_reports_django_project/static/admin/js/actions.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/admin/DateTimeShortcuts.js b/src/pe_reports/pe_reports_django_project/static/admin/js/admin/DateTimeShortcuts.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/admin/RelatedObjectLookups.js b/src/pe_reports/pe_reports_django_project/static/admin/js/admin/RelatedObjectLookups.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/autocomplete.js b/src/pe_reports/pe_reports_django_project/static/admin/js/autocomplete.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/calendar.js b/src/pe_reports/pe_reports_django_project/static/admin/js/calendar.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/cancel.js b/src/pe_reports/pe_reports_django_project/static/admin/js/cancel.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/change_form.js b/src/pe_reports/pe_reports_django_project/static/admin/js/change_form.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/collapse.js b/src/pe_reports/pe_reports_django_project/static/admin/js/collapse.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/core.js b/src/pe_reports/pe_reports_django_project/static/admin/js/core.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/filters.js b/src/pe_reports/pe_reports_django_project/static/admin/js/filters.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/inlines.js b/src/pe_reports/pe_reports_django_project/static/admin/js/inlines.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/jquery.init.js b/src/pe_reports/pe_reports_django_project/static/admin/js/jquery.init.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/nav_sidebar.js b/src/pe_reports/pe_reports_django_project/static/admin/js/nav_sidebar.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/popup_response.js b/src/pe_reports/pe_reports_django_project/static/admin/js/popup_response.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate.js b/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate_init.js b/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate_init.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/urlify.js b/src/pe_reports/pe_reports_django_project/static/admin/js/urlify.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/LICENSE.txt b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/LICENSE.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.min.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.min.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/LICENSE.md b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/LICENSE.md old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/af.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/af.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ar.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ar.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/az.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/az.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bg.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bg.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bn.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bn.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bs.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bs.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ca.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ca.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/cs.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/cs.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/da.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/da.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/de.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/de.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/dsb.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/dsb.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/el.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/el.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/en.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/en.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/es.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/es.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/et.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/et.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/eu.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/eu.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fa.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fa.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fi.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fi.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/gl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/gl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/he.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/he.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hi.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hi.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hsb.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hsb.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hu.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hu.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hy.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hy.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/id.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/id.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/is.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/is.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/it.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/it.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ja.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ja.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ka.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ka.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/km.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/km.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ko.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ko.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lt.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lt.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lv.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lv.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/mk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/mk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ms.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ms.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nb.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nb.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ne.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ne.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ps.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ps.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt-BR.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt-BR.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ro.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ro.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ru.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ru.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sq.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sq.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr-Cyrl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr-Cyrl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sv.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sv.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/th.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/th.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/uk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/uk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/vi.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/vi.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-CN.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-CN.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-TW.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-TW.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.min.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.min.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/LICENSE.txt b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/LICENSE.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.min.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.min.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/css/custom.css b/src/pe_reports/pe_reports_django_project/static/css/custom.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/js/htmx.js b/src/pe_reports/pe_reports_django_project/static/js/htmx.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/power.svg b/src/pe_reports/pe_reports_django_project/static/power.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/templates/base.html b/src/pe_reports/pe_reports_django_project/templates/base.html old mode 100644 new mode 100755 index 479220fd..95f82d41 --- a/src/pe_reports/pe_reports_django_project/templates/base.html +++ b/src/pe_reports/pe_reports_django_project/templates/base.html @@ -1,9 +1,9 @@ {% load static %} - + - {% block title %} {% endblock %} - P&E Application + {% block title %}{% endblock %} - P&E Application @@ -48,74 +48,71 @@ {% url 'home' as url %} Home + Home +
  • {% url 'peBulkUpload' as url %} Add P&E Stakeholder + Add P&E Stakeholder +
    • -
  • {% url 'report_gen' as url %} Report Generator + Report Generator +
    • -
  • {% url 'stakeholder_lite' as url %} Stakeholder Lite + Stakeholder Lite +
    • -
  • {% url 'bulkupload' as url %} Bulk File Upload + Bulk File Upload +
    • {% if user.is_staff %}
  • {% url 'weekly_status' as url %} Weekly Status + Weekly Status +
    • {% endif %} @@ -123,58 +120,53 @@ -
    +

    Welcome to P&E Reports

    - -
    -
    - - {% if messages %} {% for message in messages %} - + {% endfor %}
    - {% endif %} {% block content %} {% endblock content %} - - {% block extra_js %} - - {% endblock %} + + {% endblock %} + diff --git a/src/pe_reports/pe_reports_django_project/templates/home.html b/src/pe_reports/pe_reports_django_project/templates/home.html old mode 100644 new mode 100755 From e24e63bacf2140466fcd7bb43152aee8b1f45756 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Mon, 2 Dec 2024 09:39:41 -0800 Subject: [PATCH 5/7] Gets app running locally --- .gitignore | 3 +- src/pe_reports/Dockerfile | 2 - .../dataAPI/schemas.py | 7 + .../pe_reports_django_project/db_routers.py | 32 --- .../pe_reports_django_project/home/views.py | 2 +- .../pe_reports_django/requirements.txt | 234 +----------------- .../pe_reports_django/settings.py | 2 +- 7 files changed, 24 insertions(+), 258 deletions(-) delete mode 100644 src/pe_reports/pe_reports_django_project/db_routers.py diff --git a/.gitignore b/.gitignore index 59400e1b..9176017a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ src/pe_reports/assets/ __pycache__ .coverage .mypy_cache +venv/ .pytest_cache .python-version *.egg-info @@ -53,4 +54,4 @@ dnstwist_output.txt adhoc_investigations/adhoc_investigation.ini adhoc_investigations/input_data adhoc_investigations/output_data -adhoc_investigations/dnsmonitor_monitored_domains.csv +adhoc_investigations/dnsmonitor_monitored_domains.csv \ No newline at end of file diff --git a/src/pe_reports/Dockerfile b/src/pe_reports/Dockerfile index 10587de3..0674dfc3 100644 --- a/src/pe_reports/Dockerfile +++ b/src/pe_reports/Dockerfile @@ -33,7 +33,6 @@ WORKDIR /code # Install dependencies COPY --chown=atc_api:atc_api ./pe_reports_django_project/pe_reports_django/requirements.txt /code/ RUN pip install --no-cache-dir -r requirements.txt -RUN pip install --upgrade numpy spacy # Copy the project code COPY --chown=atc_api:atc_api pe_reports_django_project /code @@ -46,4 +45,3 @@ ENV DJANGO_SETTINGS_MODULE=pe_reports_django.settings # Run the application CMD uvicorn --workers 4 pe_reports_django.asgi:app1 --host 0.0.0.0 --port 8000 --reload - diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py index f4f14f0a..7a832c0b 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py @@ -19,6 +19,13 @@ the data type "Any" to see what the return is. """ +class GenInputOrgName(BaseModel): + """GenInputOrgUIDList schema class.""" + org_acronym: str + class Config: + """GenInputOrgUIDList config.""" + orm_mode = True + class OrgType(BaseModel): """OrgType schema.""" diff --git a/src/pe_reports/pe_reports_django_project/db_routers.py b/src/pe_reports/pe_reports_django_project/db_routers.py deleted file mode 100644 index e0a9bc66..00000000 --- a/src/pe_reports/pe_reports_django_project/db_routers.py +++ /dev/null @@ -1,32 +0,0 @@ -class MyAppRouter: - def db_for_read(self, model, **hints): - # Specify the app you want to route to the mini_data_lake database - if model._meta.app_label == 'dmz_mini_dl': - return 'mini_data_lake' - return 'default' # All other models go to the default database - - def db_for_write(self, model, **hints): - if model._meta.app_label == 'dmz_mini_dl': - return 'mini_data_lake' - return 'default' # All other models go to the default database - - def allow_relation(self, obj1, obj2, **hints): - # Check the app labels of both objects - app_label1 = obj1._meta.app_label - app_label2 = obj2._meta.app_label - - # If both objects are from the specific app, allow the relation - if app_label1 == 'dmz_mini_dl' and app_label2 == 'dmz_mini_dl': - return True - - # If only one of them is from the specific app, disallow the relation - if app_label1 == 'dmz_mini_dl' or app_label2 == 'dmz_mini_dl': - return False - - # Allow relations between all other models - return True - - def allow_migrate(self, db, app_label, model_name=None, **hints): - if app_label == 'dmz_mini_dl': - return db == 'mini_data_lake' # Migrate the specific app to the mini_data_lake database - return db == 'default' # All other apps migrate to the default database \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/home/views.py b/src/pe_reports/pe_reports_django_project/home/views.py index ea85a437..e86a5dde 100644 --- a/src/pe_reports/pe_reports_django_project/home/views.py +++ b/src/pe_reports/pe_reports_django_project/home/views.py @@ -76,7 +76,7 @@ def getUserKey(): """Get a users API key.""" - urlIDs = "http://127.0.0.1:8089/apiv1/get_key" + urlIDs = "http://127.0.0.1:8091/apiv1/get_key" payload = json.dumps({"refresh_token": f'{config("USER_REFRESH_TOKEN")}'}) headers = { "Content-Type": "application/json", diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt index 35cb5f54..26f68740 100755 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt @@ -1,229 +1,21 @@ -aioredis -alembic -amqp -anyio -arabic-reshaper -argcomplete -asgiref -async-timeout -attrs -Babel -bcrypt -beautifulsoup4 -billiard -blinker -blis -boto3 -botocore -build -CacheControl -catalogue -celery -certifi -cffi -chardet -charset-normalizer -chevron -circlify -cleo -click -click-didyoumean -click-plugins -click-repl -colorama -confection -contextlib2 -coverage -crashtest -crispy-bootstrap5 -cron-descriptor -cryptography -cycler -cymem -Cython -decorator -demoji -Deprecated -distlib Django +uvicorn +python-decouple==3.8 django-celery-beat django-crispy-forms -django-netfields -django-timezone-field -dnspython -dnstwist -docopt -docxcompose -docxtpl -dshield -dulwich -ecdsa -ecs-logging +crispy-bootstrap5 +whitenoise elastic-apm -email-validator -en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.5.0/en_core_web_lg-3.5.0-py3-none-any.whl#sha256 -et-xmlfile -exceptiongroup +psycopg2-binary +django-netfields fastapi -fastapi-limiter -filelock -Flask -Flask-Login -Flask-Migrate -Flask-SQLAlchemy -Flask-WTF -flower -freetype-py -glob2 -googletrans -greenlet -gunicorn -h11 -h2 -hpack -hstspreload -html5lib -httpcore -httpx -humanize -hyperframe -idna -importlib-metadata -importlib-resources -iniconfig -installer -itsdangerous -jaraco.classes -jeepney -Jinja2 -jmespath -joblib -jsonschema -keyring -kiwisolver -kombu -langcodes -limits -lockfile -lxml -Mako -MarkupSafe -matplotlib -mongo-db-from-config @ http://github.com/cisagov/mongo-db-from-config/tarball/develop#sha256 -more-itertools -msgpack -murmurhash -nassl -nested-lookup -netaddr -nltk -numpy -openpyxl -packaging -pandas -paramiko -pathy -pdfkit -pdfrw -e git+https://github.com/cisagov/ATC-Framework.git@CD-add-CODEOWNERS#egg=pe_reports -pexpect -pika -Pillow -pipx -pkginfo -platformdirs -pluggy -poetry -poetry-core -poetry-plugin-export -preshed -prometheus-client -prompt-toolkit -psutil -psycopg2-binary -ptyprocess -publicsuffixlist -py -pyasn1 -pycairo -pycparser -pydantic -pymongo -PyMuPDF -PyNaCl -pyOpenSSL -pyparsing -PyPDF2 -pyproject_hooks -pyrsistent -pytest -pytest-cov -python-bidi -python-crontab -python-dateutil -python-decouple -python-docx -python-jose -python-multipart -python-pptx -pytz -PyYAML -rapidfuzz -redis -regex -reportlab -requests -requests-toolbelt +en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.5.0/en_core_web_lg-3.5.0-py3-none-any.whl#sha256=c8ac64840c1eb3e3ca7bd38bd1e1c48fb0faeb2449d54d01d5ce629af4595775 retry -rfc3986 -rlPyCairo -rsa -s3transfer -schema -scikit-learn -scipy -SecretStorage -shellingham -shodan -six +email_validator +fastapi-limiter +python-jose slowapi -smart-open -sniffio -soupsieve -spacy -spacy-legacy -spacy-loggers -SQLAlchemy -sqlparse -srsly -sshtunnel -sslyze -starlette -Sublist3r -thinc -threadpoolctl -tls-parser -tomli -tomlkit -tornado -tqdm -trove-classifiers -typer -types-PyYAML -typing_extensions -tzdata -urllib3 -userpath -uvicorn -vine -virtualenv -wasabi -wcwidth -webencodings -Werkzeug -whitenoise -wrapt -WTForms -xhtml2pdf -XlsxWriter -zipp \ No newline at end of file +python-multipart +docxtpl +numpy==1.23.5 \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py index 348d4e49..27f82bfc 100755 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py @@ -196,7 +196,7 @@ CELERY_BROKER_URL = ( f"amqp://{config('RABBITMQ_USER')}:{config('RABBITMQ_PASS')}@localhost:5672/" ) -CELERY_RESULT_BACKEND = "redis://atc_redis:6379" +CELERY_RESULT_BACKEND = "redis://localhost:6379" CELERY_RESULT_EXPIRES = 86400 CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, "celerybeat-schedule.db") From da733fe9f47da427ef229e0f36d27d705fe41881 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:25:21 -0700 Subject: [PATCH 6/7] update pe_source scripts to save to mdl update pe_source scripts to save to mdl --- setup.py | 7 +- src/pe_reports/__init__.py | 3 + src/pe_reports/data/db_query.py | 8 +- .../dataAPI/schemas.py | 86 +- .../dataAPI/views.py | 696 +++++++++------ .../dmz_mini_dl/models.py | 834 ++++++++++++------ .../pe_reports_django_project/home/models.py | 88 +- .../pe_reports_django/requirements.txt | 28 +- src/pe_source/data/pe_db/db_query_source.py | 154 +++- src/pe_source/data/shodan_db/shodan_search.py | 17 +- src/pe_source/intelx_identity.py | 35 +- src/pe_source/pe_scripts.py | 44 +- src/pe_source/shodan_wrapper.py | 12 +- src/pe_source/xpanse_alert_pull.py | 160 +--- 14 files changed, 1292 insertions(+), 880 deletions(-) mode change 100755 => 100644 src/pe_reports/pe_reports_django_project/dataAPI/views.py mode change 100755 => 100644 src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py mode change 100755 => 100644 src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt diff --git a/setup.py b/setup.py index 2e227cf0..94bfff47 100644 --- a/setup.py +++ b/setup.py @@ -131,6 +131,8 @@ def get_version(version_file): "openpyxl", "pandas", "pdfkit", + "presidio-analyzer", + "presidio-anonymizer", "psutil", "psycopg2-binary", "psycopg2-binary", @@ -147,16 +149,17 @@ def get_version(version_file): "reportlab", "requests", "schema == 0.7.5", + "scrubadub", "setuptools == 58.1.0", "scikit-learn", "shodan == 1.27.0", "sshtunnel", "sslyze>=5.0.0", "spacy", - "spacy-loogers", + "spacy-loggers", "spacy-legacy", "spacy-transformers", - "spacy-alignments" + "spacy-alignments", "nltk", "beautifulsoup4", "sublist3r", diff --git a/src/pe_reports/__init__.py b/src/pe_reports/__init__.py index e69de29b..648d7c11 100644 --- a/src/pe_reports/__init__.py +++ b/src/pe_reports/__init__.py @@ -0,0 +1,3 @@ +"""The pe_reports library.""" + +CENTRAL_LOGGING_FILE = "pe_reports_logging.log" diff --git a/src/pe_reports/data/db_query.py b/src/pe_reports/data/db_query.py index 7ebb0474..9bb8f34f 100644 --- a/src/pe_reports/data/db_query.py +++ b/src/pe_reports/data/db_query.py @@ -62,7 +62,7 @@ def task_api_call(task_url, check_url, data={}, retry_time=3): # Ping task status endpoint and get status # check_task_resp = requests.get(check_task_url, headers=headers).json() check_task_resp = requests.get(check_task_url, headers=headers) - #print(check_task_resp) + # print(check_task_resp) check_task_resp = check_task_resp.json() task_status = check_task_resp.get("status") LOGGER.info( @@ -2200,7 +2200,7 @@ def query_previous_period(org_uid, prev_end_date): return assets_dict -# ---------- PE-Score API Queries, Issue 635 ---------- +# ---------- PE-Score API Queries, Issue 635 ---------- # --- Issue 635 --- def pescore_hist_domain_alert(start_date, end_date): """ @@ -3609,7 +3609,7 @@ def upsert_new_cves_tsql(new_cves): # --- 018 atc-framework OLD TSQL --- -def get_demo_orgs(conn): +def get_demo_orgs_tsql(conn): """Query organizations table for orgs we report on.""" try: cur = conn.cursor() @@ -3623,4 +3623,4 @@ def get_demo_orgs(conn): LOGGER.error("There was a problem with your database query %s", error) finally: if conn is not None: - close(conn) \ No newline at end of file + close(conn) diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py index 7a832c0b..4855a550 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py @@ -19,13 +19,6 @@ the data type "Any" to see what the return is. """ -class GenInputOrgName(BaseModel): - """GenInputOrgUIDList schema class.""" - org_acronym: str - class Config: - """GenInputOrgUIDList config.""" - orm_mode = True - class OrgType(BaseModel): """OrgType schema.""" @@ -38,6 +31,17 @@ class Config: orm_mode = True +class GenInputOrgName(BaseModel): + """GenInputOrgUIDList schema class.""" + + org_acronym: str + + class Config: + """GenInputOrgUIDList config.""" + + orm_mode = True + + class OrganizationBase(BaseModel): """OrganizationBase schema.""" @@ -3046,7 +3050,7 @@ class XpanseBusinessUnitsInsert(BaseModel): entity_type: Optional[str] = None region: Optional[str] = None rating: Optional[int] = None - cyhy_db_name: Optional[str] = None + cyhy_db_name: Optional[str] = None # --- xpanse endpoint, Issue 682 --- @@ -3669,7 +3673,7 @@ class OrganizationsFullTable(BaseModel): class Config: """OrganizationsFullTable schema config class.""" - + orm_mode = True @@ -3766,7 +3770,6 @@ class Config: orm_mode = True - # --- domain_permu_insert_dnstwist(), Issue 706 pe-reports/005 atc-framework --- # Insert multiple dnstwist records into the domain_permutations table class DomainPermuInsertDNSTwist(BaseModel): @@ -3810,12 +3813,12 @@ class Config: # --- get_root_domains(), Issue 707 pe-reports/006 atc-framework --- # This function will reuse the schemas from the /rootdomains_by_org_uid endpoint - + # --- getDataSource(), Issue 708 pe-reports/007 atc-framework --- # This function will reuse the schemas from the /data_source_by_name endpoint - + # --- execute_hibp_breach_values(), Issue 709 pe-reports/008 atc-framework --- # Insert bulk HIBP breach data into credential_breaches table class CredBreachesHIBPInsert(BaseModel): @@ -3887,7 +3890,7 @@ class Config: """CredExpHIBPInsertInput schema config class.""" orm_mode = True - + # --- get_breach_uids(), Issue 010 atc-framework --- # Retrieve all breach names and uids @@ -3905,7 +3908,7 @@ class Config: # --- query_orgs(), Issue 011 atc-framework --- # Reuses OrganizationsFullTable schema - + # --- query_PE_subs(), Issue 012 atc-framework --- class SubdomainsByOrgUIDInput(BaseModel): @@ -3930,7 +3933,7 @@ class Config: """SubdomainsByOrgUID schema config class.""" orm_mode = True - + # --- insert_shodan_assets(), Issue 016 atc-framework --- # Insert bulk Shodan data into shodan_assets table @@ -3977,37 +3980,36 @@ class Config: class ShodanVulnsInsert(BaseModel): """ShodanVulnsInsert schema class.""" - - organizations_uid:Optional[str] = None - organization:Optional[str] = None - ip:Optional[str] = None - port:Optional[str] = None - protocol:Optional[str] = None - timestamp:Optional[str] = None - cve:Optional[str] = None - severity:Optional[str] = None - cvss:Optional[float] = None - summary:Optional[str] = None - product:Optional[str] = None - attack_vector:Optional[str] = None - av_description:Optional[str] = None - attack_complexity:Optional[str] = None - ac_description:Optional[str] = None - confidentiality_impact:Optional[str] = None - ci_description:Optional[str] = None - integrity_impact:Optional[str] = None + organizations_uid: Optional[str] = None + organization: Optional[str] = None + ip: Optional[str] = None + port: Optional[str] = None + protocol: Optional[str] = None + timestamp: Optional[str] = None + cve: Optional[str] = None + severity: Optional[str] = None + cvss: Optional[float] = None + summary: Optional[str] = None + product: Optional[str] = None + attack_vector: Optional[str] = None + av_description: Optional[str] = None + attack_complexity: Optional[str] = None + ac_description: Optional[str] = None + confidentiality_impact: Optional[str] = None + ci_description: Optional[str] = None + integrity_impact: Optional[str] = None ii_description: Optional[str] = None availability_impact: Optional[str] = None ai_description: Optional[str] = None tags: Optional[List[str]] = None domains: Optional[List[str]] = None - hostnames: Optional[List[str]] = None + hostnames: Optional[List[str]] = None isn: Optional[str] = None asn: Optional[int] = None data_source_uid: Optional[str] = None type: Optional[str] = None name: Optional[str] = None - potential_vulns: Optional[List[str]] = None + potential_vulns: Optional[List[str]] = None mitigation: Optional[str] = None server: Optional[str] = None is_verified: Optional[bool] = None @@ -4026,7 +4028,7 @@ class Config: class ShodanVulnsInsertInput(BaseModel): """ShodanVulnsInsertInput schema class.""" - vulns_data: List[ShodanVulnsInsert] + vuln_data: List[ShodanVulnsInsert] class Config: """ShodanVulnsInsertInput schema config class.""" @@ -4037,6 +4039,7 @@ class Config: # --- get_demo_orgs(), Issue 018 atc-framework --- # Reuses OrganizationsFullTable schema + class OrgsAssetsPagedInput(BaseModel): """OrgsAssetsPagedInput schema class.""" @@ -4048,9 +4051,10 @@ class Config: orm_mode = True + class OrgsWithAssets(BaseModel): """OrgsWithAssets schema class.""" - + org_name: Optional[str] = None acronym: Optional[str] = None retired: Optional[bool] = None @@ -4073,7 +4077,6 @@ class OrgsWithAssets(BaseModel): parent_acronym: Optional[str] = None networks: Optional[List[str]] = None root_domains: Optional[List[str]] = None - class Config: """OrgsWithAssets schema config class.""" @@ -4081,7 +4084,7 @@ class Config: orm_mode = True validate_assignment = True - + class OrgAssetPagedResult(BaseModel): """OrgAssetPagedResult schema class.""" @@ -4089,10 +4092,11 @@ class OrgAssetPagedResult(BaseModel): current_page: int data: List[OrgsWithAssets] + class OrgAssetTaskResp(BaseModel): """OrgAssetTaskResp schema class.""" task_id: str status: str result: Optional[OrgAssetPagedResult] = None - error: Optional[str] = None \ No newline at end of file + error: Optional[str] = None diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/views.py b/src/pe_reports/pe_reports_django_project/dataAPI/views.py old mode 100755 new mode 100644 index 76e0c5d3..b518f949 --- a/src/pe_reports/pe_reports_django_project/dataAPI/views.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/views.py @@ -42,6 +42,7 @@ get_kev_list_info, get_l_stakeholders_info, get_m_stakeholders_info, + get_orgs_and_assets, get_s_stakeholders_info, get_ve_info, get_vs_info, @@ -61,7 +62,7 @@ sub_domains_by_org_task, sub_domains_table_task, top_cves_insert_task, - was_vulns_task + was_vulns_task, ) from decouple import config from django.conf import settings @@ -70,6 +71,20 @@ from django.db import transaction from django.db.models import F, Q from django.forms.models import model_to_dict +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from dmz_mini_dl.models import CredentialBreaches as MDL_CredentialBreaches +from dmz_mini_dl.models import CredentialExposures as MDL_CredentialExposures +from dmz_mini_dl.models import DataSource as MDL_DataSource +from dmz_mini_dl.models import Organization as MDL_Organization +from dmz_mini_dl.models import ShodanAssets as MDL_ShodanAssets +from dmz_mini_dl.models import ShodanVulns as MDL_ShodanVulns +from dmz_mini_dl.models import XpanseAlerts as MDL_XpanseAlerts +from dmz_mini_dl.models import XpanseAssetsMdl as MDL_XpanseAssets +from dmz_mini_dl.models import XpanseBusinessUnits as MDL_XpanseBusinessUnits +from dmz_mini_dl.models import XpanseCveServiceMdl as MDL_XpanseCveService +from dmz_mini_dl.models import XpanseCvesMdl as MDL_XpanseCves +from dmz_mini_dl.models import XpanseServicesMdl as MDL_XpanseServices from fastapi import ( APIRouter, Depends, @@ -99,6 +114,7 @@ DataSource, DomainAlerts, DomainPermutations, + Ips, Mentions, Organizations, PshttResults, @@ -135,19 +151,6 @@ XpanseCveService, XpanseServices, ) -from dmz_mini_dl.models import( - Organization as MDL_Organization, - XpanseBusinessUnits as MDL_XpanseBusinessUnits, - XpanseAlerts as MDL_XpanseAlerts, - XpanseAssetsMdl as MDL_XpanseAssets, - XpanseServicesMdl as MDL_XpanseServices, - XpanseCvesMdl as MDL_XpanseCves, - XpanseCveServiceMdl as MDL_XpanseCveService, - ShodanAssets as MDL_ShodanAssets, - ShodanVulns as MDL_ShodanVulns, - DataSource as MDL_DataSource, - -) from jose import exceptions, jwt from redis import asyncio as aioredis from slowapi import Limiter @@ -266,6 +269,7 @@ def userapiTokenUpdate(expiredaccessToken, user_refresh, theapiKey, user_id): def userapiTokenverify(theapiKey): """Check to see if api key is expired.""" tokenRecords = list(apiUser.objects.filter(apiKey=theapiKey)) + LOGGER.info(f"The user provided key is {theapiKey}") user_key = "" user_refresh = "" user_id = "" @@ -4248,8 +4252,7 @@ def rss_insert(data: schemas.RSSInsertInput, tokens: dict = Depends(get_api_key) try: # Check if record already exists ReportSummaryStats.objects.get( - organizations_uid=specified_org_uid, - start_date=data.start_date + organizations_uid=specified_org_uid, start_date=data.start_date ) # If it already exists, update ReportSummaryStats.objects.filter( @@ -4894,7 +4897,7 @@ def cred_breach_intelx( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, create task for query task = cred_breach_intelx_task.delay(data.source_uid) # Return the new task id w/ "Processing" status @@ -5600,13 +5603,38 @@ def cred_breaches_intelx_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx breach data insert_count = 0 update_count = 0 + + try: + mdl_source_inst = MDL_DataSource.objects.get(name="IntelX") + except MDL_DataSource.DoesNotExist: + LOGGER.warning("DataSource IntelX not found.") + mdl_source_inst = None # Set to None if DataSource is not found + for row in data.breach_data: # Check if record already exists row_dict = row.__dict__ + # mdl_source_inst = MDL_DataSource.objects.get( + # name='IntelX' + # ) + + MDL_CredentialBreaches.objects.update_or_create( + breach_name=row_dict["breach_name"], + defaults={ + "description": row_dict["description"], + "breach_date": row_dict["breach_date"], + "added_date": row_dict["added_date"], + "modified_date": timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), + "password_included": row_dict["password_included"], + "data_source": mdl_source_inst, + }, + ) breach_results = CredentialBreaches.objects.filter( breach_name=row_dict["breach_name"] ) @@ -5620,7 +5648,10 @@ def cred_breaches_intelx_insert( description=row_dict["description"], breach_date=row_dict["breach_date"], added_date=row_dict["added_date"], - modified_date=row_dict["modified_date"], + modified_date=timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), password_included=row_dict["password_included"], data_source_uid=curr_data_source_inst, ) @@ -5636,8 +5667,10 @@ def cred_breaches_intelx_insert( + str(update_count) + " records updated in the credential_breaches table" ) - except ObjectDoesNotExist: - LOGGER.info("API key expired please try again") + # except ObjectDoesNotExist: + # LOGGER.info("API key expired please try again") + except Exception as e: + LOGGER.info(f"Error: {e}") else: return {"message": "No api key was submitted"} @@ -5658,12 +5691,20 @@ def cred_exp_intelx_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx data create_cnt = 0 update_cnt = 0 + try: + mdl_source_inst = MDL_DataSource.objects.get(name="IntelX") + except MDL_DataSource.DoesNotExist: + LOGGER.warning("DataSource with IntelX not found.") + mdl_source_inst = None # Set to None if DataSource is not found for row in data.exp_data: row_dict = row.__dict__ + curr_org_inst = Organizations.objects.get( + organizations_uid=row_dict["organizations_uid"] + ) try: CredentialExposures.objects.get( breach_name=row_dict["breach_name"], @@ -5677,9 +5718,7 @@ def cred_exp_intelx_insert( update_cnt += 1 except CredentialExposures.DoesNotExist: # If record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( - organizations_uid=row_dict["organizations_uid"] - ) + curr_source_inst = DataSource.objects.get( data_source_uid=row_dict["data_source_uid"] ) @@ -5693,7 +5732,10 @@ def cred_exp_intelx_insert( root_domain=row_dict["root_domain"], sub_domain=row_dict["sub_domain"], breach_name=row_dict["breach_name"], - modified_date=row_dict["modified_date"], + modified_date=timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), data_source_uid=curr_source_inst, password=row_dict["password"], hash_type=row_dict["hash_type"], @@ -5701,6 +5743,44 @@ def cred_exp_intelx_insert( credential_breaches_uid=curr_breach_inst, ) create_cnt += 1 + try: + MDL_CredentialExposures.objects.get( + breach_name=row_dict["breach_name"], + email=row_dict["email"], + ) + MDL_CredentialExposures.objects.filter( + breach_name=row_dict["breach_name"], + email=row_dict["email"], + ).update(modified_date=row_dict["modified_date"]) + + except MDL_CredentialExposures.DoesNotExist: + mdl_org_inst = MDL_Organization.objects.get( + acronym=curr_org_inst.cyhy_db_name + ) + # mdl_source_inst = MDL_DataSource.objects.get( + # name='IntelX' + # ) + + mdl_breach_inst = MDL_CredentialBreaches.objects.get( + breach_name=row_dict["breach_name"], + ) + MDL_CredentialExposures.objects.create( + # credential_exposures_uid=uuid.uuid1(), + email=row_dict["email"], + organization=mdl_org_inst, + root_domain=row_dict["root_domain"], + sub_domain=row_dict["sub_domain"], + breach_name=row_dict["breach_name"], + modified_date=timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), + data_source=mdl_source_inst, + password=row_dict["password"], + hash_type=row_dict["hash_type"], + intelx_system_id=row_dict["intelx_system_id"], + credential_breaches=mdl_breach_inst, + ) # Return success message return ( str(create_cnt) @@ -5708,8 +5788,10 @@ def cred_exp_intelx_insert( + str(update_cnt) + " records updated in the credential_exposures table" ) - except ObjectDoesNotExist: - LOGGER.info("API key expired please try again") + # except ObjectDoesNotExist: + # LOGGER.info("API key expired please try again") + except Exception as e: + LOGGER.info(f"Error: {e}") else: return {"message": "No api key was submitted"} @@ -5730,23 +5812,27 @@ def xpanse_business_unit_insert_or_update( """Create API endpoint to create a record in database.""" if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) LOGGER.info(f"The api key submitted {tokens}") - + mapped_org = None mdl_mapped_org = None - #TODO: only for fceb for now, need to update for when we add more xpanse stakeholder + # TODO: only for fceb for now, need to update for when we add more xpanse stakeholder # match = re.search(r'\((.*?)\)', data.entity_name) if data.cyhy_db_name is not None: try: - mapped_org = Organizations.objects.get(cyhy_db_name=data.cyhy_db_name) + mapped_org = Organizations.objects.get( + cyhy_db_name=data.cyhy_db_name + ) except Organizations.DoesNotExist: mapped_org = None try: - mdl_mapped_org = MDL_Organization.objects.get(acronym=data.cyhy_db_name) + mdl_mapped_org = MDL_Organization.objects.get( + acronym=data.cyhy_db_name + ) except MDL_Organization.DoesNotExist: mdl_mapped_org = None - defaults={ + defaults = { "state": data.state, "county": data.county, "city": data.city, @@ -5754,10 +5840,10 @@ def xpanse_business_unit_insert_or_update( "entity_type": data.entity_type, "region": data.region, "rating": data.rating, - "cyhy_db_name": mapped_org + "cyhy_db_name": mapped_org, } - mdl_defaults={ + mdl_defaults = { "state": data.state, "county": data.county, "city": data.city, @@ -5765,11 +5851,11 @@ def xpanse_business_unit_insert_or_update( "entity_type": data.entity_type, "region": data.region, "rating": data.rating, - "cyhy_db_name": mdl_mapped_org + "cyhy_db_name": mdl_mapped_org, } # if mapped_org is not None: # defaults["cyhy_db_name"] = mapped_org - + ( mdl_business_unit_object, mdl_created, @@ -5782,8 +5868,7 @@ def xpanse_business_unit_insert_or_update( business_unit_object, created, ) = XpanseBusinessUnits.objects.update_or_create( - entity_name=data.entity_name, - defaults=defaults + entity_name=data.entity_name, defaults=defaults ) if created: LOGGER.info( @@ -5804,6 +5889,7 @@ def xpanse_business_unit_insert_or_update( else: return {"message": "No api key was submitted"} + @api_router.get( "/linked_xpanse_business_units", dependencies=[ @@ -5818,7 +5904,7 @@ def linked_xpanse_business_units(tokens: dict = Depends(get_api_key)): LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query xpanse_business_units = list( XpanseBusinessUnits.objects.filter(cyhy_db_name__isnull=False).values() @@ -5833,7 +5919,7 @@ def linked_xpanse_business_units(tokens: dict = Depends(get_api_key)): LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- xpanse endpoint, Issue 682 --- @api_router.put( @@ -5851,54 +5937,54 @@ def xpanse_alert_insert_or_update( """Create API endpoint to create a record in database.""" if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) LOGGER.info(f"The api key submitted {tokens}") LOGGER.info("Got into Xpanse Alert insert") # vender_prod_dict = data.vender_product alert_defaults = { - "time_pulled_from_xpanse": data.time_pulled_from_xpanse, - # "alert_id": data.alert_id, - "detection_timestamp": data.detection_timestamp, - "alert_name": data.alert_name, - "description": data.description, - "host_name": data.host_name, - "alert_action": data.alert_action, - "action_pretty": data.action_pretty, - "action_country": data.action_country, - "action_remote_port": data.action_remote_port, - "starred": data.starred, - "external_id": data.external_id, - "related_external_id": data.related_external_id, - "alert_occurrence": data.alert_occurrence, - "severity": data.severity, - "matching_status": data.matching_status, - "local_insert_ts": data.local_insert_ts, - "last_modified_ts": data.last_modified_ts, - "case_id": data.case_id, - "event_timestamp": data.event_timestamp, - "alert_type": data.alert_type, - "resolution_status": data.resolution_status, - "resolution_comment": data.resolution_comment, - "tags": data.tags, - "last_observed": data.last_observed, - "country_codes": data.country_codes, - "cloud_providers": data.cloud_providers, - "ipv4_addresses": data.ipv4_addresses, - "domain_names": data.domain_names, - "service_ids": data.service_ids, - "website_ids": data.website_ids, - "asset_ids": data.asset_ids, - "certificate": data.certificate, - "port_protocol": data.port_protocol, - "attack_surface_rule_name": data.attack_surface_rule_name, - "remediation_guidance": data.remediation_guidance, - "asset_identifiers": data.asset_identifiers - # business_units: Optional[List[str]] = None - # services: Optional[List[XpanseService]] = None - # assets : Optional[List[XpanseAsset]] = None - } - + "time_pulled_from_xpanse": data.time_pulled_from_xpanse, + # "alert_id": data.alert_id, + "detection_timestamp": data.detection_timestamp, + "alert_name": data.alert_name, + "description": data.description, + "host_name": data.host_name, + "alert_action": data.alert_action, + "action_pretty": data.action_pretty, + "action_country": data.action_country, + "action_remote_port": data.action_remote_port, + "starred": data.starred, + "external_id": data.external_id, + "related_external_id": data.related_external_id, + "alert_occurrence": data.alert_occurrence, + "severity": data.severity, + "matching_status": data.matching_status, + "local_insert_ts": data.local_insert_ts, + "last_modified_ts": data.last_modified_ts, + "case_id": data.case_id, + "event_timestamp": data.event_timestamp, + "alert_type": data.alert_type, + "resolution_status": data.resolution_status, + "resolution_comment": data.resolution_comment, + "tags": data.tags, + "last_observed": data.last_observed, + "country_codes": data.country_codes, + "cloud_providers": data.cloud_providers, + "ipv4_addresses": data.ipv4_addresses, + "domain_names": data.domain_names, + "service_ids": data.service_ids, + "website_ids": data.website_ids, + "asset_ids": data.asset_ids, + "certificate": data.certificate, + "port_protocol": data.port_protocol, + "attack_surface_rule_name": data.attack_surface_rule_name, + "remediation_guidance": data.remediation_guidance, + "asset_identifiers": data.asset_identifiers + # business_units: Optional[List[str]] = None + # services: Optional[List[XpanseService]] = None + # assets : Optional[List[XpanseAsset]] = None + } + # Create alerts in both databases alert_object, created = XpanseAlerts.objects.update_or_create( alert_id=data.alert_id, @@ -5908,13 +5994,17 @@ def xpanse_alert_insert_or_update( if created: LOGGER.info("new Xpanse alert record created for %s", data.alert_name) - mdl_alert_object, mdl_alert_created = MDL_XpanseAlerts.objects.update_or_create( + ( + mdl_alert_object, + mdl_alert_created, + ) = MDL_XpanseAlerts.objects.update_or_create( alert_id=data.alert_id, defaults=alert_defaults, ) if mdl_alert_created: - LOGGER.info("new Xpanse alert record created in MDL for %s", data.alert_name) - + LOGGER.info( + "new Xpanse alert record created in MDL for %s", data.alert_name + ) business_unit_list = [] mdl_business_unit_list = [] @@ -5927,45 +6017,47 @@ def xpanse_alert_insert_or_update( ) alert_object.business_units.set(business_unit_list) mdl_alert_object.business_units.set(mdl_business_unit_list) - asset_list = [] mdl_asset_list = [] for asset_data in data.assets: asset_defaults = { - "asset_name": asset_data.asset_name, - "asset_type": asset_data.asset_type, - "last_observed": asset_data.last_observed, - "first_observed": asset_data.first_observed, - "externally_detected_providers": asset_data.externally_detected_providers, - "created": asset_data.created, - "ips": asset_data.ips, - "active_external_services_types": asset_data.active_external_services_types, - "domain": asset_data.domain, - "certificate_issuer": asset_data.certificate_issuer, - "certificate_algorithm": asset_data.certificate_algorithm, - "certificate_classifications": asset_data.certificate_classifications, - "resolves": asset_data.resolves, - # details - "top_level_asset_mapper_domain": asset_data.top_level_asset_mapper_domain, - "domain_asset_type": asset_data.domain_asset_type, - "is_paid_level_domain": asset_data.is_paid_level_domain, - "domain_details": asset_data.domain_details, - "dns_zone": asset_data.dns_zone, - "latest_sampled_ip": asset_data.latest_sampled_ip, - "recent_ips": asset_data.recent_ips, - "external_services": asset_data.external_services, - "externally_inferred_vulnerability_score": asset_data.externally_inferred_vulnerability_score, - "externally_inferred_cves": asset_data.externally_inferred_cves, - "explainers": asset_data.explainers, - "tags": asset_data.tags, - } + "asset_name": asset_data.asset_name, + "asset_type": asset_data.asset_type, + "last_observed": asset_data.last_observed, + "first_observed": asset_data.first_observed, + "externally_detected_providers": asset_data.externally_detected_providers, + "created": asset_data.created, + "ips": asset_data.ips, + "active_external_services_types": asset_data.active_external_services_types, + "domain": asset_data.domain, + "certificate_issuer": asset_data.certificate_issuer, + "certificate_algorithm": asset_data.certificate_algorithm, + "certificate_classifications": asset_data.certificate_classifications, + "resolves": asset_data.resolves, + # details + "top_level_asset_mapper_domain": asset_data.top_level_asset_mapper_domain, + "domain_asset_type": asset_data.domain_asset_type, + "is_paid_level_domain": asset_data.is_paid_level_domain, + "domain_details": asset_data.domain_details, + "dns_zone": asset_data.dns_zone, + "latest_sampled_ip": asset_data.latest_sampled_ip, + "recent_ips": asset_data.recent_ips, + "external_services": asset_data.external_services, + "externally_inferred_vulnerability_score": asset_data.externally_inferred_vulnerability_score, + "externally_inferred_cves": asset_data.externally_inferred_cves, + "explainers": asset_data.explainers, + "tags": asset_data.tags, + } asset_object, created = XpanseAssets.objects.update_or_create( asm_id=asset_data.asm_id, defaults=asset_defaults, ) asset_list.append(asset_object) - mdl_asset_object, mdl_asset_created = MDL_XpanseAssets.objects.update_or_create( + ( + mdl_asset_object, + mdl_asset_created, + ) = MDL_XpanseAssets.objects.update_or_create( asm_id=asset_data.asm_id, defaults=asset_defaults, ) @@ -5977,30 +6069,33 @@ def xpanse_alert_insert_or_update( services_list = [] mdl_services_list = [] for service_data in data.services: - service_defaults={ - "service_name": service_data.service_name, - "service_type": service_data.service_type, - "ip_address": service_data.ip_address, - "domain": service_data.domain, - "externally_detected_providers": service_data.externally_detected_providers, - "is_active": service_data.is_active, - "first_observed": service_data.first_observed, - "last_observed": service_data.last_observed, - "port": service_data.port, - "protocol": service_data.protocol, - "active_classifications": service_data.active_classifications, - "inactive_classifications": service_data.inactive_classifications, - "discovery_type": service_data.discovery_type, - "externally_inferred_vulnerability_score": service_data.externally_inferred_vulnerability_score, - "externally_inferred_cves": service_data.externally_inferred_cves, - "service_key": service_data.service_key, - "service_key_type": service_data.service_key_type, - } + service_defaults = { + "service_name": service_data.service_name, + "service_type": service_data.service_type, + "ip_address": service_data.ip_address, + "domain": service_data.domain, + "externally_detected_providers": service_data.externally_detected_providers, + "is_active": service_data.is_active, + "first_observed": service_data.first_observed, + "last_observed": service_data.last_observed, + "port": service_data.port, + "protocol": service_data.protocol, + "active_classifications": service_data.active_classifications, + "inactive_classifications": service_data.inactive_classifications, + "discovery_type": service_data.discovery_type, + "externally_inferred_vulnerability_score": service_data.externally_inferred_vulnerability_score, + "externally_inferred_cves": service_data.externally_inferred_cves, + "service_key": service_data.service_key, + "service_key_type": service_data.service_key_type, + } service_object, created = XpanseServices.objects.update_or_create( service_id=service_data.service_id, defaults=service_defaults, ) - mdl_service_object, mdl_service_created = MDL_XpanseServices.objects.update_or_create( + ( + mdl_service_object, + mdl_service_created, + ) = MDL_XpanseServices.objects.update_or_create( service_id=service_data.service_id, defaults=service_defaults, ) @@ -6010,29 +6105,32 @@ def xpanse_alert_insert_or_update( LOGGER.info(cve_data) LOGGER.info(cve_match_data) cve_defaults = { - "cvss_score_v2": cve_data.cvss_score_v2, - "cve_severity_v2": cve_data.cve_severity_v2, - "cvss_score_v3": cve_data.cvss_score_v3, - "cve_severity_v3": cve_data.cve_severity_v3, - } + "cvss_score_v2": cve_data.cvss_score_v2, + "cve_severity_v2": cve_data.cve_severity_v2, + "cvss_score_v3": cve_data.cvss_score_v3, + "cve_severity_v3": cve_data.cve_severity_v3, + } cve_object, created = XpanseCves.objects.update_or_create( cve_id=cve_data.cve_id, defaults=cve_defaults, ) - mdl_cve_object, mdl_cve_created = MDL_XpanseCves.objects.update_or_create( + ( + mdl_cve_object, + mdl_cve_created, + ) = MDL_XpanseCves.objects.update_or_create( cve_id=cve_data.cve_id, defaults=cve_defaults, ) cve_service_default = { - "inferred_cve_match_type": cve_match_data.inferred_cve_match_type, - "product": cve_match_data.product, - "confidence": cve_match_data.confidence, - "vendor": cve_match_data.vendor, - "version_number": cve_match_data.version_number, - "activity_status": cve_match_data.activity_status, - "first_observed": cve_match_data.first_observed, - "last_observed": cve_match_data.last_observed, - } + "inferred_cve_match_type": cve_match_data.inferred_cve_match_type, + "product": cve_match_data.product, + "confidence": cve_match_data.confidence, + "vendor": cve_match_data.vendor, + "version_number": cve_match_data.version_number, + "activity_status": cve_match_data.activity_status, + "first_observed": cve_match_data.first_observed, + "last_observed": cve_match_data.last_observed, + } ( cve_match_object, created, @@ -6055,7 +6153,6 @@ def xpanse_alert_insert_or_update( alert_object.services.set(services_list) alert_object.save() - mdl_alert_object.services.set(mdl_services_list) mdl_alert_object.save() @@ -6442,6 +6539,7 @@ async def get_pshtt_domains_to_run_status( else: return {"message": "No api key was submitted"} + # --- get_orgs(), Issue 699 pe-reports --- @api_router.get( "/organizations_demo_or_report_on", @@ -6451,13 +6549,13 @@ async def get_pshtt_domains_to_run_status( response_model=List[schemas.OrganizationsFullTable], tags=["Retrieve data for all demo or report_on orgs."], ) -def organizations_demo(tokens: dict = Depends(get_api_key)): +def organizations_demo_or_report_on(tokens: dict = Depends(get_api_key)): """Call API endpoint to get data for all demo or report_on orgs.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query organizations_demo_or_report_on_data = list( Organizations.objects.filter(Q(demo=True) | Q(report_on=True)).values() @@ -6504,7 +6602,6 @@ def pshtt_result_update_or_insert( ) sub_domain_uid = SubDomains.objects.get(sub_domain_uid=data.sub_domain_uid) - pshtt_object, created = PshttResults.objects.update_or_create( sub_domain_uid=data.sub_domain_uid, organizations_uid=data.organizations_uid, @@ -6594,34 +6691,32 @@ def pshtt_result_update_or_insert( response_model=List[schemas.DataSourceFullTable], tags=["Retrieve data for specified data source name."], ) -def data_source_by_name(data: schemas.DataSourceByNameInput, tokens: dict = Depends(get_api_key)): +def data_source_by_name( + data: schemas.DataSourceByNameInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get data for specified data source name.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query data_source_by_name_data = list( DataSource.objects.filter(name=data.name).values() ) # also update data source record - today = dt.today().strftime("%Y-%m-%d") + today = datetime.today().strftime("%Y-%m-%d") DataSource.objects.filter(name=data.name).update(last_run=today) # Convert data types to match response model for row in data_source_by_name_data: - row["data_source_uid"] = convert_uuid_to_string( - row["data_source_uid"] - ) - row["last_run"] = convert_date_to_string( - row["last_run"] - ) + row["data_source_uid"] = convert_uuid_to_string(row["data_source_uid"]) + row["last_run"] = convert_date_to_string(row["last_run"]) return data_source_by_name_data except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_breaches(), Issue 701 pe-reports --- @api_router.get( @@ -6641,7 +6736,9 @@ def breach_names_and_uids(tokens: dict = Depends(get_api_key)): userapiTokenverify(theapiKey=tokens) # If API key valid, make query breach_names_and_uids_data = list( - CredentialBreaches.objects.all().values("breach_name", "credential_breaches_uid") + CredentialBreaches.objects.all().values( + "breach_name", "credential_breaches_uid" + ) ) # Convert data types to match response model for row in breach_names_and_uids_data: @@ -6653,7 +6750,7 @@ def breach_names_and_uids(tokens: dict = Depends(get_api_key)): LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- getSubdomain(), Issue 702 pe-reports --- @api_router.post( @@ -6664,7 +6761,9 @@ def breach_names_and_uids(tokens: dict = Depends(get_api_key)): response_model=List[schemas.SubdomainUIDByDomain], tags=["Retrieve data for the specified subdomain."], ) -def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = Depends(get_api_key)): +def subdomain_by_domain( + data: schemas.SubdomainUIDByDomainInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get data for specified subdomain.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -6673,7 +6772,9 @@ def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = userapiTokenverify(theapiKey=tokens) # If API key valid, make query subdomain_by_domain_data = list( - SubDomains.objects.filter(sub_domain=data.domain).values("sub_domain_uid") + SubDomains.objects.filter(sub_domain=data.domain).values( + "sub_domain_uid" + ) ) # Convert data types to match response model for row in subdomain_by_domain_data: @@ -6683,7 +6784,7 @@ def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- org_root_domains(), Issue 703 pe-reports --- @api_router.post( @@ -6694,25 +6795,30 @@ def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = response_model=List[schemas.RootDomainsTable], tags=["Retrieve root domains for the specified org uid."], ) -def rootdomains_by_org_uid(data: schemas.RootdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key)): +def rootdomains_by_org_uid( + data: schemas.RootdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get root domains for specified org uid.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query rootdomains_by_org_uid_data = list( RootDomains.objects.filter( - organizations_uid=data.org_uid, - enumerate_subs=True + organizations_uid=data.org_uid, enumerate_subs=True ).values() ) # Convert data types to match response model for row in rootdomains_by_org_uid_data: row["root_domain_uid"] = convert_uuid_to_string(row["root_domain_uid"]) - row["organizations_uid_id"] = convert_uuid_to_string(row["organizations_uid_id"]) - row["data_source_uid_id"] = convert_uuid_to_string(row["data_source_uid_id"]) + row["organizations_uid_id"] = convert_uuid_to_string( + row["organizations_uid_id"] + ) + row["data_source_uid_id"] = convert_uuid_to_string( + row["data_source_uid_id"] + ) return rootdomains_by_org_uid_data except ObjectDoesNotExist: LOGGER.info("API key expired please try again") @@ -6723,25 +6829,22 @@ def rootdomains_by_org_uid(data: schemas.RootdomainsByOrgUIDInput, tokens: dict @api_router.post( "/crossfeed_vulns", dependencies=[Depends(get_api_key)], - #response_model=schemas.PshttDomainToRunTaskResp,TODO, create schema for generlized output + # response_model=schemas.PshttDomainToRunTaskResp,TODO, create schema for generlized output tags=["Return all vulnerabilites formatted for crossfeed database."], ) -def crossfeed_vulns( - data: schemas.GenInputOrgName, - tokens: dict = Depends(get_api_key) - ): +def crossfeed_vulns(data: schemas.GenInputOrgName, tokens: dict = Depends(get_api_key)): """Returna all vulnerabilities for crossfeed database.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: tasks_dict = {} shodan_task = shodan_vulns_task.delay(data.org_acronym) - tasks_dict['shodan'] = shodan_task.id + tasks_dict["shodan"] = shodan_task.id cred_task = credential_breach_vulns_task.delay(data.org_acronym) - tasks_dict['creds'] = cred_task.id + tasks_dict["creds"] = cred_task.id was_task = was_vulns_task.delay(data.org_acronym) - tasks_dict['was'] = was_task.id - #TODO: add task for XPANSE data + tasks_dict["was"] = was_task.id + # TODO: add task for XPANSE data # Return the new task id w/ "Processing" status return {"tasks_dict": tasks_dict, "status": "Processing"} @@ -6862,15 +6965,15 @@ def domain_permu_insert_dnstwist( LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_root_domains(), Issue 707 pe-reports/006 atc-framework --- # This function reuses the /rootdomains_by_org_uid endpoint - + # --- getDataSource(), Issue 708 pe-reports/007 atc-framework --- # This function reuses the /data_source_by_name endpoint - + # --- execute_hibp_breach_values(), Issue 709/008 atc-framework --- @api_router.put( @@ -6927,7 +7030,7 @@ def cred_breaches_hibp_insert( ).update( modified_daate=row_dict["modified_date"], exposed_cred_count=row_dict["exposed_cred_count"], - password_included=row_dict["password_included"] + password_included=row_dict["password_included"], ) update_count += 1 return ( @@ -6940,7 +7043,7 @@ def cred_breaches_hibp_insert( LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- execute_hibp_emails_values(), Issue 710 pe-reports/009 atc-framework --- @api_router.put( @@ -6995,14 +7098,13 @@ def cred_exp_hibp_insert( create_cnt += 1 # Return success message return ( - str(create_cnt) - + " records created in the credential_exposures table" + str(create_cnt) + " records created in the credential_exposures table" ) except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_breach_uids(), Issue 010 atc-framework --- @api_router.get( @@ -7023,7 +7125,7 @@ def breach_uids(tokens: dict = Depends(get_api_key)): # If API key valid, make query breach_uids_data = list( CredentialBreaches.objects.all().values( - "breach_name", + "breach_name", "credential_breaches_uid", ) ) @@ -7037,7 +7139,7 @@ def breach_uids(tokens: dict = Depends(get_api_key)): LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- query_orgs(), Issue 011 atc-framework --- @api_router.get( @@ -7080,7 +7182,9 @@ def reported_orgs(tokens: dict = Depends(get_api_key)): response_model=List[schemas.SubdomainsByOrgUID], tags=["Retrieve subdomains for the specified org uid."], ) -def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key)): +def subdomains_by_org_uid( + data: schemas.SubdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get subdomains for specified org uid.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -7089,11 +7193,8 @@ def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = userapiTokenverify(theapiKey=tokens) # If API key valid, make query subdomains_by_org_uid_data = list( - RootDomains.objects.filter( - organizations_uid=data.org_uid - ).values( - "root_domains_uid__sub_domain", - "root_domain" + RootDomains.objects.filter(organizations_uid=data.org_uid).values( + "root_domains_uid__sub_domain", "root_domain" ) ) return subdomains_by_org_uid_data @@ -7101,7 +7202,7 @@ def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- insert_shodan_assets(), Issue 016 atc-framework --- @api_router.put( @@ -7112,29 +7213,35 @@ def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = tags=["Insert Shodan data into the shodan_assets table."], ) def shodan_assets_insert( - data: schemas.ShodanAssetsInsertInput, tokens: dict = Depends(get_api_key)): + data: schemas.ShodanAssetsInsertInput, tokens: dict = Depends(get_api_key) +): """Insert Shodan data into the shodan_assets table using the API endpoint.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key is valid, proceed with the operation update_create_count = 0 + try: + mdl_data_source = MDL_DataSource.objects.get(name="Shodan") + + except MDL_DataSource.DoesNotExist: + LOGGER.warning("DataSource 'Shodan' not found.") + mdl_data_source = None # Set to None if DataSource is not found + for row in data.asset_data: row_dict = row.__dict__ try: - org_instance = Organizations.objects.get(organizations_uid=row_dict["organizations_uid"]) - acronym = org_instance.acronym # Assuming 'acronym' is a field in Organizations model - # Assuming ExternalOrganizations is a model that matches organization acronym to an organization_id + org_instance = Organizations.objects.get( + organizations_uid=row_dict["organizations_uid"] + ) + + acronym = org_instance.cyhy_db_name + mdl_org = MDL_Organization.objects.get(acronym=acronym) - try: - mdl_data_source = MDL_DataSource.objects.get(name="Shodan") - except DataSource.DoesNotExist: - LOGGER.warning(f"DataSource with UID {row_dict['data_source_uid_id']} not found.") - mdl_data_source = None # Set to None if DataSource is not found - + mdl_asset_fields = { "asn": row_dict.get("asn"), "domains": row_dict.get("domains", []), @@ -7147,17 +7254,19 @@ def shodan_assets_insert( "location": row_dict.get("location"), "data_source": mdl_data_source, } - + mdl_obj, created = MDL_ShodanAssets.objects.update_or_create( organization=mdl_org, # Directly use organizations_uid ip=row_dict["ip"], port=row_dict["port"], protocol=row_dict["protocol"], - timestamp=row_dict["timestamp"], - defaults=mdl_asset_fields + timestamp=timezone.make_aware( + parse_datetime(row_dict["timestamp"]), timezone.timezone.utc + ), + defaults=mdl_asset_fields, ) - except: - LOGGER.warning(f"Shodan Asset failed to save to MDL.") + except Exception as e: + LOGGER.warning(f"Shodan Asset failed to save to MDL: {e}") try: # Prepare the fields that will be used for create or update (only non-unique fields here) @@ -7176,22 +7285,29 @@ def shodan_assets_insert( # Use 'update_or_create' to either create or update the record obj, created = ShodanAssets.objects.update_or_create( - organizations_uid=row_dict["organizations_uid"], # Directly use organizations_uid + organizations_uid=org_instance, # Directly use organizations_uid ip=row_dict["ip"], port=row_dict["port"], protocol=row_dict["protocol"], - timestamp=row_dict["timestamp"], - defaults=asset_fields + timestamp=timezone.make_aware( + datetime.strptime( + row_dict["timestamp"], "%Y-%m-%dT%H:%M:%S.%f" + ), + timezone.timezone.utc, + ), + defaults=asset_fields, ) if created: update_create_count += 1 - except: - LOGGER.warning(f"Shodan Asset failed to save to PE DB.") + except Exception as e: + LOGGER.warning(f"Shodan Asset failed to save to PE DB: {e}") continue # Return the success message with the count of created/updated records - return {"message": f"{update_create_count} records created/updated in the shodan_assets table."} + return { + "message": f"{update_create_count} records created/updated in the shodan_assets table." + } except ObjectDoesNotExist: LOGGER.info("API key expired or invalid. Please try again.") @@ -7202,6 +7318,7 @@ def shodan_assets_insert( else: return {"message": "No API key was submitted."} + # --- insert_shodan_vulns(), Issue 017 atc-framework --- @api_router.put( "/shodan_vulns_insert", @@ -7218,21 +7335,23 @@ def shodan_vulns_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx data create_cnt = 0 + try: + mdl_data_source = MDL_DataSource.objects.get(name="Shodan") + except DataSource.DoesNotExist: + LOGGER.warning("DataSource for 'Shodan' not found.") + mdl_data_source = None # Set to None if DataSource is not found for row in data.vuln_data: row_dict = row.__dict__ try: - org_instance = Organizations.objects.get(organizations_uid=row_dict["organizations_uid"]) - acronym = org_instance.acronym # Assuming 'acronym' is a field in Organizations model - # Assuming ExternalOrganizations is a model that matches organization acronym to an organization_id + org_instance = Organizations.objects.get( + organizations_uid=row_dict["organizations_uid"] + ) + acronym = org_instance.cyhy_db_name + mdl_org = MDL_Organization.objects.get(acronym=acronym) - try: - mdl_data_source = MDL_DataSource.objects.get(name="Shodan") - except DataSource.DoesNotExist: - LOGGER.warning(f"DataSource with UID {row_dict['data_source_uid_id']} not found.") - mdl_data_source = None # Set to None if DataSource is not found mdl_vuln_data = { "organization_name": row_dict.get("organization"), @@ -7245,7 +7364,9 @@ def shodan_vulns_insert( "av_description": row_dict.get("av_description"), "attack_complexity": row_dict.get("attack_complexity"), "ac_description": row_dict.get("ac_description"), - "confidentiality_impact": row_dict.get("confidentiality_impact"), + "confidentiality_impact": row_dict.get( + "confidentiality_impact" + ), "ci_description": row_dict.get("ci_description"), "integrity_impact": row_dict.get("integrity_impact"), "ii_description": row_dict.get("ii_description"), @@ -7265,7 +7386,7 @@ def shodan_vulns_insert( "is_verified": row_dict.get("is_verified"), "banner": row_dict.get("banner"), "version": row_dict.get("version"), - "cpe": row_dict.get("cpe") + "cpe": row_dict.get("cpe"), } mdl_obj, created = MDL_ShodanVulns.objects.update_or_create( @@ -7273,11 +7394,15 @@ def shodan_vulns_insert( ip=row_dict["ip"], port=row_dict["port"], protocol=row_dict["protocol"], - timestamp=row_dict["timestamp"], - defaults=mdl_vuln_data) - except: - LOGGER.warning(f"Shodan Vuln failed to save to MDL.") - + timestamp=timezone.make_aware( + parse_datetime(row_dict["timestamp"]) + ), + defaults=mdl_vuln_data, + ) + + except Exception as e: + LOGGER.warning(f"Shodan Vuln failed to save to MDL: {e}") + try: vuln_data = { "organization": row_dict.get("organization"), @@ -7290,7 +7415,9 @@ def shodan_vulns_insert( "av_description": row_dict.get("av_description"), "attack_complexity": row_dict.get("attack_complexity"), "ac_description": row_dict.get("ac_description"), - "confidentiality_impact": row_dict.get("confidentiality_impact"), + "confidentiality_impact": row_dict.get( + "confidentiality_impact" + ), "ci_description": row_dict.get("ci_description"), "integrity_impact": row_dict.get("integrity_impact"), "ii_description": row_dict.get("ii_description"), @@ -7310,31 +7437,34 @@ def shodan_vulns_insert( "is_verified": row_dict.get("is_verified"), "banner": row_dict.get("banner"), "version": row_dict.get("version"), - "cpe": row_dict.get("cpe") + "cpe": row_dict.get("cpe"), } - obj, created = MDL_ShodanVulns.objects.update_or_create( + obj, created = ShodanVulns.objects.update_or_create( organizations_uid=org_instance, # Directly use organizations_uid ip=row_dict["ip"], port=row_dict["port"], protocol=row_dict["protocol"], - timestamp=row_dict["timestamp"], - defaults=vuln_data) + timestamp=timezone.make_aware( + datetime.strptime( + row_dict["timestamp"], "%Y-%m-%dT%H:%M:%S.%f" + ) + ), + defaults=vuln_data, + ) + LOGGER.info(f"pe vuln created: {created}") if created: - create_cnt+=1 - except: - LOGGER.warning(f"Shodan Vuln failed to save to PE DB.") + create_cnt += 1 + except Exception as e: + LOGGER.warning(f"Shodan Vuln failed to save to PE DB: {e}") continue # Return success message - return ( - str(create_cnt) - + " records created in the shodan vulns table" - ) + return str(create_cnt) + " records created in the shodan vulns table" except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_demo_orgs(), Issue 018 atc-framework --- @api_router.get( @@ -7367,6 +7497,7 @@ def organizations_demo(tokens: dict = Depends(get_api_key)): else: return {"message": "No api key was submitted"} + # --- Endpoint for Orgs with assets query (no view) --- @api_router.post( "/orgs_and_assets", @@ -7376,7 +7507,9 @@ def organizations_demo(tokens: dict = Depends(get_api_key)): response_model=schemas.OrgAssetTaskResp, tags=["Call API endpoint to get list all orgs and their linked assets."], ) -def orgs_and_assets(data: schemas.OrgsAssetsPagedInput, tokens: dict = Depends(get_api_key)): +def orgs_and_assets( + data: schemas.OrgsAssetsPagedInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get list all orgs and their linked assets.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -7401,7 +7534,9 @@ def orgs_and_assets(data: schemas.OrgsAssetsPagedInput, tokens: dict = Depends(g response_model=schemas.OrgAssetTaskResp, tags=["Get task status for orgs and assets task."], ) -async def get_orgs_and_assets_task_status(task_id: str, tokens: dict = Depends(get_api_key)): +async def get_orgs_and_assets_task_status( + task_id: str, tokens: dict = Depends(get_api_key) +): """Get task status for orgs and assets task.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -7431,4 +7566,51 @@ async def get_orgs_and_assets_task_status(task_id: str, tokens: dict = Depends(g except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: - return {"message": "No api key was submitted"} \ No newline at end of file + return {"message": "No api key was submitted"} + + +@api_router.get( + "/query_shodan_ips/{org_uid}", + dependencies=[ + Depends(get_api_key) + ], # Depends(RateLimiter(times=200, seconds=60))], + # response_model=List[schemas.OrgsReportOnContacts], + tags=["Get all ips to run through Shodan."], +) +def query_shodan_ips(org_uid: str, tokens: dict = Depends(get_api_key)): + """Create API endpoint to get all ips to run through Shodan..""" + # Check for API key + LOGGER.info(f"The api key submitted {tokens}") + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # If API key valid, make query + ips_from_cidrs = Ips.objects.filter( + origin_cidr__organizations_uid=org_uid, + origin_cidr__isnull=False, + shodan_results=True, + current=True, + ).values_list("ip", flat=True) + + ips_from_subs = Ips.objects.filter( + ipssubs__sub_domain_uid__root_domain_uid__organizations_uid=org_uid, # Correct relationship traversal + shodan_results=True, # 'shodan_results' is True + ipssubs__sub_domain_uid__current=True, # 'current' is True for subdomains + current=True, # 'current' is True for Ips + ).values_list("ip", flat=True) + + # Convert the QuerySet to sets + in_first = set(ips_from_cidrs) + in_second = set(ips_from_subs) + + # Find IPs that are in the second query but not in the first + in_second_but_not_in_first = in_second - in_first + + # Combine the results + ips = list(ips_from_cidrs) + list(in_second_but_not_in_first) + + return ips + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py old mode 100755 new mode 100644 index 953759d4..ef9f53da --- a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py @@ -1,13 +1,18 @@ -# Create your models here. -""" Django ORM models """ +"""Django ORM models.""" +# Standard Python Libraries +import uuid -from django.db import models +# Third-Party Libraries +# from django.contrib.auth.models import User as AuthUser from django.contrib.postgres.fields import ArrayField -from django.contrib.auth.models import User as AuthUser -from netfields import InetAddressField, NetManager -import uuid +from django.db import models +from netfields import InetAddressField + +# , NetManager manage_db = True + + class ApiKey(models.Model): """The ApiKey model.""" @@ -24,7 +29,7 @@ class ApiKey(models.Model): class Meta: """Meta class for ApiKey.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "api_key" @@ -44,7 +49,7 @@ class Assessment(models.Model): class Meta: """The Meta class for Assessment.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "assessment" @@ -62,7 +67,7 @@ class Category(models.Model): class Meta: """The Meta class for Category model.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "category" @@ -79,7 +84,7 @@ class Cpe(models.Model): class Meta: """The Meta class for Cpe.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" db_table = "cpe" managed = manage_db # This ensures Django does not manage the table unique_together = (("name", "version", "vendor"),) # Unique constraint @@ -94,9 +99,15 @@ class Cve(models.Model): modified_at = models.DateTimeField(db_column="modified_at", blank=True, null=True) status = models.CharField(blank=True, null=True, max_length=255) description = models.TextField(blank=True, null=True) - cvss_v2_source = models.CharField(db_column="cvss_v2_source", blank=True, null=True, max_length=255) - cvss_v2_type = models.CharField(db_column="cvss_v2_type", blank=True, null=True, max_length=255) - cvss_v2_version = models.CharField(db_column="cvss_v2_version", blank=True, null=True, max_length=255) + cvss_v2_source = models.CharField( + db_column="cvss_v2_source", blank=True, null=True, max_length=255 + ) + cvss_v2_type = models.CharField( + db_column="cvss_v2_type", blank=True, null=True, max_length=255 + ) + cvss_v2_version = models.CharField( + db_column="cvss_v2_version", blank=True, null=True, max_length=255 + ) cvss_v2_vector_string = models.CharField( db_column="cvss_v2_vector_string", blank=True, null=True, max_length=255 ) @@ -112,9 +123,15 @@ class Cve(models.Model): cvss_v2_impact_score = models.CharField( db_column="cvss_v2_impact_score", blank=True, null=True, max_length=255 ) - cvss_v3_source = models.CharField(db_column="cvss_v3_source", blank=True, null=True, max_length=255) - cvss_v3_type = models.CharField(db_column="cvss_v3_type", blank=True, null=True, max_length=255) - cvss_v3_version = models.CharField(db_column="cvss_v3_version", blank=True, null=True, max_length=255) + cvss_v3_source = models.CharField( + db_column="cvss_v3_source", blank=True, null=True, max_length=255 + ) + cvss_v3_type = models.CharField( + db_column="cvss_v3_type", blank=True, null=True, max_length=255 + ) + cvss_v3_version = models.CharField( + db_column="cvss_v3_version", blank=True, null=True, max_length=255 + ) cvss_v3_vector_string = models.CharField( db_column="cvss_v3_vector_string", blank=True, null=True, max_length=255 ) @@ -130,9 +147,15 @@ class Cve(models.Model): cvss_v3_impact_score = models.CharField( db_column="cvss_v3_impact_score", blank=True, null=True, max_length=255 ) - cvss_v4_source = models.CharField(db_column="cvss_v4_source", blank=True, null=True, max_length=255) - cvss_v4_type = models.CharField(db_column="cvss_v4_type", blank=True, null=True, max_length=255) - cvss_v4_version = models.CharField(db_column="cvss_v4_version", blank=True, null=True, max_length=255) + cvss_v4_source = models.CharField( + db_column="cvss_v4_source", blank=True, null=True, max_length=255 + ) + cvss_v4_type = models.CharField( + db_column="cvss_v4_type", blank=True, null=True, max_length=255 + ) + cvss_v4_version = models.CharField( + db_column="cvss_v4_version", blank=True, null=True, max_length=255 + ) cvss_v4_vector_string = models.CharField( db_column="cvss_v4_vector_string", blank=True, null=True, max_length=255 ) @@ -154,77 +177,76 @@ class Cve(models.Model): max_digits=1000, decimal_places=1000, blank=True, null=True ) - cpes = models.ManyToManyField(Cpe, related_name='cves', blank=True) + cpes = models.ManyToManyField(Cpe, related_name="cves", blank=True) # tickets = models.ManyToManyField("Ticket", related_name='cves', blank=True) # vuln_scans = models.ManyToManyField("VulnScan", related_name='cves', blank=True) - class Meta: """The Meta class for Cve.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cve" - -# This will likely be handled via the many to many field -# class CveCpesCpe(models.Model): -# """The CveCpesCpe model.""" - -# cve_id = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column="cve_id") -# cpe_id = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column="cpe_id") - -# class Meta: -# """The Meta class for CveCpesCpe model.""" - -# db_table = "cve_cpes_cpe" -# managed = False # This ensures Django does not manage the table -# unique_together = (("cve", "cpe"),) # Unique constraint - -# This is crossfeeds domain model, which lines up better with the pe subdomain table -# class Domain(models.Model): -# """The Domain model.""" - -# id = models.UUIDField(primary_key=True) -# created_at = models.DateTimeField(db_column="created_at") -# updated_at = models.DateTimeField(db_column="updated_at") -# synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True) -# ip = models.CharField(max_length=255, blank=True, null=True) -# from_root_domain = models.CharField(db_column="from_root_domain", blank=True, null=True) -# subdomain_source = models.CharField( -# db_column="subdomain_source", max_length=255, blank=True, null=True -# ) -# ip_only = models.BooleanField(db_column="ip_only", default=False) -# reverse_name = models.CharField(db_column="reverse_name", max_length=512) -# name = models.CharField(max_length=512) -# screenshot = models.CharField(max_length=512, blank=True, null=True) -# country = models.CharField(max_length=255, blank=True, null=True) -# asn = models.CharField(max_length=255, blank=True, null=True) -# cloud_hosted = models.BooleanField(db_column="cloud_hosted", default=False) -# ssl = models.JSONField(blank=True, null=True) -# censys_certificates_results = models.JSONField( -# db_column="censys_certificates_results", default=dict -# ) -# trustymail_results = models.JSONField(db_column="trustymail_results", default=dict) -# discovered_by = models.ForeignKey( -# "Scan", -# on_delete=models.SET_NULL, -# db_column="discovered_by_id", -# blank=True, -# null=True, -# ) -# organization = models.ForeignKey( -# "Organization", on_delete=models.CASCADE, db_column="organization_id" -# ) - -# class Meta: -# """The meta class for Domain.""" - -# db_table = "domain" -# managed = False # This ensures Django does not manage the table -# unique_together = (("name", "organization"),) # Unique constraint + # This will likely be handled via the many to many field + # class CveCpesCpe(models.Model): + # """The CveCpesCpe model.""" + + # cve_id = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column="cve_id") + # cpe_id = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column="cpe_id") + + # class Meta: + # """The Meta class for CveCpesCpe model.""" + + # db_table = "cve_cpes_cpe" + # managed = False # This ensures Django does not manage the table + # unique_together = (("cve", "cpe"),) # Unique constraint + + # This is crossfeeds domain model, which lines up better with the pe subdomain table + # class Domain(models.Model): + # """The Domain model.""" + + # id = models.UUIDField(primary_key=True) + # created_at = models.DateTimeField(db_column="created_at") + # updated_at = models.DateTimeField(db_column="updated_at") + # synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True) + # ip = models.CharField(max_length=255, blank=True, null=True) + # from_root_domain = models.CharField(db_column="from_root_domain", blank=True, null=True) + # subdomain_source = models.CharField( + # db_column="subdomain_source", max_length=255, blank=True, null=True + # ) + # ip_only = models.BooleanField(db_column="ip_only", default=False) + # reverse_name = models.CharField(db_column="reverse_name", max_length=512) + # name = models.CharField(max_length=512) + # screenshot = models.CharField(max_length=512, blank=True, null=True) + # country = models.CharField(max_length=255, blank=True, null=True) + # asn = models.CharField(max_length=255, blank=True, null=True) + # cloud_hosted = models.BooleanField(db_column="cloud_hosted", default=False) + # ssl = models.JSONField(blank=True, null=True) + # censys_certificates_results = models.JSONField( + # db_column="censys_certificates_results", default=dict + # ) + # trustymail_results = models.JSONField(db_column="trustymail_results", default=dict) + # discovered_by = models.ForeignKey( + # "Scan", + # on_delete=models.SET_NULL, + # db_column="discovered_by_id", + # blank=True, + # null=True, + # ) + # organization = models.ForeignKey( + # "Organization", on_delete=models.CASCADE, db_column="organization_id" + # ) + + # class Meta: + # """The meta class for Domain.""" + + # db_table = "domain" + # managed = False # This ensures Django does not manage the table + # unique_together = (("name", "organization"),) # Unique constraint def save(self, *args, **kwargs): + """Format the model before saving.""" self.name = self.name.lower() self.reverseName = ".".join(reversed(self.name.split("."))) super().save(*args, **kwargs) @@ -244,13 +266,15 @@ class Notification(models.Model): db_column="maintenance_type", blank=True, null=True, max_length=255 ) status = models.CharField(blank=True, null=True, max_length=255) - updated_by = models.CharField(db_column="updated_by", blank=True, null=True, max_length=255) + updated_by = models.CharField( + db_column="updated_by", blank=True, null=True, max_length=255 + ) message = models.TextField(blank=True, null=True) class Meta: """The Meta class for Notification.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "notification" @@ -265,20 +289,30 @@ class Organization(models.Model): retired = models.BooleanField(default=False, null=True, blank=True) name = models.CharField(max_length=255) root_domains = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True, db_column="root_domains" + models.TextField(blank=True, null=True), + blank=True, + null=True, + db_column="root_domains", ) ip_blocks = models.TextField(db_column="ip_blocks") # This field type is a guess. is_passive = models.BooleanField(db_column="is_passive") pending_domains = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True, db_column="pending_domains" - ) # This field type is a guess + models.TextField(blank=True, null=True), + blank=True, + null=True, + db_column="pending_domains", + ) # This field type is a guess date_pe_first_reported = models.DateTimeField(blank=True, null=True) country = models.TextField(blank=True, null=True) country_name = models.TextField(blank=True, null=True) state = models.CharField(blank=True, null=True, max_length=255) - region_id = models.CharField(db_column="region_id", blank=True, null=True, max_length=255) + region_id = models.CharField( + db_column="region_id", blank=True, null=True, max_length=255 + ) state_fips = models.IntegerField(db_column="state_fips", blank=True, null=True) - state_name = models.CharField(db_column="state_name", blank=True, null=True, max_length=255) + state_name = models.CharField( + db_column="state_name", blank=True, null=True, max_length=255 + ) county = models.TextField(blank=True, null=True) county_fips = models.IntegerField(db_column="county_fips", blank=True, null=True) type = models.CharField(blank=True, null=True, max_length=255) @@ -298,15 +332,25 @@ class Organization(models.Model): receives_cybex_report = models.BooleanField(blank=True, null=True) init_stage = models.CharField(max_length=255, null=True, blank=True) scheduler = models.CharField(max_length=255, null=True, blank=True) - enrolled_in_vs_timestamp = models.DateTimeField(db_column="enrolled_in_vs_timestamp", auto_now=True) - period_start_vs_timestamp = models.DateTimeField(db_column="period_start_vs_timestamp", auto_now=True) + enrolled_in_vs_timestamp = models.DateTimeField( + db_column="enrolled_in_vs_timestamp", auto_now=True + ) + period_start_vs_timestamp = models.DateTimeField( + db_column="period_start_vs_timestamp", auto_now=True + ) report_types = models.JSONField(null=True, blank=True, default=list) scan_types = models.JSONField(null=True, blank=True, default=list) scan_windows = models.JSONField(null=True, blank=True, default=list) scan_limits = models.JSONField(null=True, blank=True, default=list) password = models.TextField(blank=True, null=True) cyhy_period_start = models.DateField(blank=True, null=True) - location = models.ForeignKey("Location", related_name='organizations', on_delete=models.SET_NULL, null=True, blank=True) + location = models.ForeignKey( + "Location", + related_name="organizations", + on_delete=models.SET_NULL, + null=True, + blank=True, + ) # sectors = models.ManyToManyField("Sector", related_name='organizations', blank=True) covered in sectors table already # cidrs = models.ManyToManyField("Cidr", related_name='organizations', blank=True) covered in the cidr table already # vuln_scans = models.ManyToManyField("VulnScan", related_name='organizations', blank=True) @@ -325,10 +369,11 @@ class Organization(models.Model): blank=True, null=True, ) + class Meta: """The meta class for Organization.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "organization" @@ -340,14 +385,18 @@ class OrganizationTag(models.Model): created_at = models.DateTimeField(db_column="created_at") updated_at = models.DateTimeField(db_column="updated_at") name = models.CharField(unique=True, max_length=255) - organization = models.ManyToManyField("Organization", related_name='organization_tags', blank=True) + organization = models.ManyToManyField( + "Organization", related_name="organization_tags", blank=True + ) + class Meta: """The Meta class for OrganizationTag.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "organization_tag" + # Probably can be removed and merged with a many to many relationship # class OrganizationTagOrganizationsOrganization(models.Model): # """The OrganizationTagOrganizationsOrganization model.""" @@ -372,6 +421,7 @@ class Meta: class QueryResultCache(models.Model): """The QueryResultCache model.""" + id = models.UUIDField(primary_key=True) identifier = models.CharField(blank=True, null=True, max_length=255) time = models.BigIntegerField() @@ -382,7 +432,7 @@ class QueryResultCache(models.Model): class Meta: """The Meta class for QueryResultCache.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "query-result-cache" @@ -403,10 +453,11 @@ class Meta: """The Meta class for Question.""" db_table = "question" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db unique_together = (("category", "number"),) + # Created via Many to Many field # Question and Resource many-to-many # class QuestionResourcesResource(models.Model): @@ -435,12 +486,12 @@ class Resource(models.Model): name = models.CharField(max_length=255) type = models.CharField(max_length=255) url = models.TextField(unique=True) - questions = models.ManyToManyField(Question, related_name='resources', blank=True) + questions = models.ManyToManyField(Question, related_name="resources", blank=True) class Meta: """The Meta class for Resource.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "resource" @@ -460,7 +511,7 @@ class Response(models.Model): class Meta: """The Meta class for Resource.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "response" unique_together = (("assessment_id", "question_id"),) @@ -504,7 +555,7 @@ class Role(models.Model): class Meta: """The Meta class for Role.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "role" unique_together = (("user_id", "organization_id"),) @@ -532,7 +583,7 @@ class SavedSearch(models.Model): class Meta: """The Meta class for SavedSearch.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "saved_search" @@ -556,16 +607,21 @@ class Scan(models.Model): created_by = models.ForeignKey( "User", models.DO_NOTHING, db_column="created_by", blank=True, null=True ) - organizations = models.ManyToManyField(Organization, related_name="scans", blank=True) - organization_tags = models.ManyToManyField(OrganizationTag, related_name="scans", blank=True) + organizations = models.ManyToManyField( + Organization, related_name="scans", blank=True + ) + organization_tags = models.ManyToManyField( + OrganizationTag, related_name="scans", blank=True + ) class Meta: """The Meta class for Scan.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "scan" + # Taken Care of via many to many field # class ScanOrganizationsOrganization(models.Model): # """The ScanOrganizationsOrganization model.""" @@ -611,7 +667,9 @@ class ScanTask(models.Model): updated_at = models.DateTimeField(db_column="updated_at") status = models.TextField() type = models.TextField() - fargate_task_arn = models.TextField(db_column="fargate_task_arn", blank=True, null=True) + fargate_task_arn = models.TextField( + db_column="fargate_task_arn", blank=True, null=True + ) input = models.TextField(blank=True, null=True) output = models.TextField(blank=True, null=True) requested_at = models.DateTimeField(db_column="requested_at", blank=True, null=True) @@ -628,15 +686,18 @@ class ScanTask(models.Model): scan = models.ForeignKey( Scan, models.DO_NOTHING, db_column="scan_id", blank=True, null=True ) - organization_tags = models.ManyToManyField(OrganizationTag, related_name="scan_tasks", blank=True) + organization_tags = models.ManyToManyField( + OrganizationTag, related_name="scan_tasks", blank=True + ) class Meta: """The Meta class for ScanTask.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "scan_task" + # Managed via many to many # class ScanTaskOrganizationsOrganization(models.Model): # """The ScanTaskOrganizationsOrganization model.""" @@ -683,11 +744,12 @@ class Service(models.Model): class Meta: """The Meta class for Service.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "service" unique_together = (("port", "domain"),) + # ????Not sure if this is necessary since we are removing typeorm????? # class TypeormMetadata(models.Model): # """The TypeormMetadata model.""" @@ -714,7 +776,7 @@ class User(models.Model): db_column="cognitoId", unique=True, blank=True, null=True, max_length=255 ) login_gov_id = models.CharField( - db_column="login_gov_id", unique=True, blank=True, null=True, max_length=255 + db_column="login_gov_id", unique=True, blank=True, null=True, max_length=255 ) created_at = models.DateTimeField(db_column="created_at") updated_at = models.DateTimeField(db_column="updated_at") @@ -732,16 +794,22 @@ class User(models.Model): accepted_terms_version = models.TextField( db_column="accepted_terms_version", blank=True, null=True ) - last_logged_in = models.DateTimeField(db_column="last_logged_in", blank=True, null=True) + last_logged_in = models.DateTimeField( + db_column="last_logged_in", blank=True, null=True + ) user_type = models.TextField(db_column="user_type") - region_id = models.CharField(db_column="region_id", blank=True, null=True, max_length=255) + region_id = models.CharField( + db_column="region_id", blank=True, null=True, max_length=255 + ) state = models.CharField(blank=True, null=True, max_length=255) - okta_id = models.CharField(db_column="okta_id", unique=True, blank=True, null=True, max_length=255) + okta_id = models.CharField( + db_column="okta_id", unique=True, blank=True, null=True, max_length=255 + ) class Meta: """The Meta class for User.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "user" @@ -759,9 +827,7 @@ class Vulnerability(models.Model): cpe = models.TextField(blank=True, null=True) description = models.TextField() references = models.JSONField() - cvss = models.DecimalField( - max_digits=100, decimal_places=5, blank=True, null=True - ) + cvss = models.DecimalField(max_digits=100, decimal_places=5, blank=True, null=True) severity = models.TextField(blank=True, null=True) needs_population = models.BooleanField(db_column="needs_population") state = models.TextField() @@ -782,7 +848,7 @@ class Vulnerability(models.Model): class Meta: """The Meta class for Vulnerability.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "vulnerability" unique_together = (("domain", "title"),) @@ -817,38 +883,58 @@ class Webpage(models.Model): class Meta: """The Meta class for Webpage.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "webpage" unique_together = (("url", "domain"),) -######### VS Models ######### +# ######## VS Models ######### class TicketEvent(models.Model): + """The TicketEvent model.""" + id = models.UUIDField(primary_key=True, editable=False) reference = models.CharField(max_length=255, null=True, blank=True) - vuln_scan = models.ForeignKey("VulnScan", on_delete=models.CASCADE, db_column = "vuln_scan_id", null=True, blank=True, related_name='ticket_events') + vuln_scan = models.ForeignKey( + "VulnScan", + on_delete=models.CASCADE, + db_column="vuln_scan_id", + null=True, + blank=True, + related_name="ticket_events", + ) action = models.CharField(max_length=255, null=True, blank=True) reason = models.CharField(max_length=255, null=True, blank=True) event_timestamp = models.DateTimeField(null=True, blank=True) delta = models.JSONField(default=list) - ticket = models.ForeignKey("Ticket", on_delete=models.CASCADE, db_column = "ticket_id",null=True, blank=True, related_name='ticket_events') + ticket = models.ForeignKey( + "Ticket", + on_delete=models.CASCADE, + db_column="ticket_id", + null=True, + blank=True, + related_name="ticket_events", + ) class Meta: """The Meta class for TicketEvent.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "ticket_event" - unique_together = ('event_timestamp', 'ticket', 'action') + unique_together = ("event_timestamp", "ticket", "action") + class VulnScan(models.Model): """The VS Vuln Scan model.""" + id = models.CharField(max_length=255, primary_key=True) cert_id = models.CharField(max_length=255, blank=True, null=True) cpe = models.CharField(max_length=255, blank=True, null=True) cve_string = models.CharField(max_length=255, blank=True, null=True) - cve = models.ForeignKey(Cve, related_name='vuln_scans', on_delete=models.CASCADE, blank=True, null=True) + cve = models.ForeignKey( + Cve, related_name="vuln_scans", on_delete=models.CASCADE, blank=True, null=True + ) cvss_base_score = models.CharField(max_length=255, blank=True, null=True) cvss_temporal_score = models.CharField(max_length=255, blank=True, null=True) cvss_temporal_vector = models.CharField(max_length=255, blank=True, null=True) @@ -857,11 +943,19 @@ class VulnScan(models.Model): exploit_available = models.CharField(max_length=255, blank=True, null=True) exploitability_ease = models.CharField(max_length=255, blank=True, null=True) ip_string = models.CharField(max_length=255, blank=True, null=True) - ip = models.ForeignKey("Ip", related_name='vuln_scans', on_delete=models.CASCADE, blank=True, null=True) + ip = models.ForeignKey( + "Ip", related_name="vuln_scans", on_delete=models.CASCADE, blank=True, null=True + ) latest = models.BooleanField(default=False) owner = models.CharField(max_length=255, blank=True, null=True) osvdb_id = models.CharField(max_length=255, blank=True, null=True) - organization = models.ForeignKey(Organization, related_name='vuln_scans', on_delete=models.CASCADE, blank=True, null=True) + organization = models.ForeignKey( + Organization, + related_name="vuln_scans", + on_delete=models.CASCADE, + blank=True, + null=True, + ) patch_publication_timestamp = models.DateTimeField(blank=True, null=True) cisa_known_exploited = models.DateTimeField(blank=True, null=True) port = models.IntegerField(blank=True, null=True) @@ -883,10 +977,14 @@ class VulnScan(models.Model): thorough_tests = models.BooleanField(default=False) cvss_score_rationale = models.CharField(max_length=255, blank=True, null=True) cvss_score_source = models.CharField(max_length=255, blank=True, null=True) - cvss3_base_score = models.DecimalField(max_digits=5, decimal_places=2, blank=True, null=True) + cvss3_base_score = models.DecimalField( + max_digits=5, decimal_places=2, blank=True, null=True + ) cvss3_vector = models.CharField(max_length=255, blank=True, null=True) cvss3_temporal_vector = models.CharField(max_length=255, blank=True, null=True) - cvss3_temporal_score = models.DecimalField(max_digits=5, decimal_places=2, blank=True, null=True) + cvss3_temporal_score = models.DecimalField( + max_digits=5, decimal_places=2, blank=True, null=True + ) asset_inventory = models.BooleanField(default=False) plugin_id = models.CharField(max_length=255, blank=True, null=True) plugin_modification_date = models.DateTimeField(blank=True, null=True) @@ -902,17 +1000,23 @@ class VulnScan(models.Model): # ticket_events = models.ManyToManyField(TicketEvent, related_name='vuln_scans') other_findings = models.JSONField(default=dict, blank=True) + class Meta: - """The Meta class for VulnScan.""" + """The Meta class for VulnScan.""" + + app_label = "dmz_mini_dl" + managed = manage_db + db_table = "vuln_scan" + - app_label = 'dmz_mini_dl' - managed = manage_db - db_table = "vuln_scan" - class Cidr(models.Model): + """The Cidr Model.""" + id = models.UUIDField(primary_key=True, editable=False) created_date = models.DateTimeField(auto_now_add=True) - network = InetAddressField(null=True, blank=True, unique=True) #models.TextField() # This field type is a guess. + network = InetAddressField( + null=True, blank=True, unique=True + ) # models.TextField() # This field type is a guess. start_ip = InetAddressField(null=True, blank=True) end_ip = InetAddressField(null=True, blank=True) retired = models.BooleanField(null=True, blank=True) @@ -929,20 +1033,22 @@ class Cidr(models.Model): null=True, ) - organizations = models.ManyToManyField(Organization, related_name='cidrs', blank=True) + organizations = models.ManyToManyField( + Organization, related_name="cidrs", blank=True + ) class Meta: + """The Meta class for Cidr.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cidr" - indexes = [ - models.Index(fields=['network']) - ] - + indexes = [models.Index(fields=["network"])] class Location(models.Model): + """The Location model.""" + id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) name = models.CharField(max_length=255, null=True, blank=True) country_abrv = models.CharField(max_length=255, null=True, blank=True) @@ -955,41 +1061,55 @@ class Location(models.Model): state = models.CharField(max_length=255, null=True, blank=True) class Meta: - app_label = 'dmz_mini_dl' + """The Meta class for Location.""" + + app_label = "dmz_mini_dl" managed = manage_db - db_table = 'location' - indexes = [ - models.Index(fields=['gnis_id']) - ] + db_table = "location" + indexes = [models.Index(fields=["gnis_id"])] + class Sector(models.Model): + """The Sector model.""" + id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) name = models.CharField(max_length=255, null=True, blank=True) acronym = models.CharField(max_length=255, null=True, blank=True, unique=True) retired = models.BooleanField(null=True, blank=True) - organizations = models.ManyToManyField(Organization, related_name='sectors', blank=True) + organizations = models.ManyToManyField( + Organization, related_name="sectors", blank=True + ) class Meta: - app_label = 'dmz_mini_dl' + """The Meta class for Sector.""" + + app_label = "dmz_mini_dl" managed = manage_db - db_table = 'sector' - indexes = [ - models.Index(fields=['acronym']) - ] + db_table = "sector" + indexes = [models.Index(fields=["acronym"])] + class Host(models.Model): + """The Host model.""" + id = models.CharField(max_length=255, primary_key=True) ip_string = models.CharField(max_length=255, null=True, blank=True) - ip = models.ForeignKey("Ip", related_name='hosts', on_delete=models.SET_NULL, null=True, blank=True) + ip = models.ForeignKey( + "Ip", related_name="hosts", on_delete=models.SET_NULL, null=True, blank=True + ) updated_timestamp = models.DateTimeField(null=True, blank=True) latest_netscan_1_timestamp = models.DateTimeField(null=True, blank=True) latest_netscan_2_timestamp = models.DateTimeField(null=True, blank=True) latest_vulnscan_timestamp = models.DateTimeField(null=True, blank=True) latest_portscan_timestamp = models.DateTimeField(null=True, blank=True) latest_scan_completion_timestamp = models.DateTimeField(null=True, blank=True) - location_longitude = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) - location_latitude = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) + location_longitude = models.DecimalField( + max_digits=10, decimal_places=6, null=True, blank=True + ) + location_latitude = models.DecimalField( + max_digits=10, decimal_places=6, null=True, blank=True + ) priority = models.IntegerField(null=True, blank=True) next_scan_timestamp = models.DateTimeField(null=True, blank=True) rand = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) @@ -997,20 +1117,33 @@ class Host(models.Model): host_live = models.BooleanField(null=True, blank=True) host_live_reason = models.CharField(max_length=255, null=True, blank=True) status = models.CharField(max_length=255, null=True, blank=True) - organization = models.ForeignKey(Organization, related_name='hosts', on_delete=models.CASCADE, null=True, blank=True) + organization = models.ForeignKey( + Organization, + related_name="hosts", + on_delete=models.CASCADE, + null=True, + blank=True, + ) class Meta: - app_label = 'dmz_mini_dl' + """The Meta class for Host.""" + + app_label = "dmz_mini_dl" managed = manage_db - db_table = 'host' + db_table = "host" indexes = [ - models.Index(fields=['ip_string']), + models.Index(fields=["ip_string"]), ] + class Ip(models.Model): + """The Ip model.""" + # id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) ip_hash = models.TextField(primary_key=True) - organization = models.ForeignKey(Organization, related_name='ips', on_delete=models.CASCADE) + organization = models.ForeignKey( + Organization, related_name="ips", on_delete=models.CASCADE + ) created_timestamp = models.DateTimeField(auto_now_add=True) updated_timestamp = models.DateTimeField(null=True, blank=True, auto_now=True) last_seen_timestamp = models.DateTimeField(null=True, blank=True) @@ -1020,15 +1153,15 @@ class Ip(models.Model): from_cidr = models.BooleanField(null=True, blank=True) retired = models.BooleanField(null=True, blank=True) last_reverse_lookup = models.DateTimeField(blank=True, null=True) - from_cidr = models.BooleanField(blank=True, null=True) - + from_cidr = models.BooleanField(blank=True, null=True) + # domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) # host_scans = models.ManyToManyField("HostScan", related_name='ips', blank=True) # hosts = models.ManyToManyField(Host, related_name='ips', blank=True) # tickets = models.ManyToManyField("Ticket", related_name='ips', blank=True) # vuln_scans = models.ManyToManyField(VulnScan, related_name='ips', blank=True) # port_scans = models.ManyToManyField("PortScan", related_name='ips', blank=True) - sub_domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) + sub_domains = models.ManyToManyField("SubDomains", related_name="ips", blank=True) has_shodan_results = models.BooleanField(blank=True, null=True) origin_cidr = models.ForeignKey( Cidr, on_delete=models.CASCADE, db_column="origin_cidr", blank=True, null=True @@ -1036,34 +1169,58 @@ class Ip(models.Model): current = models.BooleanField(blank=True, null=True) class Meta: - app_label = 'dmz_mini_dl' + """The Meta class for Ip.""" + + app_label = "dmz_mini_dl" managed = manage_db - db_table = 'ip' - indexes = [ - models.Index(fields=['ip', 'organization']) - ] - unique_together = ['ip', 'organization'] + db_table = "ip" + indexes = [models.Index(fields=["ip", "organization"])] + unique_together = ["ip", "organization"] class Ticket(models.Model): - id = models.CharField(max_length=255, primary_key=True) # Assuming the UUID is represented as a string + """The Ticket model.""" + + id = models.CharField( + max_length=255, primary_key=True + ) # Assuming the UUID is represented as a string cve_string = models.CharField(max_length=255, null=True, blank=True) - cve = models.ForeignKey(Cve, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) - cvss_base_score = models.DecimalField(max_digits=5, decimal_places=2, null=True, blank=True) + cve = models.ForeignKey( + Cve, related_name="tickets", null=True, blank=True, on_delete=models.CASCADE + ) + cvss_base_score = models.DecimalField( + max_digits=5, decimal_places=2, null=True, blank=True + ) cvss_version = models.CharField(max_length=255, null=True, blank=True) # kev = models.ForeignKey(Kev, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) vuln_name = models.CharField(max_length=255, null=True, blank=True) cvss_score_source = models.CharField(max_length=255, null=True, blank=True) - cvss_severity = models.DecimalField(max_digits=5, decimal_places=2, null=True, blank=True) - vpr_score = models.DecimalField(max_digits=5, decimal_places=2, null=True, blank=True) + cvss_severity = models.DecimalField( + max_digits=5, decimal_places=2, null=True, blank=True + ) + vpr_score = models.DecimalField( + max_digits=5, decimal_places=2, null=True, blank=True + ) false_positive = models.BooleanField(null=True, blank=True) ip_string = models.CharField(max_length=255, null=True, blank=True) - ip = models.ForeignKey(Ip, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + ip = models.ForeignKey( + Ip, related_name="tickets", null=True, blank=True, on_delete=models.CASCADE + ) updated_timestamp = models.DateTimeField(null=True, blank=True) - location_longitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True) - location_latitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True) + location_longitude = models.DecimalField( + max_digits=9, decimal_places=6, null=True, blank=True + ) + location_latitude = models.DecimalField( + max_digits=9, decimal_places=6, null=True, blank=True + ) found_in_latest_host_scan = models.BooleanField(null=True, blank=True) - organization = models.ForeignKey(Organization, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + organization = models.ForeignKey( + Organization, + related_name="tickets", + null=True, + blank=True, + on_delete=models.CASCADE, + ) vuln_port = models.IntegerField(null=True, blank=True) port_protocol = models.CharField(max_length=255, null=True, blank=True) snapshots_bool = models.BooleanField(null=True, blank=True) @@ -1075,15 +1232,24 @@ class Ticket(models.Model): # ticket_events = models.ManyToManyField(TicketEvent, related_name='tickets', blank=True) class Meta: - app_label = 'dmz_mini_dl' + """The Meta class for Ticket.""" + + app_label = "dmz_mini_dl" managed = manage_db - db_table = 'ticket' - unique_together = ['id'] + db_table = "ticket" + unique_together = ["id"] + class PortScan(models.Model): - id = models.CharField(max_length=255, primary_key=True) # Assuming UUIDs are stored as strings + """The PortScan model.""" + + id = models.CharField( + max_length=255, primary_key=True + ) # Assuming UUIDs are stored as strings ip_string = models.CharField(max_length=255, null=True, blank=True) - ip = models.ForeignKey(Ip, related_name='port_scans', null=True, blank=True, on_delete=models.CASCADE) + ip = models.ForeignKey( + Ip, related_name="port_scans", null=True, blank=True, on_delete=models.CASCADE + ) latest = models.BooleanField(default=False) port = models.IntegerField(null=True, blank=True) protocol = models.CharField(max_length=255, null=True, blank=True) @@ -1096,14 +1262,24 @@ class PortScan(models.Model): state = models.CharField(max_length=255, null=True, blank=True) time_scanned = models.DateTimeField(null=True, blank=True) # snapshots = models.ManyToManyField(Snapshot, related_name='port_scans', blank=True) - organization = models.ForeignKey(Organization, related_name='port_scans', null=True, blank=True, on_delete=models.CASCADE) - + organization = models.ForeignKey( + Organization, + related_name="port_scans", + null=True, + blank=True, + on_delete=models.CASCADE, + ) + class Meta: + """The Meta class for PortScan.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db - db_table = 'port_scan' -######### WAS Models ######### + db_table = "port_scan" + + +# ####### WAS Models ######### + class WasTrackerCustomerdata(models.Model): """Define WasTrackerCustomerdata model.""" @@ -1135,35 +1311,38 @@ class WasTrackerCustomerdata(models.Model): class Meta: """Set WasTrackerCustomerdata model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "was_tracker_customer_data" + """ -- WARNING: It may differ from actual native database DDL CREATE TABLE information_schema.was_findings ( - finding_uid uuid NOT NULL, - finding_type varchar(10485760) NULL, - webapp_id int4 NULL, - was_org_id text NULL, - owasp_category varchar(10485760) NULL, - severity varchar(10485760) NULL, - times_detected int4 NULL, - base_score float8 NULL, - temporal_score float8 NULL, - fstatus varchar(10485760) NULL, - last_detected date NULL, - first_detected date NULL, - is_remediated bool NULL, - potential bool NULL, - webapp_url text NULL, - webapp_name text NULL, - "name" text NULL, - cvss_v3_attack_vector text NULL, - cwe_list _int4 NULL, - wasc_list jsonb NULL + finding_uid uuid NOT NULL, + finding_type varchar(10485760) NULL, + webapp_id int4 NULL, + was_org_id text NULL, + owasp_category varchar(10485760) NULL, + severity varchar(10485760) NULL, + times_detected int4 NULL, + base_score float8 NULL, + temporal_score float8 NULL, + fstatus varchar(10485760) NULL, + last_detected date NULL, + first_detected date NULL, + is_remediated bool NULL, + potential bool NULL, + webapp_url text NULL, + webapp_name text NULL, + "name" text NULL, + cvss_v3_attack_vector text NULL, + cwe_list _int4 NULL, + wasc_list jsonb NULL ); """ + + class WasFindings(models.Model): """Define WasFindings model.""" @@ -1195,15 +1374,18 @@ class WasFindings(models.Model): url = models.TextField(blank=True, null=True) qid = models.IntegerField(blank=True, null=True) response = models.TextField(blank=True, null=True) + class Meta: """Set WasFindings model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "was_findings" -class WasHistory(models.Model): + +class WasHistory(models.Model): """Define WasHistory model.""" + was_org_id = models.TextField(blank=True, null=True) date_scanned = models.DateField() vuln_cnt = models.IntegerField(blank=True, null=True) @@ -1217,28 +1399,35 @@ class WasHistory(models.Model): high_rem_cnt = models.IntegerField(blank=True, null=True) crit_rem_cnt = models.IntegerField(blank=True, null=True) total_potential = models.IntegerField(blank=True, null=True) + class Meta: """Set WasHistory model metadata.""" - app_label = 'dmz_mini_dl' + + app_label = "dmz_mini_dl" managed = manage_db db_table = "was_history" - unique_together = (('was_org_id', 'date_scanned'),) - + unique_together = (("was_org_id", "date_scanned"),) + class WasMap(models.Model): """Define WasMap model.""" + was_org_id = models.TextField(blank=True, primary_key=True) pe_org_id = models.UUIDField(blank=True, null=True) report_on = models.BooleanField(blank=True, null=True) last_scanned = models.DateField(blank=True, null=True) + class Meta: """Set WasMap model metadata.""" - app_label = 'dmz_mini_dl' + + app_label = "dmz_mini_dl" managed = manage_db db_table = "was_map" class WasReport(models.Model): + """The WasReport model.""" + org_name = models.TextField(blank=True, null=True) date_pulled = models.DateTimeField(blank=True, null=True) last_scan_date = models.DateTimeField(blank=True, null=True) @@ -1289,13 +1478,15 @@ class WasReport(models.Model): pdf_obj = models.BinaryField(blank=True, null=True) class Meta: - db_table = 'was_report' - unique_together = ('last_scan_date', 'org_was_acronym') - app_label = 'dmz_mini_dl' + """The Meta class for WasReport.""" + + db_table = "was_report" + unique_together = ("last_scan_date", "org_was_acronym") + app_label = "dmz_mini_dl" managed = manage_db -######### PE Models ######### +# ######## PE Models ######### class PeUsers(models.Model): """Define Users model.""" @@ -1310,10 +1501,11 @@ class PeUsers(models.Model): class Meta: """Set User model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "pe_users" + # ?????? not sure if we use this anywhere class AlembicVersion(models.Model): """Define AlembicVersion model.""" @@ -1323,10 +1515,11 @@ class AlembicVersion(models.Model): class Meta: """Set AlembicVersion model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "alembic_version" + class SixgillAlerts(models.Model): """Define Alerts model.""" @@ -1357,7 +1550,7 @@ class SixgillAlerts(models.Model): class Meta: """Set Alerts model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "sixgill_alerts" @@ -1374,10 +1567,11 @@ class Alias(models.Model): class Meta: """Set Alias model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "alias" + # ?????? class AssetHeaders(models.Model): """Define AssetHeaders model.""" @@ -1400,11 +1594,12 @@ class AssetHeaders(models.Model): class Meta: """Set AssetHeaders model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "asset_headers" unique_together = (("organization", "sub_url"),) + # # ?????? no data currently # class AuthGroup(models.Model): # """Define AuthGroup model.""" @@ -1536,6 +1731,7 @@ class Meta: # db_table = "cidrs" # unique_together = (("organization_uid", "network"),) + class CredentialBreaches(models.Model): """Define CredentialBreaches model.""" @@ -1548,7 +1744,7 @@ class CredentialBreaches(models.Model): modified_date = models.DateTimeField(blank=True, null=True) data_classes = ArrayField( models.TextField(blank=True, null=True), blank=True, null=True - ) # This field type is a guess. + ) # This field type is a guess. password_included = models.BooleanField(blank=True, null=True) is_verified = models.BooleanField(blank=True, null=True) is_fabricated = models.BooleanField(blank=True, null=True) @@ -1556,13 +1752,17 @@ class CredentialBreaches(models.Model): is_retired = models.BooleanField(blank=True, null=True) is_spam_list = models.BooleanField(blank=True, null=True) data_source = models.ForeignKey( - "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, ) class Meta: """Set CredentialBreaches model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "credential_breaches" @@ -1572,7 +1772,7 @@ class CredentialExposures(models.Model): credential_exposures_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) email = models.TextField() - organization_uid = models.ForeignKey( + organization = models.ForeignKey( "Organization", on_delete=models.CASCADE, db_column="organization_uid" ) root_domain = models.TextField(blank=True, null=True) @@ -1585,7 +1785,11 @@ class CredentialExposures(models.Model): db_column="credential_breaches_uid", ) data_source = models.ForeignKey( - "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, ) name = models.TextField(blank=True, null=True) login_id = models.TextField(blank=True, null=True) @@ -1597,11 +1801,12 @@ class CredentialExposures(models.Model): class Meta: """Set CredentialExposures model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "credential_exposures" unique_together = (("breach_name", "email"),) + # needs merging # class CveInfo(models.Model): # """Define CveInfo model.""" @@ -1628,6 +1833,7 @@ class Meta: # managed = False # db_table = "cve_info" + class CyhyContacts(models.Model): """Define CyhyContacts model.""" @@ -1648,11 +1854,12 @@ class CyhyContacts(models.Model): class Meta: """Set CyhyContacts model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cyhy_contacts" unique_together = (("org_id", "contact_type", "email", "name"),) + class CyhyDbAssets(models.Model): """Define CyhyDbAssets model.""" @@ -1674,11 +1881,12 @@ class CyhyDbAssets(models.Model): class Meta: """Set CyhyDbAssets model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cyhy_db_assets" unique_together = (("org_id", "network"),) + # Probably included in VS models # class CyhyPortScans(models.Model): # """Define CyhyPortScans model.""" @@ -1736,10 +1944,11 @@ class DataSource(models.Model): class Meta: """Set DataSource model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "data_source" + # # ?????? # class DjangoAdminLog(models.Model): # """Define DjangoAdminLog model.""" @@ -1807,6 +2016,7 @@ class Meta: # managed = manage_db # db_table = "django_session" + class DnsRecords(models.Model): """Define DnsRecords model.""" @@ -1893,10 +2103,11 @@ class DnsRecords(models.Model): class Meta: """Set DnsRecords model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "dns_records" + # Possibly shodan class DomainAlerts(models.Model): """Define DomainAlerts model.""" @@ -1918,11 +2129,12 @@ class DomainAlerts(models.Model): class Meta: """Set DomainAlerts model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "domain_alerts" unique_together = (("alert_type", "sub_domain", "date", "new_value"),) + class DomainPermutations(models.Model): """Define DomainPermutations model.""" @@ -1958,11 +2170,12 @@ class DomainPermutations(models.Model): class Meta: """Set DomainPermutations model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "domain_permutations" unique_together = (("domain_permutation", "organization"),) + class DotgovDomains(models.Model): """Define DotgovDomains model.""" @@ -1978,10 +2191,11 @@ class DotgovDomains(models.Model): class Meta: """Set DotgovDomains model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "dotgov_domains" + class Executives(models.Model): """Define Executives model.""" @@ -1994,10 +2208,11 @@ class Executives(models.Model): class Meta: """Set Executives model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "executives" + # merged with vs's IP table # class Ips(models.Model): # """Define Ips model.""" @@ -2040,6 +2255,7 @@ class Meta: # # db_table = 'ips_subs' # unique_together = (("ip_hash", "sub_domain_uid"),) + class Mentions(models.Model): """Define Mentions model.""" @@ -2072,10 +2288,11 @@ class Mentions(models.Model): class Meta: """Set Mentions model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "mentions" + # Likely can be removed class OrgIdMap(models.Model): """Define OrgIdMap model.""" @@ -2087,7 +2304,7 @@ class OrgIdMap(models.Model): class Meta: """Set OrgIdMap model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "org_id_map" unique_together = (("cyhy_id", "pe_org_id"),) @@ -2102,7 +2319,7 @@ class OrgType(models.Model): class Meta: """Set OrgType model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "org_type" @@ -2249,13 +2466,12 @@ class PshttResults(models.Model): class Meta: """Set PshttResults model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "pshtt_results" unique_together = (("organization", "sub_domain"),) - class PeReportSummaryStats(models.Model): """Define ReportSummaryStats model.""" @@ -2296,11 +2512,12 @@ class PeReportSummaryStats(models.Model): class Meta: """Set ReportSummaryStats model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "pe_report_summary_stats" unique_together = (("organization", "start_date"),) + class RootDomains(models.Model): """Define RootDomains model.""" @@ -2318,7 +2535,7 @@ class RootDomains(models.Model): class Meta: """Set RootDomains model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "root_domains" unique_together = (("root_domain", "organization"),) @@ -2339,19 +2556,26 @@ class PeTeamMembers(models.Model): class Meta: """Set TeamMembers model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "pe_team_members" + class ShodanAssets(models.Model): """Define ShodanAssets model.""" shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid", blank=True, null=True + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + blank=True, + null=True, ) # If you still need to store the organization name or acronym, use a separate field for that - organization_name = models.TextField(blank=True, null=True) # New field to store the name or acronym + organization_name = models.TextField( + blank=True, null=True + ) # New field to store the name or acronym ip = models.TextField(blank=True, null=True) port = models.IntegerField(blank=True, null=True) protocol = models.TextField(blank=True, null=True) @@ -2359,12 +2583,20 @@ class ShodanAssets(models.Model): product = models.TextField(blank=True, null=True) server = models.TextField(blank=True, null=True) tags = models.JSONField(blank=True, null=True) # Store tags as a list (JSON format) - domains = models.JSONField(blank=True, null=True) # Store domains as a list (JSON format) - hostnames = models.JSONField(blank=True, null=True) # Store hostnames as a list (JSON format) + domains = models.JSONField( + blank=True, null=True + ) # Store domains as a list (JSON format) + hostnames = models.JSONField( + blank=True, null=True + ) # Store hostnames as a list (JSON format) isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid", blank=True, null=True + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, ) country_code = models.TextField(blank=True, null=True) location = models.TextField(blank=True, null=True) @@ -2372,12 +2604,11 @@ class ShodanAssets(models.Model): class Meta: """Set ShodanAssets model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "shodan_assets" - unique_together = ( - ("organization", "ip", "port", "protocol", "timestamp"), - ) + unique_together = (("organization", "ip", "port", "protocol", "timestamp"),) + # class ShodanInsecureProtocolsUnverifiedVulns(models.Model): # """Define ShodanInsecureProtocolsUnverifiedVulns model.""" @@ -2455,7 +2686,7 @@ class ShodanVulns(models.Model): ) # This field type is a guess. hostnames = ArrayField( models.TextField(blank=True, null=True), blank=True, null=True - ) # This field type is a guess. + ) # This field type is a guess. isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source = models.ForeignKey( @@ -2472,25 +2703,22 @@ class ShodanVulns(models.Model): banner = models.TextField(blank=True, null=True) version = models.TextField(blank=True, null=True) mitigation = models.TextField(blank=True, null=True) - cpe = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True - ) + cpe = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) class Meta: """Set ShodanVulns model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "shodan_vulns" - unique_together = ( - ("organization", "ip", "port", "protocol", "timestamp"), - ) + unique_together = (("organization", "ip", "port", "protocol", "timestamp"),) + class SubDomains(models.Model): """Define SubDomains model.""" sub_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) - sub_domain = models.TextField()# Crossfeed Domains name field + sub_domain = models.TextField() # Crossfeed Domains name field root_domain = models.ForeignKey( RootDomains, on_delete=models.CASCADE, db_column="root_domain_uid" ) @@ -2511,34 +2739,48 @@ class SubDomains(models.Model): updated_at = models.DateTimeField(db_column="updated_at") current = models.BooleanField(blank=True, null=True) identified = models.BooleanField(blank=True, null=True) - ip_address = models.TextField(blank=True, null=True)# XFD column - synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True)# XFD column - from_root_domain = models.TextField(db_column="from_root_domain", blank=True, null=True)# XFD column + ip_address = models.TextField(blank=True, null=True) # XFD column + synced_at = models.DateTimeField( + db_column="synced_at", blank=True, null=True + ) # XFD column + from_root_domain = models.TextField( + db_column="from_root_domain", blank=True, null=True + ) # XFD column subdomain_source = models.TextField( db_column="subdomain_source", max_length=255, blank=True, null=True - )# XFD column + ) # XFD column organization = models.ForeignKey( Organization, on_delete=models.CASCADE, db_column="organization_uid" ) - ip_only = models.BooleanField(db_column="ip_only", default=False)# XFD column - reverse_name = models.CharField(db_column="reverse_name", max_length=512)# XFD column - screenshot = models.CharField(max_length=512, blank=True, null=True)# XFD Crossfeed Domains screenshot field - country = models.CharField(max_length=255, blank=True, null=True)# XFD column - asn = models.CharField(max_length=255, blank=True, null=True)# XFD column - cloud_hosted = models.BooleanField(db_column="cloud_hosted", default=False)# XFD column - ssl = models.JSONField(blank=True, null=True)# XFD columnv + ip_only = models.BooleanField(db_column="ip_only", default=False) # XFD column + reverse_name = models.CharField( + db_column="reverse_name", max_length=512 + ) # XFD column + screenshot = models.CharField( + max_length=512, blank=True, null=True + ) # XFD Crossfeed Domains screenshot field + country = models.CharField(max_length=255, blank=True, null=True) # XFD column + asn = models.CharField(max_length=255, blank=True, null=True) # XFD column + cloud_hosted = models.BooleanField( + db_column="cloud_hosted", default=False + ) # XFD column + ssl = models.JSONField(blank=True, null=True) # XFD columnv censys_certificates_results = models.JSONField( db_column="censys_certificates_results", default=dict - )# XFD column - trustymail_results = models.JSONField(db_column="trustymail_results", default=dict)# XFD column + ) # XFD column + trustymail_results = models.JSONField( + db_column="trustymail_results", default=dict + ) # XFD column + class Meta: """Set SubDomains model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "sub_domains" unique_together = (("sub_domain", "root_domain"),) + class TopCves(models.Model): """Define TopCves model.""" @@ -2555,11 +2797,12 @@ class TopCves(models.Model): class Meta: """Set TopCves model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "top_cves" unique_together = (("cve_id", "date"),) + # Not sure if this is still used class TopicTotals(models.Model): """Define TopicTotals model.""" @@ -2572,10 +2815,11 @@ class TopicTotals(models.Model): class Meta: """Set TopicTotals model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "topic_totals" + # Not sure if this is still used class UniqueSoftware(models.Model): """Define UniqueSoftware model.""" @@ -2588,10 +2832,11 @@ class UniqueSoftware(models.Model): class Meta: """Set UniqueSoftware model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "unique_software" + class WebAssets(models.Model): """Define WebAssets model.""" @@ -2614,11 +2859,12 @@ class WebAssets(models.Model): class Meta: """Set WebAssets model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "web_assets" unique_together = (("asset", "organization"),) + class WeeklyStatusesMdl(models.Model): """Define WeeklyStatusesMdl model.""" @@ -2640,7 +2886,7 @@ class Meta: # unique_together = (('week_ending', 'user_status'),) - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "weekly_statuses_mdl" @@ -2657,7 +2903,7 @@ class CyhyKevs(models.Model): class Meta: """Set CyhyKevs model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cyhy_kevs" @@ -2668,9 +2914,12 @@ class XpanseBusinessUnits(models.Model): xpanse_business_unit_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) entity_name = models.TextField(unique=True, blank=True, null=True) cyhy_db_name = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="cyhy_db_name", to_field="acronym", + "Organization", + on_delete=models.CASCADE, + db_column="cyhy_db_name", + to_field="acronym", null=True, # Allow NULL values - blank=True + blank=True, ) state = models.TextField(blank=True, null=True) county = models.TextField(blank=True, null=True) @@ -2683,7 +2932,7 @@ class XpanseBusinessUnits(models.Model): class Meta: """Set XpanseBusinessUnits metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "xpanse_business_units" @@ -2736,7 +2985,7 @@ class XpanseAssetsMdl(models.Model): class Meta: """Set XpanseAssetsMdl metdata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "xpanse_assets_mdl" @@ -2758,7 +3007,7 @@ class XpanseCvesMdl(models.Model): class Meta: """Set XpanseCvesMdl metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "xpanse_cves_mdl" @@ -2803,7 +3052,7 @@ class XpanseServicesMdl(models.Model): class Meta: """Set XpanseServicesMdl metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "xpanse_services_mdl" @@ -2825,7 +3074,7 @@ class XpanseCveServiceMdl(models.Model): class Meta: """Set XpanseCveServiceMdl metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "xpanse_cve_services_mdl" unique_together = (("xpanse_inferred_cve", "xpanse_service"),) @@ -2934,7 +3183,7 @@ class XpanseAlerts(models.Model): class Meta: """Set XpanseAlerts model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "xpanse_alerts_mdl" @@ -2948,7 +3197,7 @@ class CpeVender(models.Model): class Meta: """Set CpeVender model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cpe_vender" @@ -2969,13 +3218,13 @@ class CpeProduct(models.Model): class Meta: """Set CpeProduct model metadata.""" - app_label = 'dmz_mini_dl' + app_label = "dmz_mini_dl" managed = manage_db db_table = "cpe_product_mdl" unique_together = (("cpe_product_name", "version_number"),) -# # THese are all views, so they shouldn't be generated via the ORM +# # THese are all views, so they shouldn't be generated via the ORM # # This should be a view not a table # class VwPshttDomainsToRun(models.Model): @@ -3735,4 +3984,3 @@ class Meta: # managed = False # db_table = "vw_iscore_orgs_ip_counts"""" Django ORM models """ - diff --git a/src/pe_reports/pe_reports_django_project/home/models.py b/src/pe_reports/pe_reports_django_project/home/models.py index e97094fb..5ef93222 100644 --- a/src/pe_reports/pe_reports_django_project/home/models.py +++ b/src/pe_reports/pe_reports_django_project/home/models.py @@ -731,7 +731,7 @@ class Meta: """Set IpsSubs model metadata.""" managed = False - # db_table = 'ips_subs' + db_table = "ips_subs" unique_together = (("ip_hash", "sub_domain_uid"),) @@ -1049,7 +1049,7 @@ class Meta: class ShodanAssets(models.Model): """Define ShodanAssets model.""" - shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organizations_uid = models.ForeignKey( Organizations, on_delete=models.CASCADE, db_column="organizations_uid" ) @@ -1060,9 +1060,11 @@ class ShodanAssets(models.Model): timestamp = models.DateTimeField(blank=True, null=True) product = models.TextField(blank=True, null=True) server = models.TextField(blank=True, null=True) - tags = models.TextField(blank=True, null=True) # This field type is a guess. - domains = models.TextField(blank=True, null=True) # This field type is a guess. - hostnames = models.TextField(blank=True, null=True) # This field type is a guess. + tags = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + domains = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + hostnames = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source_uid = models.ForeignKey( @@ -1084,7 +1086,7 @@ class Meta: class ShodanInsecureProtocolsUnverifiedVulns(models.Model): """Define ShodanInsecureProtocolsUnverifiedVulns model.""" - insecure_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + insecure_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organizations_uid = models.ForeignKey( Organizations, on_delete=models.CASCADE, db_column="organizations_uid" ) @@ -1123,7 +1125,7 @@ class Meta: class ShodanVulns(models.Model): """Define ShodanVulns model.""" - shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organizations_uid = models.ForeignKey( Organizations, on_delete=models.CASCADE, db_column="organizations_uid" ) @@ -1149,9 +1151,11 @@ class ShodanVulns(models.Model): ii_description = models.TextField(blank=True, null=True) availability_impact = models.TextField(blank=True, null=True) ai_description = models.TextField(blank=True, null=True) - tags = models.TextField(blank=True, null=True) # This field type is a guess. - domains = models.TextField(blank=True, null=True) # This field type is a guess. - hostnames = models.TextField(blank=True, null=True) # This field type is a guess. + tags = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + domains = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + hostnames = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source_uid = models.ForeignKey( @@ -1159,18 +1163,16 @@ class ShodanVulns(models.Model): ) type = models.TextField(blank=True, null=True) name = models.TextField(blank=True, null=True) - potential_vulns = models.TextField( - blank=True, null=True - ) # This field type is a guess. + potential_vulns = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) mitigation = models.TextField(blank=True, null=True) server = models.TextField(blank=True, null=True) is_verified = models.BooleanField(blank=True, null=True) banner = models.TextField(blank=True, null=True) version = models.TextField(blank=True, null=True) mitigation = models.TextField(blank=True, null=True) - cpe = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True - ) + cpe = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) class Meta: """Set ShodanVulns model metadata.""" @@ -1447,9 +1449,7 @@ class VwShodanvulnsVerified(models.Model): data_source = models.TextField(blank=True, null=True) banner = models.TextField(blank=True, null=True) version = models.TextField(blank=True, null=True) - cpe = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True - ) + cpe = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) class Meta: """Set VwShodanvulnsVerified model metadata.""" @@ -2116,6 +2116,12 @@ class XpanseBusinessUnits(models.Model): xpanse_business_unit_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) entity_name = models.TextField(unique=True, blank=True, null=True) + cyhy_db_name = models.ForeignKey( + "Organizations", + on_delete=models.CASCADE, + db_column="cyhy_db_name", + to_field="cyhy_db_name", + ) state = models.TextField(blank=True, null=True) county = models.TextField(blank=True, null=True) city = models.TextField(blank=True, null=True) @@ -2464,28 +2470,30 @@ class Meta: """ -- WARNING: It may differ from actual native database DDL CREATE TABLE information_schema.was_findings ( - finding_uid uuid NOT NULL, - finding_type varchar(10485760) NULL, - webapp_id int4 NULL, - was_org_id text NULL, - owasp_category varchar(10485760) NULL, - severity varchar(10485760) NULL, - times_detected int4 NULL, - base_score float8 NULL, - temporal_score float8 NULL, - fstatus varchar(10485760) NULL, - last_detected date NULL, - first_detected date NULL, - is_remidiated bool NULL, - potential bool NULL, - webapp_url text NULL, - webapp_name text NULL, - "name" text NULL, - cvss_v3_attack_vector text NULL, - cwe_list _int4 NULL, - wasc_list jsonb NULL + finding_uid uuid NOT NULL, + finding_type varchar(10485760) NULL, + webapp_id int4 NULL, + was_org_id text NULL, + owasp_category varchar(10485760) NULL, + severity varchar(10485760) NULL, + times_detected int4 NULL, + base_score float8 NULL, + temporal_score float8 NULL, + fstatus varchar(10485760) NULL, + last_detected date NULL, + first_detected date NULL, + is_remidiated bool NULL, + potential bool NULL, + webapp_url text NULL, + webapp_name text NULL, + "name" text NULL, + cvss_v3_attack_vector text NULL, + cwe_list _int4 NULL, + wasc_list jsonb NULL ); """ + + class WasFindings(models.Model): """Define WasFindings model.""" @@ -2511,7 +2519,9 @@ class WasFindings(models.Model): models.IntegerField(blank=True, null=True), blank=True, null=True ) wasc_list = models.JSONField(blank=True, null=True) + class Meta: """Set WasFindings model metadata.""" + managed = False db_table = "was_findings" diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt old mode 100755 new mode 100644 index 26f68740..55ebd005 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt @@ -1,21 +1,23 @@ +crispy-bootstrap5 Django -uvicorn -python-decouple==3.8 django-celery-beat django-crispy-forms -crispy-bootstrap5 -whitenoise -elastic-apm -psycopg2-binary django-netfields -fastapi --e git+https://github.com/cisagov/ATC-Framework.git@CD-add-CODEOWNERS#egg=pe_reports +docxtpl +elastic-apm +email_validator en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.5.0/en_core_web_lg-3.5.0-py3-none-any.whl#sha256=c8ac64840c1eb3e3ca7bd38bd1e1c48fb0faeb2449d54d01d5ce629af4595775 -retry -email_validator +fastapi fastapi-limiter +flower +numpy==1.23.5 +-e git+https://github.com/cisagov/ATC-Framework.git@CD-add-CODEOWNERS#egg=pe_reports +psycopg2-binary +python-decouple==3.8 python-jose -slowapi python-multipart -docxtpl -numpy==1.23.5 \ No newline at end of file +rabbitmq +retry +slowapi +uvicorn +whitenoise diff --git a/src/pe_source/data/pe_db/db_query_source.py b/src/pe_source/data/pe_db/db_query_source.py index 413011ae..275b7345 100644 --- a/src/pe_source/data/pe_db/db_query_source.py +++ b/src/pe_source/data/pe_db/db_query_source.py @@ -17,7 +17,7 @@ import requests # cisagov Libraries -from pe_reports import app +# from pe_reports import app from pe_reports.data.config import config, staging_config from pe_reports.data.db_query import task_api_call @@ -610,6 +610,7 @@ def insert_or_update_business_unit(business_unit_dict): except json.decoder.JSONDecodeError as err: LOGGER.error(err) + # --- Issue 699 pe-reports --- def get_linked_xpanse_business_units(): """ @@ -639,7 +640,7 @@ def get_linked_xpanse_business_units(): except json.decoder.JSONDecodeError as err: LOGGER.error(err) - + # --- Issue 682 --- def api_xpanse_alert_insert(xpanse_alert_dict): """ @@ -1005,9 +1006,13 @@ def get_orgs(): # Process data and return for row in result: if row.get("date_first_reported") is not None: - row["date_first_reported"] = datetime.strptime(row.get("date_first_reported"), "%Y-%m-%d") + row["date_first_reported"] = datetime.strptime( + row.get("date_first_reported"), "%Y-%m-%d" + ) if row.get("cyhy_period_start") is not None: - row["cyhy_period_start"] = datetime.strptime(row.get("cyhy_period_start"), "%Y-%m-%d") + row["cyhy_period_start"] = datetime.strptime( + row.get("cyhy_period_start"), "%Y-%m-%d" + ) if row.get("county_fips") is not None: row["county_fips"] = Decimal(row.get("county_fips")) if row.get("state_fips") is not None: @@ -1148,10 +1153,10 @@ def org_root_domains(org_uid): result = requests.post(endpoint_url, headers=headers, data=data).json() # Process data and return result_df = pd.DataFrame.from_dict(result) - result_df.rename(columns={ - "root_domain_uid": "root_uid", - "organizations_uid": "org_uid" - }, inplace=True) + result_df.rename( + columns={"root_domain_uid": "root_uid", "organizations_uid": "org_uid"}, + inplace=True, + ) result_dict_list = result_df.to_dict("records") return result_dict_list except requests.exceptions.HTTPError as errh: @@ -1443,7 +1448,7 @@ def query_PE_subs(org_uid): # --- Issue 016 atc-framework --- -def insert_shodan_assets(asset_data): +def insert_shodan_assets(asset_data, failed): """ Query API to insert Shodan data into the shodan_assets table. @@ -1464,18 +1469,24 @@ def insert_shodan_assets(asset_data): LOGGER.info(result) except requests.exceptions.HTTPError as errh: LOGGER.error(errh) + failed.append("Failed inserting shodan assets: {}".format(errh)) except requests.exceptions.ConnectionError as errc: LOGGER.error(errc) + failed.append("Failed inserting shodan assets: {}".format(errc)) except requests.exceptions.Timeout as errt: LOGGER.error(errt) + failed.append("Failed inserting shodan assets: {}".format(errt)) except requests.exceptions.RequestException as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) except json.decoder.JSONDecodeError as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) + return failed # --- Issue 017 atc-framework --- -def insert_shodan_vulns(vuln_data): +def insert_shodan_vulns(vuln_data, failed): """ Query API to insert Shodan data into the shodan_vulns table. @@ -1483,7 +1494,7 @@ def insert_shodan_vulns(vuln_data): data: Dataframe of the shodan data to be inserted into shodan_vulns. """ # Endpoint info - endpoint_url = pe_api_url + "shodan_vulns_inserts" + endpoint_url = pe_api_url + "shodan_vulns_insert" headers = { "Content-Type": "application/json", "access_token": pe_api_key, @@ -1496,21 +1507,27 @@ def insert_shodan_vulns(vuln_data): LOGGER.info(result) except requests.exceptions.HTTPError as errh: LOGGER.error(errh) + failed.append("Failed inserting shodan assets: {}".format(errh)) except requests.exceptions.ConnectionError as errc: LOGGER.error(errc) + failed.append("Failed inserting shodan assets: {}".format(errc)) except requests.exceptions.Timeout as errt: LOGGER.error(errt) + failed.append("Failed inserting shodan assets: {}".format(errt)) except requests.exceptions.RequestException as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) except json.decoder.JSONDecodeError as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) + return failed # v ===== ACTIVE TSQL THAT STILL NEEDS CONVERSION ===== v # Conversion in progress -def get_ips(org_uid): +def get_ips_old(org_uid): """Get IP data.""" conn = connect() sql1 = """SELECT i.ip_hash, i.ip, ct.network FROM ips i @@ -1547,6 +1564,36 @@ def get_ips(org_uid): return ips +def get_ips(org_uid): + """ + Query API to get all ips for an org to run through Shodan. + + Return: + All ips to run through Shodan + """ + # Endpoint info + endpoint_url = pe_api_url + "query_shodan_ips/" + org_uid + headers = { + "Content-Type": "application/json", + "access_token": pe_api_key, + } + try: + result = requests.get(endpoint_url, headers=headers).json() + # Process data and return + print(result) + return result + except requests.exceptions.HTTPError as errh: + LOGGER.error(errh) + except requests.exceptions.ConnectionError as errc: + LOGGER.error(errc) + except requests.exceptions.Timeout as errt: + LOGGER.error(errt) + except requests.exceptions.RequestException as err: + LOGGER.error(err) + except json.decoder.JSONDecodeError as err: + LOGGER.error(err) + + # ??? def query_orgs_rev(): """Query orgs in reverse.""" @@ -1930,12 +1977,13 @@ def insert_intelx_credentials_tsql(df): conn.rollback() cursor.close() + # WAS queries def getPotential(org_id): """Get findings for specific time period in months.""" conn = connect() cur = conn.cursor() - sql = """ SELECT COUNT(*) FROM was_findings + sql = """ SELECT COUNT(*) FROM was_findings WHERE was_org_id = '{}' AND potential IS TRUE; """ cur.execute(sql.format(org_id), conn) @@ -1956,7 +2004,7 @@ def insertWASIds(listIds): ON CONFLICT (was_org_id) DO NOTHING;""" cur = conn.cursor() for id in listIds: - if id[1] == '': + if id[1] == "": cur.execute(sqlNoUUID.format(id[0])) else: cur.execute(sql.format(id[0], id[1])) @@ -1964,26 +2012,28 @@ def insertWASIds(listIds): close(conn) print("Success adding WAS IDs to database.") -def getPreviousFindingsHistorical(org_id,monthsAgo): + +def getPreviousFindingsHistorical(org_id, monthsAgo): """Get findings for specific time period in months.""" conn = connect() cur = conn.cursor() - sql = """ SELECT * FROM was_findings + sql = """ SELECT * FROM was_findings WHERE was_org_id = '{}'; """ - cur.execute(sql.format(org_id,monthsAgo-1,monthsAgo), conn) + cur.execute(sql.format(org_id, monthsAgo - 1, monthsAgo), conn) ret = cur.fetchall() cur.close() close(conn) return ret + def insertFindingData(findingList): """Insert finding data into database.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available conn = connect() sql = """INSERT INTO was_findings (finding_uid, finding_type, webapp_id, webapp_url, webapp_name, was_org_id, name, owasp_category, severity, times_detected, cvss_v3_attack_vector, base_score, temporal_score, fstatus, last_detected, first_detected, potential, cwe_list, wasc_list) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}') - ON CONFLICT (finding_uid) DO UPDATE + ON CONFLICT (finding_uid) DO UPDATE SET is_remidiated = CASE WHEN was_findings.fstatus != 'FIXED' AND excluded.fstatus = 'FIXED' THEN TRUE ELSE was_findings.is_remidiated @@ -2001,7 +2051,7 @@ def insertFindingData(findingList): ;""" cur = conn.cursor() for finding in findingList: - finding['cwe_list'] = '{' + ','.join(map(str,finding["cwe_list"])) + '}' + finding["cwe_list"] = "{" + ",".join(map(str, finding["cwe_list"])) + "}" try: cur.execute( sql.format( @@ -2023,7 +2073,7 @@ def insertFindingData(findingList): finding["first_detected"], finding["potential"], finding["cwe_list"], - json.dumps(finding["wasc_list"]) + json.dumps(finding["wasc_list"]), ) ) except KeyError: @@ -2033,6 +2083,7 @@ def insertFindingData(findingList): close(conn) print("Success adding finding data to database.") + def queryVulnWebAppCount(org_id): """Query the amount of webapps with vulnerabilities.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available @@ -2051,6 +2102,7 @@ def queryVulnWebAppCount(org_id): close(conn) return len(set(webIdsList)) + def queryWASOrgList(): """Query the list of WAS orgs.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available @@ -2063,7 +2115,7 @@ def queryWASOrgList(): def getPEuuid(org_id): - """Query the org uuid given a certain cyhy db name""" + """Query the org uuid given a certain cyhy db name.""" conn = connect() sql = """SELECT organizations_uid FROM organizations WHERE cyhy_db_name = '{}'""" cur = conn.cursor() @@ -2072,21 +2124,23 @@ def getPEuuid(org_id): close(conn) return ret -def getPreviousFindings(org_id,monthsAgo): + +def getPreviousFindings(org_id, monthsAgo): """Get findings for specific time period in months.""" conn = connect() cur = conn.cursor() - sql = """ SELECT * FROM was_findings + sql = """ SELECT * FROM was_findings WHERE was_org_id = '{}' AND last_detected >= date_trunc('month', now() - interval '{} month') AND last_detected < date_trunc('month', now() - interval '{} month'); """ - cur.execute(sql.format(org_id,monthsAgo,monthsAgo-1), conn) + cur.execute(sql.format(org_id, monthsAgo, monthsAgo - 1), conn) ret = cur.fetchall() cur.close() close(conn) return ret + def queryVulnCountAll(org_id): """Query the amount of webapps with vulnerabilities.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available @@ -2105,7 +2159,8 @@ def queryVulnCountAll(org_id): close(conn) return len(webIdsList) -def queryVulnCountSeverity(org_id,severity): + +def queryVulnCountSeverity(org_id, severity): """Query the amount of webapps with vulnerabilities.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available conn = connect() @@ -2119,11 +2174,12 @@ def queryVulnCountSeverity(org_id,severity): OR fstatus = 'REOPENED' ); """ - df = pd.read_sql_query(sql.format(org_id,severity), conn) + df = pd.read_sql_query(sql.format(org_id, severity), conn) webIdsList = df["webapp_id"].values.tolist() close(conn) return len(webIdsList) + def insertWASVulnData(data): """Insert WAS vulnerability data into database.""" conn = connect() @@ -2132,23 +2188,23 @@ def insertWASVulnData(data): VALUES ('{}','{}',{},{},{}, (CASE WHEN {} = 0 THEN NULL ELSE {} END), (CASE WHEN {} = 0 THEN NULL ELSE {} END),'{}',{},{},{},{},{}) """ cur.execute( sql.format( - data['was_org_id'], - data['date_scanned'], - data['vuln_cnt'], - data['vuln_webapp_cnt'], - data['web_app_cnt'], - data['high_rem_time'], - data['high_rem_time'], - data['crit_rem_time'], - data['crit_rem_time'], - data['report_period'], - data['high_vuln_cnt'], - data['crit_vuln_cnt'], - data['high_rem_cnt'], - data['crit_rem_cnt'], - data['total_potential'], - ) + data["was_org_id"], + data["date_scanned"], + data["vuln_cnt"], + data["vuln_webapp_cnt"], + data["web_app_cnt"], + data["high_rem_time"], + data["high_rem_time"], + data["crit_rem_time"], + data["crit_rem_time"], + data["report_period"], + data["high_vuln_cnt"], + data["crit_vuln_cnt"], + data["high_rem_cnt"], + data["crit_rem_cnt"], + data["total_potential"], ) + ) conn.commit() close(conn) print("Success adding finding data to database.") @@ -2230,7 +2286,7 @@ def getSubdomain_tsql(domain): if conn is not None: close(conn) - + # --- 703 pe-reports OLD TSQL --- def org_root_domains_tsql(conn, org_uid): """Get root domains from database given the org_uid.""" @@ -2291,12 +2347,12 @@ def getDataSource(conn, source): # --- 709 pe-reports/008 atc-framework OLD TSQL --- -# This TSQL is from the execute_hibp_breach_values() function in +# This TSQL is from the execute_hibp_breach_values() function in # hibp_latest.py and hibp_latest_rev.py # --- 710 pe-reports/009 atc-framework OLD TSQL --- -# This TSQL is from the execute_hibp_emails_values() function in +# This TSQL is from the execute_hibp_emails_values() function in # hibp_latest.py and hibp_latest_rev.py @@ -2306,12 +2362,12 @@ def getDataSource(conn, source): # --- 011 atc-framework OLD TSQL --- -# This TSQL is from the query_orgs() function in +# This TSQL is from the query_orgs() function in # adhoc/data/run.py # --- 012 atc-framework OLD TSQL --- -# This TSQL is from the query_PE_subs() function in +# This TSQL is from the query_PE_subs() function in # hibp_latest.py and hibp_latest_rev.py @@ -2320,7 +2376,7 @@ def getDataSource(conn, source): def insert_shodan_data(dataframe, table, thread, org_name, failed): """Insert Shodan data into database.""" # get rid of \x00 characters - dataframe = dataframe.replace({'\x00': ''}, regex=True) + dataframe = dataframe.replace({"\x00": ""}, regex=True) conn = connect() tpls = [tuple(x) for x in dataframe.to_numpy()] cols = ",".join(list(dataframe.columns)) @@ -2349,4 +2405,4 @@ def insert_shodan_data(dataframe, table, thread, org_name, failed): failed.append("{} failed inserting into {}".format(org_name, table)) conn.rollback() cursor.close() - return failed \ No newline at end of file + return failed diff --git a/src/pe_source/data/shodan_db/shodan_search.py b/src/pe_source/data/shodan_db/shodan_search.py index 5341cff6..f465852a 100644 --- a/src/pe_source/data/shodan_db/shodan_search.py +++ b/src/pe_source/data/shodan_db/shodan_search.py @@ -6,7 +6,7 @@ import time # Third-Party Libraries -import pandas as pd +# import pandas as pd import requests import shodan @@ -15,7 +15,7 @@ get_data_source_uid, get_ips, insert_shodan_assets, - insert_shodan_vulns + insert_shodan_vulns, ) LOGGER = logging.getLogger(__name__) @@ -170,7 +170,7 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "cpe": d.get("cpe", None), "banner": d.get("data", None), "version": d.get("version", None), - "data_source_uid": source_uid + "data_source_uid": source_uid, } ) elif d["_shodan"]["module"] in risky_ports: @@ -215,7 +215,7 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "cpe": d.get("cpe", None), "banner": d.get("data", None), "version": d.get("version", None), - "data_source_uid": source_uid + "data_source_uid": source_uid, } ) @@ -236,7 +236,7 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "timestamp": d["timestamp"], "country_code": location["country_code"], "location": str(location), - "data_source_uid": source_uid + "data_source_uid": source_uid, } ) @@ -277,12 +277,9 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): all_vulns = vuln_data + risk_data # Grab the data source uid and add to each dataframe - # Insert data into the PE database - failed = insert_shodan_assets(data, "shodan_assets", thread_name, org_name, failed) - failed = insert_shodan_vulns( - all_vulns, "shodan_vulns", thread_name, org_name, failed - ) + failed = insert_shodan_assets(data, failed) + failed = insert_shodan_vulns(all_vulns, failed) return failed diff --git a/src/pe_source/intelx_identity.py b/src/pe_source/intelx_identity.py index 5462f29b..19d0324a 100644 --- a/src/pe_source/intelx_identity.py +++ b/src/pe_source/intelx_identity.py @@ -11,12 +11,10 @@ import requests from .data.pe_db.config import get_params -from .data.pe_db.db_query_source import ( - connect, +from .data.pe_db.db_query_source import ( # get_intelx_breaches,; connect, get_data_source_uid, - get_intelx_breaches, get_orgs, - get_root_domains, + get_root_domains_api, insert_intelx_breaches, insert_intelx_credentials, ) @@ -77,21 +75,25 @@ def run_intelx(self): # Verify the org is in the list of orgs to scan if cyhy_org_id in orgs_list or orgs_list == "all" or orgs_list == "DEMO": - LOGGER.info(f"Running IntelX on {cyhy_org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Running IntelX on {cyhy_org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if self.get_credentials(cyhy_org_id, pe_org_uid) == 1: LOGGER.error("Failed to get credentials for %s", cyhy_org_id) failed += 1 else: - success +=1 - - LOGGER.info(f"IntelX scan ran successfully for {success}/{len(pe_orgs_final)} organizations") + success += 1 + LOGGER.info( + f"IntelX scan ran successfully for {success}/{len(pe_orgs_final)} organizations" + ) def get_credentials(self, cyhy_org_id, pe_org_uid): """Get credentials for a provided org.""" try: - conn = connect() - roots_df = get_root_domains(conn, pe_org_uid) + # conn = connect() + roots_df = get_root_domains_api(pe_org_uid) + except Exception as e: LOGGER.error("Failed fetching root domains for %s", cyhy_org_id) LOGGER.error(e) @@ -100,6 +102,7 @@ def get_credentials(self, cyhy_org_id, pe_org_uid): leaks_json = self.find_credential_leaks( roots_df["root_domain"].values.tolist(), START_DATE, END_DATE ) + print(leaks_json) if len(leaks_json) < 1: LOGGER.info(f"No credentials found for {cyhy_org_id}") return 0 @@ -112,11 +115,11 @@ def get_credentials(self, cyhy_org_id, pe_org_uid): LOGGER.error(e) return 1 - breach_dict = get_intelx_breaches(SOURCE_UID) - breach_dict = dict(breach_dict) - for cred_index, cred_row in creds_df.iterrows(): - breach_uid = breach_dict[cred_row["breach_name"]] - creds_df.at[cred_index, "credential_breaches_uid"] = breach_uid + # breach_dict = get_intelx_breaches(SOURCE_UID) + # breach_dict = dict(breach_dict) + # for cred_index, cred_row in creds_df.iterrows(): + # breach_uid = breach_dict[cred_row["breach_name"]] + # creds_df.at[cred_index, "credential_breaches_uid"] = breach_uid try: insert_intelx_credentials(creds_df) except Exception as e: @@ -314,4 +317,4 @@ def process_leaks_results(self, leaks_json, org_uid): ] ] - return creds_df, breaches_df \ No newline at end of file + return creds_df, breaches_df diff --git a/src/pe_source/pe_scripts.py b/src/pe_source/pe_scripts.py index eb5a98c8..24e6fd75 100644 --- a/src/pe_source/pe_scripts.py +++ b/src/pe_source/pe_scripts.py @@ -5,7 +5,7 @@ Arguments: DATA_SOURCE Source to collect data from. Valid values are "cybersixgill", - "dnstwist", "hibp", "intelx", "pshtt", and "shodan". + "dnstwist", "xpanse" "hibp", "intelx", "pshtt", and "shodan". Options: -h --help Show this message. @@ -28,11 +28,9 @@ # Standard Python Libraries import logging import sys -import time from typing import Any, Dict # Third-Party Libraries -from datetime import timedelta import docopt from schema import And, Schema, SchemaError, Use @@ -46,6 +44,7 @@ from .intelx_identity import IntelX from .pshtt_wrapper import launch_pe_pshtt from .shodan_wrapper import Get_shodan +from .xpanse_alert_pull import run_xpanse_scans LOGGER = logging.getLogger(__name__) @@ -53,65 +52,34 @@ def run_pe_script(source, orgs_list, cybersix_methods, soc_med_included): """Collect data from the source specified.""" # If not "all", separate orgs string into a list of orgs - if orgs_list != "all" and orgs_list != "DEMO": + if orgs_list != "all" and orgs_list != "DEMO" and source != "xpanse": orgs_list = orgs_list.split(",") # If not "all", separate Cybersixgill methods string into a list - sixgill_scan_name = cybersix_methods.title() if cybersix_methods == "all": cybersix_methods = ["alerts", "mentions", "credentials", "topCVEs"] else: cybersix_methods = cybersix_methods.split(",") - # LOGGER.info("Running %s on these orgs: %s", source, orgs_list) + LOGGER.info("Running %s on these orgs: %s", source, orgs_list) if source == "cybersixgill": - if sixgill_scan_name == "Topcves": - sixgill_scan_name = "Top CVEs" - LOGGER.info(f"--- Cybersixgill {sixgill_scan_name} Scan Starting ---") - LOGGER.info(f"Running Cybersixgill {sixgill_scan_name} on these orgs: {orgs_list}") - sixgill_start_time = time.time() cybersix = Cybersixgill(orgs_list, cybersix_methods, soc_med_included) cybersix.run_cybersixgill() - sixgill_end_time = time.time() - LOGGER.info(f"Execution time for Cybersixgill {sixgill_scan_name} scan: {str(timedelta(seconds=(sixgill_end_time - sixgill_start_time)))} (H:M:S)") - LOGGER.info(f"--- Cybersixgill {sixgill_scan_name} Scan Complete ---") elif source == "shodan": - LOGGER.info("--- Shodan Scan Starting ---") - LOGGER.info(f"Running Shodan on these orgs: {orgs_list}") - shodan_start_time = time.time() shodan = Get_shodan(orgs_list) shodan.run_shodan() - shodan_end_time = time.time() - LOGGER.info(f"Execution time for Shodan scan: {str(timedelta(seconds=(shodan_end_time - shodan_start_time)))} (H:M:S)") - LOGGER.info("--- Shodan Scan Complete ---") elif source == "dnsmonitor": - LOGGER.info("--- DNSMonitor Scan Starting ---") - LOGGER.info(f"Running DNSMonitor on these orgs: {orgs_list}") - dnsmonitor_start_time = time.time() dnsMonitor = DNSMonitor(orgs_list) dnsMonitor.run_dnsMonitor() - dnsmonitor_end_time = time.time() - LOGGER.info(f"Execution time for DNSMonitor scan: {str(timedelta(seconds=(dnsmonitor_end_time - dnsmonitor_start_time)))} (H:M:S)") - LOGGER.info("--- DNSMonitor Scan Complete ---") elif source == "dnstwist": - LOGGER.info("--- DNSTwist Scan Starting ---") - LOGGER.info(f"Running DNSTwist on these orgs: {orgs_list}") - dnstwist_start_time = time.time() run_dnstwist(orgs_list) - dnstwist_end_time = time.time() - LOGGER.info(f"Execution time for DNSTwist scan: {str(timedelta(seconds=(dnstwist_end_time - dnstwist_start_time)))} (H:M:S)") - LOGGER.info("--- DNSTwist Scan Complete ---") elif source == "intelx": - LOGGER.info("--- IntelX Scan Starting ---") - LOGGER.info(f"Running IntelX on these orgs: {orgs_list}") - intelx_start_time = time.time() intelx = IntelX(orgs_list) intelx.run_intelx() - intelx_end_time = time.time() - LOGGER.info(f"Execution time for IntelX scan: {str(timedelta(seconds=(intelx_end_time - intelx_start_time)))} (H:M:S)") - LOGGER.info("--- IntelX Scan Complete ---") elif source == "pshtt": launch_pe_pshtt() + elif source == "xpanse": + run_xpanse_scans("", orgs_list) else: logging.error( "Not a valid source name. Correct values are cybersixgill or shodan." diff --git a/src/pe_source/shodan_wrapper.py b/src/pe_source/shodan_wrapper.py index d2e441e6..c34ed677 100644 --- a/src/pe_source/shodan_wrapper.py +++ b/src/pe_source/shodan_wrapper.py @@ -1,21 +1,24 @@ """Collect Shodan data.""" # Standard Python Libraries +# from datetime import timedelta import logging import threading -import time # Third-Party Libraries -from datetime import timedelta import numpy from .data.pe_db.config import shodan_api_init from .data.pe_db.db_query_source import get_orgs from .data.shodan_db.shodan_search import run_shodan_thread +# import time + + # Logging LOGGER = logging.getLogger(__name__) + class Get_shodan: """Fetch Shodan data.""" @@ -29,7 +32,6 @@ def run_shodan(self): # Get orgs from PE database pe_orgs = get_orgs() - # Filter orgs if specified pe_orgs_final = [] if orgs_list == "all": @@ -61,7 +63,7 @@ def run_shodan(self): i = 0 thread_list = [] while i < len(chunked_orgs_list): - thread_name = f"Thread {i+1}:" + thread_name = f"Thread {i + 1}:" # Start thread t = threading.Thread( target=run_shodan_thread, @@ -73,4 +75,4 @@ def run_shodan(self): # Wait until all threads finish to continue for thread in thread_list: - thread.join() \ No newline at end of file + thread.join() diff --git a/src/pe_source/xpanse_alert_pull.py b/src/pe_source/xpanse_alert_pull.py index b3780622..c08f20fd 100644 --- a/src/pe_source/xpanse_alert_pull.py +++ b/src/pe_source/xpanse_alert_pull.py @@ -19,31 +19,35 @@ """ # Standard Python Libraries -import csv import datetime import json import logging -import sys -from typing import Any, Dict +import time # Third-Party Libraries -from _version import __version__ -from data.pe_db.db_query_source import ( # api_pull_xpanse_vulns, - api_xpanse_alert_insert, - get_linked_xpanse_business_units, -) -import docopt +# import docopt import pytz import requests -from schema import And, Or, Schema, SchemaError, Use # cisagov Libraries -import pe_reports +# import pe_reports from pe_reports.data.config import staging_config +# from _version import __version__ +from .data.pe_db.db_query_source import ( # api_pull_xpanse_vulns, + api_xpanse_alert_insert, + get_linked_xpanse_business_units, +) + +# import sys +# from typing import Any, Dict + + +# from schema import And, Or, Schema, SchemaError, Use + + API_DIC = staging_config(section="xpanse") xpanse_url = "https://api-cisa-xpanse.crtx.gv.paloaltonetworks.com/public_api/" -# "https://api-cisa.crtx.federal.paloaltonetworks.com/public_api/" api_key = API_DIC.get("api_key") auth_id = API_DIC.get("auth_id") @@ -80,16 +84,12 @@ def pull_alerts_data(linked_org_list, business_units_list=[]): """Pull alerts data from the Xpanse API.""" url = xpanse_url + "v2/alerts/get_alerts_multi_events" - + if len(business_units_list) == 0: - business_units_list = list(map(lambda d: d['entity_name'], linked_org_list)) + business_units_list = list(map(lambda d: d["entity_name"], linked_org_list)) for org in business_units_list: - request_data = { - "use_page_token": True, - "search_from":0, - "search_to": 5000, - } + request_data = {"use_page_token": True, "search_from": 0, "search_to": 5000} filters = [] LOGGER.info("Running Xpanse alert pull on %s", org) filters.append( @@ -115,6 +115,7 @@ def pull_alerts_data(linked_org_list, business_units_list=[]): try: response = requests.request("POST", url, headers=headers, data=payload) resp_dict = response.json() + print(resp_dict) page_token = resp_dict["reply"]["next_page_token"] LOGGER.info( @@ -122,7 +123,7 @@ def pull_alerts_data(linked_org_list, business_units_list=[]): ) formatted_alerts = format_alerts(resp_dict["reply"]["alerts"]) - + for alert in formatted_alerts: api_xpanse_alert_insert(alert) @@ -140,6 +141,7 @@ def pull_alerts_data(linked_org_list, business_units_list=[]): for alert in formatted_alerts: api_xpanse_alert_insert(alert) + LOGGER.info("Done Xpanse alert pull on %s", org) except Exception as e: LOGGER.error("Error querying assets for %s: %s.", org, e) @@ -183,7 +185,7 @@ def pull_alerts_data(linked_org_list, business_units_list=[]): # "tags": asset.get("tags", None), # } - # return asset_dict +# return asset_dict def format_alerts(alerts): @@ -192,9 +194,9 @@ def format_alerts(alerts): service_ids = [] for alert in alerts: try: - service_ids += alert.get('service_ids', []) - alert_services_dict[alert['alert_id']] = alert.get('service_ids', []) - except: + service_ids += alert.get("service_ids", []) + alert_services_dict[alert["alert_id"]] = alert.get("service_ids", []) + except Exception: continue services = [] @@ -206,7 +208,7 @@ def format_alerts(alerts): for service_chunk in service_id_chunks: max_retries = 3 - retry_delay = 5 + retry_delay = 5 for retry_count in range(max_retries): try: service_response = pull_service_data(service_chunk) @@ -223,7 +225,6 @@ def format_alerts(alerts): else: # If it's the last retry, set it to None and it will skip to the next chunk service_response = None - if service_response is None: continue @@ -264,15 +265,9 @@ def format_alerts(alerts): "version_number": cve["inferredCve"][ "inferredCveMatchMetadata" ].get("version", None), - "activity_status": cve.get( - "activityStatus", None - ), - "first_observed": cve.get( - "firstObserved", None - ), - "last_observed": cve.get( - "lastObserved", None - ), + "activity_status": cve.get("activityStatus", None), + "first_observed": cve.get("firstObserved", None), + "last_observed": cve.get("lastObserved", None), }, ) ) @@ -331,11 +326,11 @@ def format_alerts(alerts): if tag.startswith("BU:"): business_units_list.append(tag[3:].strip()) # print(business_units_list) - except: + except Exception: business_units_list = [] - + assets = [] - ##Uncomment to track assets + # #Uncomment to track assets # asset_ids = alert["asset_ids"] # if asset_ids is not None: # asset_id_chunks = [ @@ -348,15 +343,18 @@ def format_alerts(alerts): current_services = [] try: for service in alert.get("service_ids", []): - service_identified = next((d for d in services if d.get('service_id') == service), None) + service_identified = next( + (d for d in services if d.get("service_id") == service), None + ) if service_identified: - current_services.append(service_identified) - except: + current_services.append(service_identified) + except Exception: pass - alert_dict = { - "time_pulled_from_xpanse": datetime.datetime.utcnow().replace(tzinfo=pytz.utc), + "time_pulled_from_xpanse": datetime.datetime.utcnow().replace( + tzinfo=pytz.utc + ), "alert_id": alert.get("alert_id", None), "detection_timestamp": alert.get("detection_timestamp", None), "alert_name": alert.get("name", None), @@ -393,8 +391,10 @@ def format_alerts(alerts): "matching_status": alert.get("matching_status", None), # end_match_attempt_ts ??? null, "local_insert_ts": alert.get("local_insert_ts", None), - "last_modified_ts": alert.get("last_modified_ts") if alert.get("last_modified_ts") is not None else alert.get("local_insert_ts", None), - "case_id": alert.get("case_id", None), + "last_modified_ts": alert.get("last_modified_ts") + if alert.get("last_modified_ts") is not None + else alert.get("local_insert_ts", None), + "case_id": alert.get("case_id", None), # deduplicate_tokens ??? null, # filter_rule_id ??? null, # event_id ??? null, @@ -453,6 +453,7 @@ def format_alerts(alerts): alert_list.append(alert_dict) return alert_list + def pull_service_data(service_id_list): """Pull service info from the Xpanse API using a service_id.""" url = xpanse_url + "v1/assets/get_external_service" @@ -481,75 +482,8 @@ def run_xpanse_scans(last_modified, orgs_list): orgs_list = [] linked_org_list = get_linked_xpanse_business_units() - + pull_alerts_data(linked_org_list, orgs_list) # api_pull_xpanse_vulns(orgs_list[0], datetime.datetime(2023, 10, 10, 1, 00)) return 1 - - -def main(): - """Launch Xpanse scans.""" - args: Dict[str, str] = docopt.docopt(__doc__, version=__version__) - - schema: Schema = Schema( - { - "--log-level": And( - str, - Use(str.lower), - lambda n: n in ("debug", "info", "warning", "error", "critical"), - error="Possible values for --log-level are " - + "debug, info, warning, error, and critical.", - ), - str: object, # Don't care about other keys, if any - } - ) - - try: - validated_args: Dict[str, Any] = schema.validate(args) - except SchemaError as err: - # Exit because one or more of the arguments were invalid - print(err, file=sys.stderr) - sys.exit(1) - - # Assign validated arguments to variables - log_level: str = validated_args["--log-level"] - - # Set up logging - logging.basicConfig( - filename=pe_reports.CENTRAL_LOGGING_FILE, - filemode="a", - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - datefmt="%m/%d/%Y %I:%M:%S", - level=log_level.upper(), - ) - - run_xpanse_scans( - validated_args["--last_modified"], - validated_args["--orgs"], - ) - -def print_start_time(): - global start_time - start_time = datetime.datetime.now() - print(f"Script started at: {start_time}") - -# Function to print the end time and calculate duration -def print_end_time(): - end_time = datetime.datetime.now() - print(f"Script ended at: {end_time}") - - # Calculate duration - duration = end_time - start_time - - # Convert duration to hours, minutes, seconds - hours, remainder = divmod(duration.seconds, 3600) - minutes, seconds = divmod(remainder, 60) - - print(f"Script took {hours} hours, {minutes} minutes, and {seconds} seconds to run.") - - -if __name__ == "__main__": - print_start_time() - main() - print_end_time() \ No newline at end of file From 5120bfdd4bf4588a5524e05529dec2559f786e51 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:08:43 -0700 Subject: [PATCH 7/7] Add WAS scan and help text to models Add WAS scan updates and help text to models --- .../dataAPI/schemas.py | 153 + .../dataAPI/tasks.py | 165 +- .../dataAPI/views.py | 195 +- .../dmz_mini_dl/models.py | 5596 ++++++++++++----- .../pe_reports_django_project/home/models.py | 50 + src/pe_source/cybersixgill.py | 52 +- 6 files changed, 4581 insertions(+), 1630 deletions(-) diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py index 4855a550..bfc502ab 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py @@ -4100,3 +4100,156 @@ class OrgAssetTaskResp(BaseModel): status: str result: Optional[OrgAssetPagedResult] = None error: Optional[str] = None + + +class WasFindingInsert(BaseModel): + """WasFindingResult schema class""" + finding_uid: str + finding_type: Optional[str] = None + webapp_id: Optional[str] = None + was_org_id: Optional[str] = None + owasp_category: Optional[str] = None + severity: Optional[str] = None + times_detected: Optional[int] = None + base_score: Optional[float] = None + temporal_score: Optional[float] = None + fstatus: Optional[str] = None + last_detected: Optional[str] = None + first_detected: Optional[str] = None + is_remediated: Optional[bool] = None + potential: Optional[bool] = None + webapp_url: Optional[str] = None + webapp_name: Optional[str] = None + name: Optional[str] = None + cvss_v3_attack_vector: Optional[str] = None + cwe_list: Optional[List[str]] = None + wasc_list: Optional[List[Dict]] = None + last_tested: Optional[str] = None + fixed_date: Optional[str] = None + is_ignored: Optional[bool] = None + url: Optional[str] = None + qid: Optional[int] = None + response: Optional[str] = None + +class WasFindingResult(BaseModel): + """WasFindingResult schema class""" + finding_uid: str + finding_type: Optional[str] = None + webapp_id: Optional[str] = None + was_org_id: Optional[str] = None + owasp_category: Optional[str] = None + severity: Optional[str] = None + times_detected: Optional[int] = None + base_score: Optional[float] = None + temporal_score: Optional[float] = None + fstatus: Optional[str] = None + last_detected: Optional[str] = None + first_detected: Optional[str] = None + is_remediated: Optional[bool] = None + potential: Optional[bool] = None + webapp_url: Optional[str] = None + webapp_name: Optional[str] = None + name: Optional[str] = None + cvss_v3_attack_vector: Optional[str] = None + cwe_list: Optional[List[str]] = None + wasc_list: Optional[List[Dict]] = None + last_tested: Optional[str] = None + fixed_date: Optional[str] = None + is_ignored: Optional[bool] = None + url: Optional[str] = None + qid: Optional[int] = None + response: Optional[str] = None + +class WasFindingPagedResp(BaseModel): + """OrgAssetPagedResult schema class.""" + + total_pages: int + current_page: int + data: Optional[List[WasFindingResult]] = None + +class WasFindingsTaskResp(BaseModel): + """WasFindingsTaskResp schema class.""" + task_id: str + status: str + result: Optional[WasFindingPagedResp] = None + error: Optional[str] = None + + +class WasFindingsPagedInput(BaseModel): + """WasFindingsPagedInput schema class.""" + + org_acronym: str + page: int + per_page: int + + class Config: + """WasFindingsPagedInput schema config class.""" + + orm_mode = True + + + +class WasReportInsert(BaseModel): + """WasReportInsert schema class.""" + + org_name: Optional[str] = Field(default=None) + date_pulled: Optional[str] = Field(default=None) + last_scan_date: Optional[str] = Field(default=None) + security_risk: Optional[str] = Field(default=None) + total_info: Optional[int] = Field(default=None) + num_apps: Optional[int] = Field(default=None) + risk_color: Optional[str] = Field(default=None) + sensitive_count: Optional[int] = Field(default=None) + sensitive_color: Optional[str] = Field(default=None) + max_days_open_urgent: Optional[int] = Field(default=None) + max_days_open_critical: Optional[int] = Field(default=None) + urgent_color: Optional[str] = Field(default=None) + critical_color: Optional[str] = Field(default=None) + org_was_acronym: Optional[str] = Field(default=None) + name_len: Optional[str] = Field(default=None) + vuln_csv_dict: Optional[Dict] = Field(default={}) + ssn_cc_dict: Optional[Dict] = Field(default={}) + app_overview_csv_dict: Optional[Dict] = Field(default={}) + details_csv: Optional[List] = Field(default=[]) + info_csv: Optional[List] = Field(default=[]) + links_crawled: Optional[List] = Field(default=[]) + links_rejected: Optional[List] = Field(default=[]) + emails_found: Optional[List] = Field(default=[]) + owasp_count_dict: Optional[Dict] = Field(default={}) + group_count_dict: Optional[Dict] = Field(default={}) + fixed: Optional[int] = Field(default=None) + total: Optional[int] = Field(default=None) + vulns_monthly_dict: Optional[Dict] = Field(default={}) + path_disc: Optional[int] = Field(default=None) + info_disc: Optional[int] = Field(default=None) + cross_site: Optional[int] = Field(default=None) + burp: Optional[int] = Field(default=None) + sql_inj: Optional[int] = Field(default=None) + bugcrowd: Optional[int] = Field(default=None) + reopened: Optional[int] = Field(default=None) + reopened_color: Optional[str] = Field(default=None) + new_vulns: Optional[int] = Field(default=None) + new_vulns_color: Optional[str] = Field(default=None) + tot_vulns: Optional[int] = Field(default=None) + tot_vulns_color: Optional[str] = Field(default=None) + lev1: Optional[int] = Field(default=None) + lev2: Optional[int] = Field(default=None) + lev3: Optional[int] = Field(default=None) + lev4: Optional[int] = Field(default=None) + lev5: Optional[int] = Field(default=None) + severities: Optional[List[int]] = Field(default=[]) + ages: Optional[List[int]] = Field(default=[]) + pdf_obj: Optional[str] = Field(default=None) + + class Config: + """WasReportInsert schema config class.""" + + orm_mode = True + + +class WasReportTaskResp(BaseModel): + """WasReportTaskResp schema class.""" + task_id: str + status: str + result: Optional[WasReportInsert] = None + error: Optional[str] = None \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py b/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py index 5ba2dd66..b7c25ca2 100755 --- a/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py @@ -60,8 +60,16 @@ VwShodanvulnsSuspected, VwShodanvulnsVerified, WasFindings, + WasReport, XpanseAlerts, ) +from dmz_mini_dl.models import ( + CredentialBreaches as MDL_CredentialBreaches, + CredentialExposures as MDL_CredentialExposures, + DataSource as MDL_DataSource, + Organization as MDL_Organization, + WasReport as MDL_WasReport +) # cisagov Libraries from pe_reports.helpers import ip_passthrough @@ -1223,8 +1231,31 @@ def cred_breach_sixgill_task(self, new_breaches: List[dict]): """Task function for the cred_breaches_sixgill_insert API endpoint.""" create_ct = 0 update_ct = 0 + try: + mdl_source_inst = MDL_DataSource.objects.get(name="Sixgill") + except MDL_DataSource.DoesNotExist: + LOGGER.warning(f"DataSource Sixgill not found.") + mdl_source_inst = None # Set to None if DataSource is not found for new_breach in new_breaches: # Insert each row of data + try: + MDL_CredentialBreaches.objects.get(breach_name=new_breach["breach_name"]) + + MDL_CredentialBreaches.objects.filter(breach_name=new_breach["breach_name"] + ).update( + password_included=new_breach["password_included"], + ) + except MDL_CredentialBreaches.DoesNotExist: + MDL_CredentialBreaches.objects.create( + credential_breaches_uid=uuid.uuid1(), + breach_name=new_breach["breach_name"], + description=new_breach["description"], + breach_date=new_breach["breach_date"], + password_included=new_breach["password_included"], + data_source=mdl_source_inst, + modified_date=new_breach["modified_date"], + ) + try: CredentialBreaches.objects.get(breach_name=new_breach["breach_name"]) # If record already exists, update @@ -1264,7 +1295,43 @@ def cred_exp_sixgill_task(self, new_exposures: List[dict]): """Task function for the credexp_insert API endpoint.""" update_ct = 0 create_ct = 0 + try: + mdl_source_inst = MDL_DataSource.objects.get(name="Sixgill") + except MDL_DataSource.DoesNotExist: + LOGGER.warning(f"DataSource Sixgill not found.") + mdl_source_inst = None # Set to None if DataSource is not found for new_exposure in new_exposures: + curr_org_inst = Organizations.objects.get( + organizations_uid=new_exposure["organizations_uid"] + ) + try: + MDL_CredentialExposures.objects.get( + breach_name=new_exposure["breach_name"], + email=new_exposure["email"], + ) + except MDL_CredentialExposures.DoesNotExist: + mdl_org_inst = MDL_Organization.objects.get( + acronym=curr_org_inst.cyhy_db_name + ) + + mdl_breach_inst = CredentialBreaches.objects.get( + breach_name=new_exposure["breach_name"] + ) + CredentialExposures.objects.create( + credential_exposures_uid=uuid.uuid1(), + modified_date=new_exposure["modified_date"], + sub_domain=new_exposure["sub_domain"], + email=new_exposure["email"], + hash_type=new_exposure["hash_type"], + name=new_exposure["name"], + login_id=new_exposure["login_id"], + password=new_exposure["password"], + phone=new_exposure["phone"], + breach_name=new_exposure["breach_name"], + organization=mdl_org_inst, + data_source=mdl_source_inst, + credential_breaches=mdl_breach_inst, + ) try: CredentialExposures.objects.get( breach_name=new_exposure["breach_name"], @@ -1272,9 +1339,7 @@ def cred_exp_sixgill_task(self, new_exposures: List[dict]): ) except CredentialExposures.DoesNotExist: # If cred exp record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( - organizations_uid=new_exposure["organizations_uid"] - ) + curr_source_inst = DataSource.objects.get( data_source_uid=new_exposure["data_source_uid"] ) @@ -1658,4 +1723,96 @@ def get_orgs_and_assets(self, page: int, per_page: int): "current_page": page, "data": orgs_list, } - return result \ No newline at end of file + return result + +def base64_to_bytes(base64_str) -> bytes: + """Convert a Base64-encoded string to binary data.""" + if base64_str: + return base64.b64decode(base64_str) + else: + return base64_str + +@shared_task(bind=True) +def insert_was_report(self, data): + # try: + LOGGER.info('Top of Was Report Task') + LOGGER.info(data) + defaults_dict={ + "org_name": data.get('org_name'), + "date_pulled": data.get('date_pulled'), + "last_scan_date": data.get('last_scan_date'), + "security_risk": data.get('security_risk'), + "total_info": data.get('total_info'), + "num_apps": data.get('num_apps'), + "risk_color": data.get('risk_color'), + "sensitive_count": data.get('sensitive_count'), + "sensitive_color": data.get('sensitive_color'), + "max_days_open_urgent": data.get('max_days_open_urgent'), + "max_days_open_critical": data.get('max_days_open_critical'), + "urgent_color": data.get('urgent_color'), + "critical_color": data.get('critical_color'), + "name_len": data.get('name_len'), + "vuln_csv_dict": data.get('vuln_csv_dict'), + "ssn_cc_dict": data.get('ssn_cc_dict'), + "app_overview_csv_dict": data.get('app_overview_csv_dict'), + "details_csv": data.get('details_csv'), + "info_csv": data.get('info_csv'), + "links_crawled": data.get('links_crawled'), + "links_rejected": data.get('links_rejected'), + "emails_found": data.get('emails_found'), + "owasp_count_dict": data.get('owasp_count_dict'), + "group_count_dict": data.get('group_count_dict'), + "fixed": data.get('fixed'), + "total": data.get('total'), + "vulns_monthly_dict": data.get('vulns_monthly_dict'), + "path_disc": data.get('path_disc'), + "info_disc": data.get('info_disc'), + "cross_site": data.get('cross_site'), + "burp": data.get('burp'), + "sql_inj": data.get('sql_inj'), + "bugcrowd": data.get('bugcrowd'), + "reopened": data.get('reopened'), + "reopened_color": data.get('reopened_color'), + "new_vulns": data.get('new_vulns'), + "new_vulns_color": data.get('new_vulns_color'), + "tot_vulns": data.get('tot_vulns'), + "tot_vulns_color": data.get('tot_vulns_color'), + "lev1": data.get('lev1'), + "lev2": data.get('lev2'), + "lev3": data.get('lev3'), + "lev4": data.get('lev4'), + "lev5": data.get('lev5'), + "severities": data.get('severities'), + "ages": data.get('ages'), + "pdf_obj": base64_to_bytes(data.get('pdf_obj')) + + } + was_report_object, created = WasReport.objects.update_or_create( + org_was_acronym = data.get('org_was_acronym'), + last_scan_date = data.get('last_scan_date'), + defaults=defaults_dict + ) + try: + mdl_was_report_object, mdl_created = MDL_WasReport.objects.update_or_create( + org_was_acronym = data.get('org_was_acronym'), + last_scan_date = data.get('last_scan_date'), + defaults=defaults_dict + ) + except Exception: + LOGGER.info(f"Failed to insert WAS report for {data.get('org_was_acronym')}") + + + if created: + LOGGER.info("New Was record created for %s", data.get('org_was_acronym')) + return { + "message": "New Was record created.", + "was_report_id": was_report_object.id, + } + else: + return {"message": "Record updated successfully.", "was_report_id": was_report_object.id} + + # except Exception as e: + # LOGGER.error(e) + # print("failed to insert or update") + # return {"message": "Failed to insert or update.", "was_report_obj": None, "error": e} + # LOGGER.info("API key expired please try again") \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/views.py b/src/pe_reports/pe_reports_django_project/dataAPI/views.py index b518f949..a623ea82 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/views.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/views.py @@ -63,6 +63,7 @@ sub_domains_table_task, top_cves_insert_task, was_vulns_task, + insert_was_report ) from decouple import config from django.conf import settings @@ -85,6 +86,7 @@ from dmz_mini_dl.models import XpanseCveServiceMdl as MDL_XpanseCveService from dmz_mini_dl.models import XpanseCvesMdl as MDL_XpanseCves from dmz_mini_dl.models import XpanseServicesMdl as MDL_XpanseServices +from dmz_mini_dl.models import WasFindings as MDL_WasFindings from fastapi import ( APIRouter, Depends, @@ -143,6 +145,8 @@ VwShodanvulnsSuspected, VwShodanvulnsVerified, WasTrackerCustomerdata, + WasFindings, + WasReport, WeeklyStatuses, XpanseAlerts, XpanseAssets, @@ -5617,24 +5621,24 @@ def cred_breaches_intelx_insert( for row in data.breach_data: # Check if record already exists row_dict = row.__dict__ - # mdl_source_inst = MDL_DataSource.objects.get( - # name='IntelX' - # ) - - MDL_CredentialBreaches.objects.update_or_create( - breach_name=row_dict["breach_name"], - defaults={ - "description": row_dict["description"], - "breach_date": row_dict["breach_date"], - "added_date": row_dict["added_date"], - "modified_date": timezone.make_aware( - parse_datetime(row_dict["modified_date"]), - timezone.timezone.utc, - ), - "password_included": row_dict["password_included"], - "data_source": mdl_source_inst, - }, - ) + try: + MDL_CredentialBreaches.objects.update_or_create( + breach_name=row_dict["breach_name"], + defaults={ + "description": row_dict["description"], + "breach_date": row_dict["breach_date"], + "added_date": row_dict["added_date"], + "modified_date": timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), + "password_included": row_dict["password_included"], + "data_source": mdl_source_inst, + }, + ) + except Exception: + LOGGER.warn('Failed to save or update Credential Breach to MDL.') + breach_results = CredentialBreaches.objects.filter( breach_name=row_dict["breach_name"] ) @@ -7614,3 +7618,158 @@ def query_shodan_ips(org_uid: str, tokens: dict = Depends(get_api_key)): LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} + + +@api_router.put( + "/was_finding_insert_or_update", + dependencies=[Depends(get_api_key)], + # response_model=Dict[schemas.PshttDataBase], + tags=["Update or insert WAS finding"], +) +# @transaction.atomic +def was_finding_insert_or_update( + # tag: str, + data: schemas.WasFindingInsert, + tokens: dict = Depends(get_api_key), +): + """Create API endpoint to create a record in database.""" + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + LOGGER.info(f"The api key submitted {tokens}") + defaults={ + 'finding_type': data.finding_type, + 'webapp_id': data.webapp_id, + 'webapp_url': data.webapp_url, + 'webapp_name': data.webapp_name, + 'was_org_id': data.was_org_id, + 'name': data.name, + 'owasp_category': data.owasp_category, + 'severity': data.severity, + 'times_detected': data.times_detected, + 'cvss_v3_attack_vector': data.cvss_v3_attack_vector, + 'base_score': data.base_score, + 'temporal_score': data.temporal_score, + 'fstatus': data.fstatus, + 'last_detected': data.last_detected, + 'first_detected': data.first_detected, + 'potential': data.potential, + 'cwe_list': data.cwe_list, + 'wasc_list': data.wasc_list, + 'last_tested': data.last_tested, + 'fixed_date': data.fixed_date, + 'is_ignored': data.is_ignored, + 'is_remediated': True if data.fstatus == "FIXED" else False, + 'url': data.url, + 'qid': data.qid, + 'response': data.response + } + ( + was_finding_object, + created, + ) = WasFindings.objects.update_or_create( + finding_uid= data.finding_uid, + defaults=defaults, + ) + try: + ( + mdl_was_finding_object, + mdl_created, + ) = MDL_WasFindings.objects.update_or_create( + finding_uid= data.finding_uid, + defaults=defaults, + ) + except Exception: + LOGGER.info(f'Failed to insert WAS finding to MDL: {data.finding_uid}') + + if created: + LOGGER.info( + "New WAS finding record created for %s", data.was_org_id + ) + return { + "message": "New WAS finding created.", + "was_finding_obj": was_finding_object, + } + return { + "message": "WAS finding updated.", + "was_finding_obj": was_finding_object, + } + except Exception as e: + LOGGER.warning(e) + print("failed to insert or update") + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} + +@api_router.post( + "/was_report_insert_or_update", + dependencies=[Depends(get_api_key)], + # response_model=Dict[schemas.PshttDataBase], + tags=["Update or insert Was Report."], +) +def was_report_insert_or_update( + # tag: str, + data: schemas.WasReportInsert, + tokens: dict = Depends(get_api_key), +): + """Create API endpoint to create a record in database.""" + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # LOGGER.info(f"The api key submitted {tokens}") + LOGGER.info(data) + serialized_data = data.dict() + LOGGER.info(serialized_data) + # Pass serialized data to Celery task + was_report_task = insert_was_report.delay(serialized_data) + # was_report_task = insert_was_report.delay(data) + was_report_task_id = was_report_task.id + # Return the new task id w/ "Processing" status + return {"task_id": was_report_task_id, "status": "Processing"} + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + except Exception as e: + print('In View Basic Exception') + LOGGER.error(e) + + else: + return {"message": "No api key was submitted"} + +@api_router.get( + "/was_report_insert_or_update/task/{task_id}", + dependencies=[ + Depends(get_api_key) + ], # Depends(RateLimiter(times=200, seconds=60))], + # response_model=schemas.WasReportTaskResp, + tags=["Check task status WAS Report insert."], +) +async def was_report_insert_task_status(task_id: str, tokens: dict = Depends(get_api_key)): + """Check task status WAS Report insert""" + # Check for API key + LOGGER.info(f"The api key submitted {tokens}") + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # Retrieve task status + task = insert_was_report.AsyncResult(task_id) + # Return appropriate message for status + if task.state == "SUCCESS": + return { + "task_id": task_id, + "status": "Completed", + "result": task.result, + } + elif task.state == "PENDING": + return {"task_id": task_id, "status": "Pending"} + elif task.state == "FAILURE": + return { + "task_id": task_id, + "status": "Failed", + "error": str(task.result), + } + else: + return {"task_id": task_id, "status": task.state} + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py index ef9f53da..b671f91f 100644 --- a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py @@ -11,80 +11,73 @@ # , NetManager manage_db = True +app_label_name = "xfd_mini_dl" class ApiKey(models.Model): """The ApiKey model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(auto_now_add=True, db_column="created_at") - updated_at = models.DateTimeField(auto_now=True, db_column="updated_at") - last_used = models.DateTimeField(db_column="last_used", blank=True, null=True) - hashed_key = models.TextField(db_column="hashed_key") - last_four = models.TextField(db_column="last_four") + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for an API key object." + ) + created_at = models.DateTimeField( + auto_now_add=True, + db_column="created_at", + help_text="Date and time the API key object was created.", + ) + updated_at = models.DateTimeField( + auto_now=True, + db_column="updated_at", + help_text="Date and time the API key object was last updated.", + ) + last_used = models.DateTimeField( + db_column="last_used", + blank=True, + null=True, + help_text="Last date and time the API key was used.", + ) + hashed_key = models.TextField( + db_column="hashed_key", help_text="Cryptographic hash of the API key" + ) + last_four = models.TextField( + db_column="last_four", help_text="Last for characters of the API key" + ) user = models.ForeignKey( - "User", models.CASCADE, db_column="user_id", blank=True, null=True + "User", + models.CASCADE, + db_column="user_id", + blank=True, + null=True, + help_text="FK: foreign key relationship to the user who owns the API key.", ) class Meta: """Meta class for ApiKey.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "api_key" -class Assessment(models.Model): - """The Assessment model.""" - - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - rsc_id = models.CharField(db_column="rsc_id", unique=True, max_length=255) - type = models.CharField(max_length=255) - user = models.ForeignKey( - "User", db_column="user_id", blank=True, null=True, on_delete=models.CASCADE - ) - - class Meta: - """The Meta class for Assessment.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "assessment" - - -class Category(models.Model): - """The Category model.""" - - id = models.UUIDField(primary_key=True) - name = models.CharField(max_length=255) - number = models.CharField(max_length=255, unique=True) - short_name = models.CharField( - db_column="short_name", max_length=255, blank=True, null=True - ) - - class Meta: - """The Meta class for Category model.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "category" - - class Cpe(models.Model): """The Cpe model.""" - id = models.UUIDField(primary_key=True) - name = models.CharField(max_length=255) - version = models.CharField(max_length=255) - vendor = models.CharField(max_length=255) - last_seen_at = models.DateTimeField(db_column="last_seen_at") + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for a CPE Product object." + ) + name = models.CharField(max_length=255, help_text="Name of the product.") + version = models.CharField(max_length=255, help_text="Version of the product.") + vendor = models.CharField( + max_length=255, help_text="Vendorr who created the product." + ) + last_seen_at = models.DateTimeField( + db_column="last_seen_at", help_text="Last datetime the CPE was seen." + ) class Meta: """The Meta class for Cpe.""" - app_label = "dmz_mini_dl" + app_label = app_label_name db_table = "cpe" managed = manage_db # This ensures Django does not manage the table unique_together = (("name", "version", "vendor"),) # Unique constraint @@ -93,158 +86,232 @@ class Meta: class Cve(models.Model): """The Cve model.""" - id = models.UUIDField(primary_key=True) - name = models.CharField(unique=True, blank=True, null=True, max_length=255) - published_at = models.DateTimeField(db_column="published_at", blank=True, null=True) - modified_at = models.DateTimeField(db_column="modified_at", blank=True, null=True) - status = models.CharField(blank=True, null=True, max_length=255) - description = models.TextField(blank=True, null=True) + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for a CVE object." + ) + name = models.CharField( + unique=True, blank=True, null=True, max_length=255, help_text="Name of the CVE." + ) + published_at = models.DateTimeField( + db_column="published_at", + blank=True, + null=True, + help_text="Date the CVE was published by NIST.", + ) + modified_at = models.DateTimeField( + db_column="modified_at", + blank=True, + null=True, + help_text="Datte the CVE was modified.", + ) + status = models.CharField( + blank=True, null=True, max_length=255, help_text="Status of the CVE." + ) + description = models.TextField( + blank=True, null=True, help_text="Description of the CVE." + ) cvss_v2_source = models.CharField( - db_column="cvss_v2_source", blank=True, null=True, max_length=255 + db_column="cvss_v2_source", + blank=True, + null=True, + max_length=255, + help_text="Organization or entity that assigned a CVSS v2 (Common Vulnerability Scoring System version 2) score to a particular vulnerability.", ) cvss_v2_type = models.CharField( - db_column="cvss_v2_type", blank=True, null=True, max_length=255 + db_column="cvss_v2_type", + blank=True, + null=True, + max_length=255, + help_text="Type of CVVS v2 score. (Primary, Secondary)", ) cvss_v2_version = models.CharField( - db_column="cvss_v2_version", blank=True, null=True, max_length=255 + db_column="cvss_v2_version", + blank=True, + null=True, + max_length=255, + help_text="Version of the CVSS v2 score.", ) cvss_v2_vector_string = models.CharField( - db_column="cvss_v2_vector_string", blank=True, null=True, max_length=255 + db_column="cvss_v2_vector_string", + blank=True, + null=True, + max_length=255, + help_text="Textual representation of the specific metrics used to calculate the CVSS v2 score.", ) cvss_v2_base_score = models.CharField( - db_column="cvss_v2_base_score", blank=True, null=True, max_length=255 + db_column="cvss_v2_base_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the severity of the vulnerability.", ) cvss_v2_base_severity = models.CharField( - db_column="cvss_v2_base_severity", blank=True, null=True, max_length=255 + db_column="cvss_v2_base_severity", + blank=True, + null=True, + max_length=255, + help_text="Qualitative categorization of a vulnerability's Base Score that helps assess its overall risk level in a more human-readable way.", ) cvss_v2_exploitability_score = models.CharField( - db_column="cvss_v2_exploitability_score", blank=True, null=True, max_length=255 + db_column="cvss_v2_exploitability_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the exploitability of the vulnerability.", ) cvss_v2_impact_score = models.CharField( - db_column="cvss_v2_impact_score", blank=True, null=True, max_length=255 + db_column="cvss_v2_impact_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the potential impact of the vulnerability.", ) cvss_v3_source = models.CharField( - db_column="cvss_v3_source", blank=True, null=True, max_length=255 + db_column="cvss_v3_source", + blank=True, + null=True, + max_length=255, + help_text="Organization or entity that has provided or published the CVSS v3 score for a given vulnerability.", ) cvss_v3_type = models.CharField( - db_column="cvss_v3_type", blank=True, null=True, max_length=255 + db_column="cvss_v3_type", + blank=True, + null=True, + max_length=255, + help_text="Type of CVVS v3 score. (Primary, Secondary)", ) cvss_v3_version = models.CharField( - db_column="cvss_v3_version", blank=True, null=True, max_length=255 + db_column="cvss_v3_version", + blank=True, + null=True, + max_length=255, + help_text="Version of the CVSS v3 score.", ) cvss_v3_vector_string = models.CharField( - db_column="cvss_v3_vector_string", blank=True, null=True, max_length=255 + db_column="cvss_v3_vector_string", + blank=True, + null=True, + max_length=255, + help_text="Textual representation of the specific metrics used to calculate the CVSS v3 score.", ) cvss_v3_base_score = models.CharField( - db_column="cvss_v3_base_score", blank=True, null=True, max_length=255 + db_column="cvss_v3_base_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the severity of the vulnerability.", ) cvss_v3_base_severity = models.CharField( - db_column="cvss_v3_base_severity", blank=True, null=True, max_length=255 + db_column="cvss_v3_base_severity", + blank=True, + null=True, + max_length=255, + help_text="Qualitative categorization of a vulnerability's Base Score that helps assess its overall risk level in a more human-readable way.", ) cvss_v3_exploitability_score = models.CharField( - db_column="cvss_v3_exploitability_score", blank=True, null=True, max_length=255 + db_column="cvss_v3_exploitability_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the exploitability of the vulnerability.", ) cvss_v3_impact_score = models.CharField( - db_column="cvss_v3_impact_score", blank=True, null=True, max_length=255 + db_column="cvss_v3_impact_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the potential impact of the vulnerability.", ) cvss_v4_source = models.CharField( - db_column="cvss_v4_source", blank=True, null=True, max_length=255 + db_column="cvss_v4_source", + blank=True, + null=True, + max_length=255, + help_text="Organization or entity that has provided or published the CVSS v4 score for a given vulnerability.", ) cvss_v4_type = models.CharField( - db_column="cvss_v4_type", blank=True, null=True, max_length=255 + db_column="cvss_v4_type", + blank=True, + null=True, + max_length=255, + help_text="Type of CVVS v4 score. (Primary, Secondary)", ) cvss_v4_version = models.CharField( - db_column="cvss_v4_version", blank=True, null=True, max_length=255 + db_column="cvss_v4_version", + blank=True, + null=True, + max_length=255, + help_text="Version of the CVSS v4 score.", ) cvss_v4_vector_string = models.CharField( - db_column="cvss_v4_vector_string", blank=True, null=True, max_length=255 + db_column="cvss_v4_vector_string", + blank=True, + null=True, + max_length=255, + help_text="Textual representation of the specific metrics used to calculate the CVSS v4 score.", ) cvss_v4_base_score = models.CharField( - db_column="cvss_v4_base_score", blank=True, null=True, max_length=255 + db_column="cvss_v4_base_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the severity of the vulnerability.", ) cvss_v4_base_severity = models.CharField( - db_column="cvss_v4_base_severity", blank=True, null=True, max_length=255 + db_column="cvss_v4_base_severity", + blank=True, + null=True, + max_length=255, + help_text="Qualitative categorization of a vulnerability's Base Score that helps assess its overall risk level in a more human-readable way.", ) cvss_v4_exploitability_score = models.CharField( - db_column="cvss_v4_exploitability_score", blank=True, null=True, max_length=255 + db_column="cvss_v4_exploitability_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the exploitability of the vulnerability.", ) cvss_v4_impact_score = models.CharField( - db_column="cvss_v4_impact_score", blank=True, null=True, max_length=255 + db_column="cvss_v4_impact_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the potential impact of the vulnerability.", + ) + weaknesses = models.TextField( + blank=True, + null=True, + help_text="Weaknesses (CWE) associated with the vulnerability.", + ) + references = models.TextField( + blank=True, + null=True, + help_text="URLs to references associated with the vulnerability.", ) - weaknesses = models.TextField(blank=True, null=True) - references = models.TextField(blank=True, null=True) dve_score = models.DecimalField( - max_digits=1000, decimal_places=1000, blank=True, null=True + max_digits=1000, + decimal_places=1000, + blank=True, + null=True, + help_text="CyberSixGill's Dynamic Vulnerability Exploit (DVE) Score, this state-of-the-art machine learning model automatically predicts the probability of a CVE being exploited.", ) - cpes = models.ManyToManyField(Cpe, related_name="cves", blank=True) + cpes = models.ManyToManyField( + Cpe, + related_name="cves", + blank=True, + help_text="Many to many relationship to list of affected Products (CPE).", + ) # tickets = models.ManyToManyField("Ticket", related_name='cves', blank=True) # vuln_scans = models.ManyToManyField("VulnScan", related_name='cves', blank=True) class Meta: """The Meta class for Cve.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cve" - # This will likely be handled via the many to many field - # class CveCpesCpe(models.Model): - # """The CveCpesCpe model.""" - - # cve_id = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column="cve_id") - # cpe_id = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column="cpe_id") - - # class Meta: - # """The Meta class for CveCpesCpe model.""" - - # db_table = "cve_cpes_cpe" - # managed = False # This ensures Django does not manage the table - # unique_together = (("cve", "cpe"),) # Unique constraint - - # This is crossfeeds domain model, which lines up better with the pe subdomain table - # class Domain(models.Model): - # """The Domain model.""" - - # id = models.UUIDField(primary_key=True) - # created_at = models.DateTimeField(db_column="created_at") - # updated_at = models.DateTimeField(db_column="updated_at") - # synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True) - # ip = models.CharField(max_length=255, blank=True, null=True) - # from_root_domain = models.CharField(db_column="from_root_domain", blank=True, null=True) - # subdomain_source = models.CharField( - # db_column="subdomain_source", max_length=255, blank=True, null=True - # ) - # ip_only = models.BooleanField(db_column="ip_only", default=False) - # reverse_name = models.CharField(db_column="reverse_name", max_length=512) - # name = models.CharField(max_length=512) - # screenshot = models.CharField(max_length=512, blank=True, null=True) - # country = models.CharField(max_length=255, blank=True, null=True) - # asn = models.CharField(max_length=255, blank=True, null=True) - # cloud_hosted = models.BooleanField(db_column="cloud_hosted", default=False) - # ssl = models.JSONField(blank=True, null=True) - # censys_certificates_results = models.JSONField( - # db_column="censys_certificates_results", default=dict - # ) - # trustymail_results = models.JSONField(db_column="trustymail_results", default=dict) - # discovered_by = models.ForeignKey( - # "Scan", - # on_delete=models.SET_NULL, - # db_column="discovered_by_id", - # blank=True, - # null=True, - # ) - # organization = models.ForeignKey( - # "Organization", on_delete=models.CASCADE, db_column="organization_id" - # ) - - # class Meta: - # """The meta class for Domain.""" - - # db_table = "domain" - # managed = False # This ensures Django does not manage the table - # unique_together = (("name", "organization"),) # Unique constraint - def save(self, *args, **kwargs): """Format the model before saving.""" self.name = self.name.lower() @@ -255,26 +322,59 @@ def save(self, *args, **kwargs): class Notification(models.Model): """The Notification model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for a notification object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Datetime the notification object was created.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Datetime the notification object was last updated in the database.", + ) start_datetime = models.DateTimeField( - db_column="start_datetime", blank=True, null=True + db_column="start_datetime", + blank=True, + null=True, + help_text="Datetime the notification should start being displayed on the cyhy dashboard.", + ) + end_datetime = models.DateTimeField( + db_column="end_datetime", + blank=True, + null=True, + help_text="Datetime the notification should stop being displayed on the cyhy dashboard.", ) - end_datetime = models.DateTimeField(db_column="end_datetime", blank=True, null=True) maintenance_type = models.CharField( - db_column="maintenance_type", blank=True, null=True, max_length=255 + db_column="maintenance_type", + blank=True, + null=True, + max_length=255, + help_text="Type of maintenance being done. (Major, Minor)", + ) + status = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="Status of the notification. (Active, Inactive)", ) - status = models.CharField(blank=True, null=True, max_length=255) updated_by = models.CharField( - db_column="updated_by", blank=True, null=True, max_length=255 + db_column="updated_by", + blank=True, + null=True, + max_length=255, + help_text="User who updated the notification", + ) + message = models.TextField( + blank=True, + null=True, + help_text="Message to be displayed on the cyhy dashboard.", ) - message = models.TextField(blank=True, null=True) class Meta: """The Meta class for Notification.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "notification" @@ -282,74 +382,251 @@ class Meta: class Organization(models.Model): """The Organization model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at", auto_now_add=True) - updated_at = models.DateTimeField(db_column="updated_at", auto_now=True) - acronym = models.CharField(unique=True, blank=True, null=True, max_length=255) - retired = models.BooleanField(default=False, null=True, blank=True) - name = models.CharField(max_length=255) + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a stakeholder Organization." + ) + created_at = models.DateTimeField( + db_column="created_at", + auto_now_add=True, + help_text="Date and time the organization object was created in the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + auto_now=True, + help_text="Last date and time the organization object was updated.", + ) + acronym = models.CharField( + unique=True, + blank=True, + null=True, + max_length=255, + help_text="Short name used to identify the organization. This should match ServiceNow and the cyhy mongo database org id.", + ) + retired = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean field to flag organizations that have been retired a", + ) + name = models.CharField(max_length=255, help_text="Full name of the organization") root_domains = ArrayField( models.TextField(blank=True, null=True), blank=True, null=True, db_column="root_domains", + help_text="List of root domains attributed to the organization", + ) + ip_blocks = models.TextField( + db_column="ip_blocks", + help_text="IP blocks attributed to or provided by a stakeholder.", + ) # This field type is a guess. + is_passive = models.BooleanField( + db_column="is_passive", + help_text="Boolean to flag if only passive data collection can be used on the stakeholder's assets.", ) - ip_blocks = models.TextField(db_column="ip_blocks") # This field type is a guess. - is_passive = models.BooleanField(db_column="is_passive") pending_domains = ArrayField( models.TextField(blank=True, null=True), blank=True, null=True, db_column="pending_domains", + help_text="List of domains that have not yet been run through the setup/enumeration process.", ) # This field type is a guess - date_pe_first_reported = models.DateTimeField(blank=True, null=True) - country = models.TextField(blank=True, null=True) - country_name = models.TextField(blank=True, null=True) - state = models.CharField(blank=True, null=True, max_length=255) + date_pe_first_reported = models.DateTimeField( + blank=True, + null=True, + help_text="Date that PE first delivered reports to this stakeholder", + ) + country = models.TextField( + blank=True, + null=True, + help_text="Abbreviation of the country the organization is based in.", + ) + country_name = models.TextField( + blank=True, + null=True, + help_text="Full name of the country the organization is based in.", + ) + state = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="Abbreviation of the US state the organization is based in.", + ) region_id = models.CharField( - db_column="region_id", blank=True, null=True, max_length=255 + db_column="region_id", + blank=True, + null=True, + max_length=255, + help_text="Region number that the organization is found in.", + ) + state_fips = models.IntegerField( + db_column="state_fips", + blank=True, + null=True, + help_text="Federal Information Processing Standards code for the US state where the organization is found.", ) - state_fips = models.IntegerField(db_column="state_fips", blank=True, null=True) state_name = models.CharField( - db_column="state_name", blank=True, null=True, max_length=255 - ) - county = models.TextField(blank=True, null=True) - county_fips = models.IntegerField(db_column="county_fips", blank=True, null=True) - type = models.CharField(blank=True, null=True, max_length=255) - pe_report_on = models.BooleanField(default=False, null=True, blank=True) - pe_premium = models.BooleanField(default=False, null=True, blank=True) - pe_demo = models.BooleanField(default=False, null=True, blank=True) - agency_type = models.TextField(blank=True, null=True) - is_parent = models.BooleanField(blank=True, null=True) - pe_run_scans = models.BooleanField(default=False, null=True, blank=True) - stakeholder = models.BooleanField(default=False, null=True, blank=True) - election = models.BooleanField(blank=True, null=True) - was_stakeholder = models.BooleanField(default=False, null=True, blank=True) - vs_stakeholder = models.BooleanField(default=False, null=True, blank=True) - pe_stakeholder = models.BooleanField(default=False, null=True, blank=True) - receives_cyhy_report = models.BooleanField(blank=True, null=True) - receives_bod_report = models.BooleanField(blank=True, null=True) - receives_cybex_report = models.BooleanField(blank=True, null=True) - init_stage = models.CharField(max_length=255, null=True, blank=True) - scheduler = models.CharField(max_length=255, null=True, blank=True) + db_column="state_name", + blank=True, + null=True, + max_length=255, + help_text="Full name of the US state the organization is based in.", + ) + county = models.TextField( + blank=True, + null=True, + help_text="Full name of the county the organization is found in.", + ) + county_fips = models.IntegerField( + db_column="county_fips", + blank=True, + null=True, + help_text="Federal Information Processing Standards code for the US county where the organization is found.", + ) + type = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="The type of organization, brought over from legacy crossfeed, but not sure if currently used.", + ) + pe_report_on = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if the organization receives PE reports.", + ) + pe_premium = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization receives a premium PE report.", + ) + pe_demo = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is in demo status for PE. This means that scans are run for the organization, but reports are not delivered.", + ) + agency_type = models.TextField( + blank=True, + null=True, + help_text="Type of organization pulled from the Cyhy mongo database (Federal, State, Local, Private).", + ) + is_parent = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if an organization has children organizations associated with it.", + ) + pe_run_scans = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean field to determine if pe scans should be run an organization's assets.", + ) + stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a cyhy stakeholder.", + ) + election = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization is an election entity.", + ) + was_stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a WAS customer.", + ) + vs_stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a VS customer.", + ) + pe_stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a PE customer.", + ) + receives_cyhy_report = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization receives a cyhy report.", + ) + receives_bod_report = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization receives a cyhy bod report.", + ) + receives_cybex_report = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization receives a cyhy cybex report.", + ) + init_stage = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="First scan run in the VS scan process.", + ) + scheduler = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Type of scheduler the VS scan uses when running VS scans.", + ) enrolled_in_vs_timestamp = models.DateTimeField( - db_column="enrolled_in_vs_timestamp", auto_now=True + db_column="enrolled_in_vs_timestamp", + auto_now=True, + help_text="Date the stakeholder enrolled in VS.", ) period_start_vs_timestamp = models.DateTimeField( - db_column="period_start_vs_timestamp", auto_now=True - ) - report_types = models.JSONField(null=True, blank=True, default=list) - scan_types = models.JSONField(null=True, blank=True, default=list) - scan_windows = models.JSONField(null=True, blank=True, default=list) - scan_limits = models.JSONField(null=True, blank=True, default=list) - password = models.TextField(blank=True, null=True) - cyhy_period_start = models.DateField(blank=True, null=True) + db_column="period_start_vs_timestamp", + auto_now=True, + help_text="Period start for the last report period VS ran.?????", + ) + report_types = models.JSONField( + null=True, + blank=True, + default=list, + help_text="List of types of CyHy reports the stakeholder receives ", + ) + scan_types = models.JSONField( + null=True, blank=True, default=list, help_text="Types of scans run by Cyhy." + ) + scan_windows = models.JSONField( + null=True, + blank=True, + default=list, + help_text="List of time windows when VS can scan a stakeholder's assets.", + ) + scan_limits = models.JSONField( + null=True, + blank=True, + default=list, + help_text="Limits placed on a VS scan by the stakeholder.", + ) + password = models.TextField( + blank=True, + null=True, + help_text="Encrypted password used to encrypt and decrypt reports sent to the stakeholder.", + ) + cyhy_period_start = models.DateField( + blank=True, + null=True, + help_text="Timestamp when scanning can begin for this organization.", + ) location = models.ForeignKey( "Location", related_name="organizations", on_delete=models.SET_NULL, null=True, blank=True, + help_text="Foreign Key linking to a related Location object.", ) # sectors = models.ManyToManyField("Sector", related_name='organizations', blank=True) covered in sectors table already # cidrs = models.ManyToManyField("Cidr", related_name='organizations', blank=True) covered in the cidr table already @@ -357,10 +634,20 @@ class Organization(models.Model): # hosts = models.ManyToManyField("Host", related_name='organizations', blank=True) covered in hosts table already # port_scans = models.ManyToManyField("PortScan", related_name='organizations', blank=True) parent = models.ForeignKey( - "self", models.DO_NOTHING, db_column="parent_id", blank=True, null=True + "self", + models.DO_NOTHING, + db_column="parent_id", + blank=True, + null=True, + help_text="Foreign Key linking to a related organization parent object.", ) created_by = models.ForeignKey( - "User", models.DO_NOTHING, db_column="created_by_id", blank=True, null=True + "User", + models.DO_NOTHING, + db_column="created_by_id", + blank=True, + null=True, + help_text="Foreign Key linking to the user who create the organization.", ) org_type = models.ForeignKey( "OrgType", @@ -368,12 +655,13 @@ class Organization(models.Model): db_column="org_type_id", blank=True, null=True, + help_text="Foreign Key to the related orgType object. ", ) class Meta: """The meta class for Organization.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "organization" @@ -381,18 +669,33 @@ class Meta: class OrganizationTag(models.Model): """The OrganizationTag model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - name = models.CharField(unique=True, max_length=255) + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for an Organization tag object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the organization tag was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the organization tag object was updated in the database.", + ) + name = models.CharField( + unique=True, + max_length=255, + help_text="The name of the tag used to link common organizations.", + ) organization = models.ManyToManyField( - "Organization", related_name="organization_tags", blank=True + "Organization", + related_name="organization_tags", + blank=True, + help_text="Many to many relationship to link a tag to many organizations", ) class Meta: """The Meta class for OrganizationTag.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "organization_tag" @@ -422,111 +725,61 @@ class Meta: class QueryResultCache(models.Model): """The QueryResultCache model.""" - id = models.UUIDField(primary_key=True) - identifier = models.CharField(blank=True, null=True, max_length=255) - time = models.BigIntegerField() - duration = models.IntegerField() - query = models.TextField() - result = models.TextField() + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for the query result object being cached.", + ) + identifier = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="Another identifier for the query being cached.", + ) + time = models.BigIntegerField( + help_text="Time the query was run against the database." + ) + duration = models.IntegerField(help_text="How long the query took to run.") + query = models.TextField( + help_text="The Query run against the database to be cached." + ) + result = models.TextField(help_text="Result from the query performed in Crossfeed.") class Meta: """The Meta class for QueryResultCache.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "query-result-cache" -class Question(models.Model): - """The Question model.""" +class Role(models.Model): + """The Role model.""" - id = models.UUIDField(primary_key=True) - name = models.CharField(max_length=255) - description = models.TextField(blank=True, null=True) - long_form = models.TextField(db_column="long_form") - number = models.CharField(max_length=255) - category = models.ForeignKey( - Category, models.DO_NOTHING, db_column="category_id", blank=True, null=True + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for the role object." ) - - class Meta: - """The Meta class for Question.""" - - db_table = "question" - app_label = "dmz_mini_dl" - managed = manage_db - unique_together = (("category", "number"),) - - -# Created via Many to Many field -# Question and Resource many-to-many -# class QuestionResourcesResource(models.Model): -# """The QuestionResourcesResource model.""" - -# question_id = models.ForeignKey( -# "Question", on_delete=models.CASCADE, db_column="questionId" -# ) -# resource_id = models.ForeignKey( -# "Resource", on_delete=models.CASCADE, db_column="resourceId" -# ) - -# class Meta: -# """The Meta class for QuestionResourcesResource.""" - -# db_table = "question_resources_resource" -# managed = False -# unique_together = (("question", "resource"),) - - -class Resource(models.Model): - """The Resource model.""" - - id = models.UUIDField(primary_key=True) - description = models.TextField() - name = models.CharField(max_length=255) - type = models.CharField(max_length=255) - url = models.TextField(unique=True) - questions = models.ManyToManyField(Question, related_name="resources", blank=True) - - class Meta: - """The Meta class for Resource.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "resource" - - -class Response(models.Model): - """The Response model.""" - - id = models.UUIDField(primary_key=True) - selection = models.TextField() - assessment = models.ForeignKey( - Assessment, models.DO_NOTHING, db_column="assessment_id", blank=True, null=True + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the role object was added to the database.", ) - question = models.ForeignKey( - Question, models.DO_NOTHING, db_column="question_id", blank=True, null=True + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the role object was updated in the database.", + ) + role = models.CharField( + max_length=255, + help_text="A role that a user can be assigned to, granting them specific access on the crossfeed platform.", + ) + approved = models.BooleanField( + help_text="A boolean flag to determine if the user has been approved to have the assigned role." ) - - class Meta: - """The Meta class for Resource.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "response" - unique_together = (("assessment_id", "question_id"),) - - -class Role(models.Model): - """The Role model.""" - - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - role = models.CharField(max_length=255) - approved = models.BooleanField() created_by = models.ForeignKey( - "User", models.DO_NOTHING, db_column="created_by_id", blank=True, null=True + "User", + models.DO_NOTHING, + db_column="created_by_id", + blank=True, + null=True, + help_text="Foreign key linking to the user who created the role in the database.", ) approved_by = models.ForeignKey( "User", @@ -535,6 +788,7 @@ class Role(models.Model): related_name="role_approved_by_id_set", blank=True, null=True, + help_text="Foreign key to the user who approved the role assignation.", ) user = models.ForeignKey( "User", @@ -543,6 +797,7 @@ class Role(models.Model): related_name="role_user_id_set", blank=True, null=True, + help_text="Foreign key to the user being assigned the role.", ) organization = models.ForeignKey( Organization, @@ -550,12 +805,13 @@ class Role(models.Model): db_column="organization_id", blank=True, null=True, + help_text="Foreign key to the organization the user is aligned to and whos data the user can access via their role.", ) class Meta: """The Meta class for Role.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "role" unique_together = (("user_id", "organization_id"),) @@ -564,26 +820,58 @@ class Meta: class SavedSearch(models.Model): """The SavedSearch model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - name = models.CharField(max_length=255) - search_term = models.CharField(db_column="search_term", max_length=255) - sort_direction = models.CharField(db_column="sort_direction", max_length=255) - sort_field = models.CharField(db_column="sort_field", max_length=255) - count = models.IntegerField() - filters = models.JSONField() - search_path = models.CharField(db_column="search_path", max_length=255) - create_vulnerabilities = models.BooleanField(db_column="create_vulnerabilities") - vulnerability_template = models.JSONField(db_column="vulnerability_template") + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for the Saved Search object" + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the saved search object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the saved search object was updated in the database.", + ) + name = models.CharField( + max_length=255, + help_text="User provided name of the saved search provided in the cyhy dashboard.", + ) + search_term = models.CharField( + db_column="search_term", + max_length=255, + help_text="The term being searched for in the cyhy dashboard.", + ) + sort_direction = models.CharField( + db_column="sort_direction", + max_length=255, + help_text="Direction of the sort (asc or desc).", + ) + sort_field = models.CharField( + db_column="sort_field", max_length=255, help_text="The field to sort based on." + ) + count = models.IntegerField( + help_text="Number of results returned when the search was run." + ) + filters = models.JSONField(help_text="Filters applied in the search.") + search_path = models.CharField( + db_column="search_path", + max_length=255, + help_text="Search path used to call create the search against the ORM.", + ) + # create_vulnerabilities = models.BooleanField(db_column="create_vulnerabilities", help_text="") # No longer used + # vulnerability_template = models.JSONField(db_column="vulnerability_template", help_text="") # No longer used created_by = models.ForeignKey( - "User", models.DO_NOTHING, db_column="created_by_id", blank=True, null=True + "User", + models.DO_NOTHING, + db_column="created_by_id", + blank=True, + null=True, + help_text="Foreign key linking to the user who created the saved search in the dashboard.", ) class Meta: """The Meta class for SavedSearch.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "saved_search" @@ -591,225 +879,348 @@ class Meta: class Scan(models.Model): """The Scan model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - name = models.CharField(max_length=255) - arguments = models.JSONField() - frequency = models.IntegerField() - last_run = models.DateTimeField(db_column="last_run", blank=True, null=True) - is_granular = models.BooleanField(db_column="is_granular") + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for a cyhy dashboard scan object.", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the scan object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the scan object was updated in the database.", + ) + name = models.CharField( + max_length=255, help_text="The name of the cyhy dashboard scan." + ) + arguments = models.JSONField( + help_text="A dictionary of arguments to pass to the scan." + ) + frequency = models.IntegerField( + help_text="How often the scan should run in seconds." + ) + last_run = models.DateTimeField( + db_column="last_run", + blank=True, + null=True, + help_text="Last day the scan was run.", + ) + is_granular = models.BooleanField( + db_column="is_granular", + help_text="A boolean flag to specify if the scan is granular. Granular scans are only run on specified organizations. Global scans cannot be granular scans.", + ) is_user_modifiable = models.BooleanField( - db_column="is_user_modifiable", blank=True, null=True + db_column="is_user_modifiable", + blank=True, + null=True, + help_text="Whether the scan is user-modifiable. User-modifiable scans are granular scans that can be viewed and toggled on/off by organization admins themselves.", + ) + is_single_scan = models.BooleanField( + db_column="is_single_scan", + help_text="A boolean to flag scans that should only be run once and not on a reoccuring basis.", + ) + manual_run_pending = models.BooleanField( + db_column="manual_run_pending", + help_text="A boolean to flag if a manually called scan is still waiting to be run.", ) - is_single_scan = models.BooleanField(db_column="is_single_scan") - manual_run_pending = models.BooleanField(db_column="manual_run_pending") created_by = models.ForeignKey( - "User", models.DO_NOTHING, db_column="created_by", blank=True, null=True + "User", + models.DO_NOTHING, + db_column="created_by", + blank=True, + null=True, + help_text="A foreign key linking to the user who created the scan.", ) organizations = models.ManyToManyField( - Organization, related_name="scans", blank=True + Organization, + related_name="scans", + blank=True, + help_text="A many to many relationship linking to all the organizations the scan should be run on.", ) organization_tags = models.ManyToManyField( - OrganizationTag, related_name="scans", blank=True + OrganizationTag, + related_name="scans", + blank=True, + help_text="A many to many relationship linking to all the organization tags that should be run on.", ) class Meta: """The Meta class for Scan.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "scan" -# Taken Care of via many to many field -# class ScanOrganizationsOrganization(models.Model): -# """The ScanOrganizationsOrganization model.""" - -# scan_id = models.OneToOneField( -# Scan, models.DO_NOTHING, db_column="scanId", primary_key=True -# ) # The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. -# organization_id = models.ForeignKey( -# Organization, models.DO_NOTHING, db_column="organizationId" -# ) - -# class Meta: -# """The Meta class for ScanOrganizationsOrganization.""" - -# managed = False -# db_table = "scan_organizations_organization" -# unique_together = (("scanId", "organizationId"),) - -# Completed via many to many -# class ScanTagsOrganizationTag(models.Model): -# """The ScanTagsOrganizationTag model.""" - -# scan_id = models.OneToOneField( -# Scan, models.DO_NOTHING, db_column="scanId", primary_key=True -# ) # The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. -# organization_tag_id = models.ForeignKey( -# OrganizationTag, models.DO_NOTHING, db_column="organizationTagId" -# ) - -# class Meta: -# """The Meta class for ScanTagsOrganizationTag.""" - -# managed = False -# db_table = "scan_tags_organization_tag" -# unique_together = (("scanId", "organizationTagId"),) - - class ScanTask(models.Model): """The ScanTask model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - status = models.TextField() - type = models.TextField() + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a scan task object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the scan task object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the scan task object was updated in the database.", + ) + status = models.TextField( + help_text="The scan task's status. ('created', 'queued', 'requested', 'started', 'failed','finished')" + ) + type = models.TextField(help_text="Type of scan task. ('fargate', 'lambda')") fargate_task_arn = models.TextField( - db_column="fargate_task_arn", blank=True, null=True - ) - input = models.TextField(blank=True, null=True) - output = models.TextField(blank=True, null=True) - requested_at = models.DateTimeField(db_column="requested_at", blank=True, null=True) - started_at = models.DateTimeField(db_column="started_at", blank=True, null=True) - finished_at = models.DateTimeField(db_column="finished_at", blank=True, null=True) - queued_at = models.DateTimeField(db_column="queued_at", blank=True, null=True) + db_column="fargate_task_arn", + blank=True, + null=True, + help_text="Unique identifier for the fargate container running the task.", + ) + input = models.TextField( + blank=True, + null=True, + help_text="All data necessary to run the scan task. (organizations, scan_id, scanName, scanTaskId, isSingleScan)", + ) + output = models.TextField( + blank=True, + null=True, + help_text="All the data returned from the scan task, dependant on the type of scan.", + ) + requested_at = models.DateTimeField( + db_column="requested_at", + blank=True, + null=True, + help_text="Date and time the scan task was requested.", + ) + started_at = models.DateTimeField( + db_column="started_at", + blank=True, + null=True, + help_text="Date and time the scan task was started.", + ) + finished_at = models.DateTimeField( + db_column="finished_at", + blank=True, + null=True, + help_text="Date and time the scan task finished.", + ) + queued_at = models.DateTimeField( + db_column="queued_at", + blank=True, + null=True, + help_text="Date and time the scan task was added to the queue.", + ) organization = models.ForeignKey( Organization, models.DO_NOTHING, db_column="organization_id", blank=True, null=True, + help_text="Foreign key to the organization instance the scan is being run on if it is a single scan.", ) scan = models.ForeignKey( - Scan, models.DO_NOTHING, db_column="scan_id", blank=True, null=True + Scan, + models.DO_NOTHING, + db_column="scan_id", + blank=True, + null=True, + help_text="Foreign key to the scan the scan task was based off of.", ) organization_tags = models.ManyToManyField( - OrganizationTag, related_name="scan_tasks", blank=True + OrganizationTag, + related_name="scan_tasks", + blank=True, + help_text="List of organization tags that the scan task is running on.", ) class Meta: """The Meta class for ScanTask.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "scan_task" -# Managed via many to many -# class ScanTaskOrganizationsOrganization(models.Model): -# """The ScanTaskOrganizationsOrganization model.""" - -# scan_task_id = models.OneToOneField( -# ScanTask, models.DO_NOTHING, db_column="scanTaskId", primary_key=True -# ) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. -# organization_id = models.ForeignKey( -# Organization, models.DO_NOTHING, db_column="organizationId" -# ) - -# class Meta: -# """The Meta class for ScanTaskOrganizationsOrganization.""" - -# managed = False -# db_table = "scan_task_organizations_organization" -# unique_together = (("scanTaskId", "organizationId"),) - - class Service(models.Model): """The Service model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - service_source = models.TextField(db_column="service_source", blank=True, null=True) - port = models.IntegerField() - service = models.TextField(blank=True, null=True) - last_seen = models.DateTimeField(db_column="last_seen", blank=True, null=True) - banner = models.TextField(blank=True, null=True) - products = models.JSONField() - censys_metadata = models.JSONField(db_column="censys_metadata") - censys_ipv4_results = models.JSONField(db_column="censys_ipv4_results") - intrigue_ident_results = models.JSONField(db_column="intrigue_ident_results") - shodan_results = models.JSONField(db_column="shodan_results") - wappalyzer_results = models.JSONField(db_column="wappalyzer_results") + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for a web service running on a stakeholders attack surface.", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the service object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the service object was updated in the database.", + ) + service_source = models.TextField( + db_column="service_source", + blank=True, + null=True, + help_text="The source of the service, which scan identified the service.", + ) + port = models.IntegerField(help_text="The port the service is running on.") + service = models.TextField(blank=True, null=True, help_text="Name of the service.") + last_seen = models.DateTimeField( + db_column="last_seen", + blank=True, + null=True, + help_text="Late date the service was seen running on the asset.", + ) + banner = models.TextField( + blank=True, + null=True, + help_text="Text that is automatically sent back to a client when they connect to the service.", + ) + products = models.JSONField(help_text="Products identified running on the port.") + censys_metadata = models.JSONField( + db_column="censys_metadata", + help_text="Metadata provided from the Censys scan of the service.", + ) + censys_ipv4_results = models.JSONField( + db_column="censys_ipv4_results", + help_text="IPv4 results provided from the Censys scan of the service.", + ) + intrigue_ident_results = models.JSONField( + db_column="intrigue_ident_results", + help_text="Additional details about the service provided by Intrigue scans.", + ) + shodan_results = models.JSONField( + db_column="shodan_results", + help_text="Details about the service identified through the Shodan scan.", + ) + wappalyzer_results = models.JSONField( + db_column="wappalyzer_results", + help_text="Details about the service identified by the wappalyzer scan.", + ) domain = models.ForeignKey( - "SubDomains", models.DO_NOTHING, db_column="domain_id", blank=True, null=True + "SubDomains", + models.DO_NOTHING, + db_column="domain_id", + blank=True, + null=True, + help_text="Foreign key relationship to the domain the service is running on.", ) discovered_by = models.ForeignKey( - Scan, models.DO_NOTHING, db_column="discovered_by_id", blank=True, null=True + Scan, + models.DO_NOTHING, + db_column="discovered_by_id", + blank=True, + null=True, + help_text="Foreign key to the scan that discovered the service.", ) class Meta: """The Meta class for Service.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "service" unique_together = (("port", "domain"),) -# ????Not sure if this is necessary since we are removing typeorm????? -# class TypeormMetadata(models.Model): -# """The TypeormMetadata model.""" - -# type = models.CharField() -# database = models.CharField(blank=True, null=True) -# schema = models.CharField(blank=True, null=True) -# table = models.CharField(blank=True, null=True) -# name = models.CharField(blank=True, null=True) -# value = models.TextField(blank=True, null=True) - -# class Meta: -# """The Meta class for TypeormMetadata.""" - -# managed = False -# db_table = "typeorm_metadata" - - class User(models.Model): """The User model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a user object." + ) cognito_id = models.CharField( - db_column="cognitoId", unique=True, blank=True, null=True, max_length=255 + db_column="cognitoId", + unique=True, + blank=True, + null=True, + max_length=255, + help_text="Identifier for the user in the cognito system. This is necessary to log into the cyhy dashboard application.", ) login_gov_id = models.CharField( - db_column="login_gov_id", unique=True, blank=True, null=True, max_length=255 - ) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - first_name = models.CharField(db_column="first_name", max_length=255) - last_name = models.CharField(db_column="last_name", max_length=255) - full_name = models.CharField(db_column="full_name", max_length=255) - email = models.CharField(unique=True, max_length=255) - invite_pending = models.BooleanField(db_column="invite_pending") + db_column="login_gov_id", + unique=True, + blank=True, + null=True, + max_length=255, + help_text="Identifier for the user in the login.gov system. This is also used to log in to the cyhy dashboard.", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the user object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the user object was updated in the database.", + ) + first_name = models.CharField( + db_column="first_name", max_length=255, help_text="First name of the user." + ) + last_name = models.CharField( + db_column="last_name", max_length=255, help_text="Last name of the user." + ) + full_name = models.CharField( + db_column="full_name", max_length=255, help_text="Full name of the user." + ) + email = models.CharField( + unique=True, max_length=255, help_text="User's email address." + ) + invite_pending = models.BooleanField( + db_column="invite_pending", + help_text="A boolean field flagging if the user's invite is pending.", + ) login_blocked_by_maintenance = models.BooleanField( - db_column="login_blocked_by_maintenance" + db_column="login_blocked_by_maintenance", + help_text="A boolean flag identifying whether the user is blocked by maintenance to login", ) date_accepted_terms = models.DateTimeField( - db_column="date_accepted_terms", blank=True, null=True + db_column="date_accepted_terms", + blank=True, + null=True, + help_text="Date the user accepted the cyhy dashboard terms of service.", ) accepted_terms_version = models.TextField( - db_column="accepted_terms_version", blank=True, null=True + db_column="accepted_terms_version", + blank=True, + null=True, + help_text="The version of the the terms of service the user accepted.", ) last_logged_in = models.DateTimeField( - db_column="last_logged_in", blank=True, null=True + db_column="last_logged_in", + blank=True, + null=True, + help_text="Datetime the last time the user logged in.", + ) + user_type = models.TextField( + db_column="user_type", + help_text="The type of user. This determines what parts of the cyhy dashboard can view and what data he is permitted to see.", ) - user_type = models.TextField(db_column="user_type") region_id = models.CharField( - db_column="region_id", blank=True, null=True, max_length=255 + db_column="region_id", + blank=True, + null=True, + max_length=255, + help_text="What region the user belongs to.", + ) + state = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="The state the user resides in.", ) - state = models.CharField(blank=True, null=True, max_length=255) okta_id = models.CharField( - db_column="okta_id", unique=True, blank=True, null=True, max_length=255 + db_column="okta_id", + unique=True, + blank=True, + null=True, + max_length=255, + help_text="The Okta id associated with the user.", ) class Meta: """The Meta class for User.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "user" @@ -817,38 +1228,112 @@ class Meta: class Vulnerability(models.Model): """The Vulnerability model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - last_seen = models.DateTimeField(db_column="last_seen", blank=True, null=True) - title = models.TextField() - cve = models.TextField(blank=True, null=True) - cwe = models.TextField(blank=True, null=True) - cpe = models.TextField(blank=True, null=True) - description = models.TextField() - references = models.JSONField() - cvss = models.DecimalField(max_digits=100, decimal_places=5, blank=True, null=True) - severity = models.TextField(blank=True, null=True) - needs_population = models.BooleanField(db_column="needs_population") - state = models.TextField() - substate = models.TextField() - source = models.TextField() - notes = models.TextField() - actions = models.JSONField() - structured_data = models.JSONField(db_column="structured_data") - is_kev = models.BooleanField(db_column="is_kev", blank=True, null=True) - kev_results = models.JSONField(db_column="kev_results", blank=True, null=True) + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for a vulnerability object found in the cyhy dashboard", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the vulnerability object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the vulnerability object was updated in the database.", + ) + last_seen = models.DateTimeField( + db_column="last_seen", + blank=True, + null=True, + help_text="Last date the vulnerability was seen.", + ) + title = models.TextField(help_text="The name or title of the vulnerability.") + cve = models.TextField( + blank=True, + null=True, + help_text="CVE (Common Vulnerabilities and Exposures) id for the vulnerability.", + ) + cwe = models.TextField( + blank=True, + null=True, + help_text="Common Weakness Enumeration (CWE) id for the weakness or vulnerability.", + ) + cpe = models.TextField( + blank=True, + null=True, + help_text="Common Platform Enumeration (CPE) id for the product the vulnerability was found on.", + ) + description = models.TextField( + help_text="Human readable description of the vulnerability if available." + ) + references = models.JSONField( + help_text="Additional links to references and sources associates with the vulnerability." + ) + cvss = models.DecimalField( + max_digits=100, + decimal_places=5, + blank=True, + null=True, + help_text="CVSS (Common Vulnerability Scoring System) is the score reperesenting the severity of the vulnerability from 0 (None) to 10 (Critical)", + ) + severity = models.TextField( + blank=True, + null=True, + help_text="The severity level of the vulnerability determined by the cvss score. (None, Low, Medium, High, Critical)", + ) + needs_population = models.BooleanField( + db_column="needs_population", + help_text="A boolean field to flag vulnerabilities that need to be populated additional findings.", + ) + state = models.TextField( + help_text="The state the vulnerability is in, as of the last scan (Open, Closed)" + ) + substate = models.TextField( + help_text="Substate of the vulnerability ('unconfirmed', 'exploitable', 'false-positive', 'accepted-risk', 'remediated')" + ) + source = models.TextField(help_text="The scan that identified the vulnerability.") + notes = models.TextField( + help_text="Notes about the vulnerability, provided by the user of the cyhy dashboard." + ) + actions = models.JSONField( + help_text="A list of state changes of the vulnerability, tracking its status from intially created to closed." + ) + structured_data = models.JSONField( + db_column="structured_data", + help_text="Any additional data that does not fit into the vulnerability table pertinent to the end user.", + ) + is_kev = models.BooleanField( + db_column="is_kev", + blank=True, + null=True, + help_text="A boolean field to flag if a vulnerability has been on the CISA Known Exploited Vulnerability (KEV) list.", + ) + kev_results = models.JSONField( + db_column="kev_results", + blank=True, + null=True, + help_text="The CISA provided KEV information assocaited with KEV vulnerabilities.", + ) domain = models.ForeignKey( - "SubDomains", models.DO_NOTHING, db_column="domain_id", blank=True, null=True + "SubDomains", + models.DO_NOTHING, + db_column="domain_id", + blank=True, + null=True, + help_text="Foreign key relationship to the domain the vulnerability was found on.", ) service = models.ForeignKey( - Service, models.DO_NOTHING, db_column="service_id", blank=True, null=True + Service, + models.DO_NOTHING, + db_column="service_id", + blank=True, + null=True, + help_text="Foreign key relationship to the service the vulnerability was found on.", ) class Meta: """The Meta class for Vulnerability.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "vulnerability" unique_together = (("domain", "title"),) @@ -857,33 +1342,69 @@ class Meta: class Webpage(models.Model): """The Webpage model.""" - id = models.UUIDField(primary_key=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - synced_at = models.DateTimeField(db_column="synced_at", blank=True, null=True) - last_seen = models.DateTimeField(db_column="last_seen", blank=True, null=True) - s3key = models.TextField(db_column="s3Key", blank=True, null=True) - url = models.TextField() - status = models.DecimalField(max_digits=100, decimal_places=5) + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for the webpage object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the webpage object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the webpage object was updated in the database.", + ) + synced_at = models.DateTimeField( + db_column="synced_at", + blank=True, + null=True, + help_text="When this model was last synced with Elasticsearch.", + ) + last_seen = models.DateTimeField( + db_column="last_seen", + blank=True, + null=True, + help_text="Last time the webpage was seen.", + ) + s3key = models.TextField( + db_column="s3Key", + blank=True, + null=True, + help_text="The AWS S3 key that corresponds to this webpage's contents.", + ) + url = models.TextField(help_text="URL to the webpage.") + status = models.DecimalField( + max_digits=100, decimal_places=5, help_text="The status of the HTTP response." + ) response_size = models.DecimalField( db_column="response_size", max_digits=100, decimal_places=5, blank=True, null=True, + help_text="The size of the url response.", ) - headers = models.JSONField() + headers = models.JSONField(help_text="The header returned from the url response.") domain = models.ForeignKey( - "SubDomains", models.DO_NOTHING, db_column="domain_id", blank=True, null=True + "SubDomains", + models.DO_NOTHING, + db_column="domain_id", + blank=True, + null=True, + help_text="The domain associated with the webpage.", ) discovered_by = models.ForeignKey( - Scan, models.DO_NOTHING, db_column="discovered_by_id", blank=True, null=True + Scan, + models.DO_NOTHING, + db_column="discovered_by_id", + blank=True, + null=True, + help_text="The scan that discovered the webpage.", ) class Meta: """The Meta class for Webpage.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "webpage" unique_together = (("url", "domain"),) @@ -893,8 +1414,17 @@ class Meta: class TicketEvent(models.Model): """The TicketEvent model.""" - id = models.UUIDField(primary_key=True, editable=False) - reference = models.CharField(max_length=255, null=True, blank=True) + id = models.UUIDField( + primary_key=True, + editable=False, + help_text="Unique id for a ticket event object in the database.", + ) + reference = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="The identifier for the vulnerability scan related to the event", + ) vuln_scan = models.ForeignKey( "VulnScan", on_delete=models.CASCADE, @@ -902,11 +1432,26 @@ class TicketEvent(models.Model): null=True, blank=True, related_name="ticket_events", + help_text="A foreign key relationship to the Vuln scan related to the event.", + ) + action = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Event action type. (OPENED, VERIFIED, CHANGED, CLOSED, REOPENED, UNVERIFIED)", + ) + reason = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Short description of the event", + ) + event_timestamp = models.DateTimeField( + null=True, blank=True, help_text="Timestamp indicating when the event occurred" + ) + delta = models.JSONField( + default=list, help_text="List of what changed; only applies to 'CHANGED' events" ) - action = models.CharField(max_length=255, null=True, blank=True) - reason = models.CharField(max_length=255, null=True, blank=True) - event_timestamp = models.DateTimeField(null=True, blank=True) - delta = models.JSONField(default=list) ticket = models.ForeignKey( "Ticket", on_delete=models.CASCADE, @@ -914,12 +1459,13 @@ class TicketEvent(models.Model): null=True, blank=True, related_name="ticket_events", + help_text="Foreign key relationship to the ticket the event references.", ) class Meta: """The Meta class for TicketEvent.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "ticket_event" unique_together = ("event_timestamp", "ticket", "action") @@ -928,119 +1474,374 @@ class Meta: class VulnScan(models.Model): """The VS Vuln Scan model.""" - id = models.CharField(max_length=255, primary_key=True) - cert_id = models.CharField(max_length=255, blank=True, null=True) - cpe = models.CharField(max_length=255, blank=True, null=True) - cve_string = models.CharField(max_length=255, blank=True, null=True) - cve = models.ForeignKey( - Cve, related_name="vuln_scans", on_delete=models.CASCADE, blank=True, null=True - ) - cvss_base_score = models.CharField(max_length=255, blank=True, null=True) - cvss_temporal_score = models.CharField(max_length=255, blank=True, null=True) - cvss_temporal_vector = models.CharField(max_length=255, blank=True, null=True) - cvss_vector = models.CharField(max_length=255, blank=True, null=True) - description = models.CharField(max_length=255, blank=True, null=True) - exploit_available = models.CharField(max_length=255, blank=True, null=True) - exploitability_ease = models.CharField(max_length=255, blank=True, null=True) - ip_string = models.CharField(max_length=255, blank=True, null=True) - ip = models.ForeignKey( - "Ip", related_name="vuln_scans", on_delete=models.CASCADE, blank=True, null=True + id = models.CharField( + max_length=255, + primary_key=True, + help_text="Unique identifier for the webpage object.", ) - latest = models.BooleanField(default=False) - owner = models.CharField(max_length=255, blank=True, null=True) - osvdb_id = models.CharField(max_length=255, blank=True, null=True) - organization = models.ForeignKey( - Organization, + cert_id = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Id to look up a vulnerability int the CERT Vulnerability Notes Database. https://www.kb.cert.org/vuls/", + ) + cpe = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Common Platform Enumeration (CPE) id for the product the vulnerability was found on.", + ) + cve_string = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="CVE (Common Vulnerabilities and Exposures) id for the vulnerability.", + ) + cve = models.ForeignKey( + Cve, + related_name="vuln_scans", + on_delete=models.CASCADE, + blank=True, + null=True, + help_text="Foreign key relationship to the related CVE object.", + ) + cvss_base_score = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Numerical value that measures the severity of a vulnerability using the Common Vulnerability Scoring System (CVSS)", + ) + cvss_temporal_score = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Score representing a vulnerabilities urgency at specific points in time.", + ) + cvss_temporal_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the metric values used to determine the temporal score.", + ) + cvss_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the set of CVSS metrics.", + ) + description = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Description of the vulnerability, according to the vulnerability scanner.", + ) + exploit_available = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A boolean field flagging whether or not an exploit is available, according to the vulnerability scanner.", + ) + exploitability_ease = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Ease of exploitation, according to the vulnerability scanner.", + ) + ip_string = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="IPv4 or IPv6 address where the vulnerability was identified.", + ) + ip = models.ForeignKey( + "Ip", + related_name="vuln_scans", + on_delete=models.CASCADE, + blank=True, + null=True, + help_text="Foreign key relationship to the related IP object.", + ) + latest = models.BooleanField( + default=False, + help_text="A boolean field flagging if this is the latest vulnerability scan of this port/protocol/host.", + ) + owner = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Acronym of the organization that claims the IP address associated with this vulnerability scan.", + ) + osvdb_id = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Open Source Vulnerability Database identifier for the detected vulnerability.", + ) + organization = models.ForeignKey( + Organization, related_name="vuln_scans", on_delete=models.CASCADE, blank=True, null=True, + help_text="Foreign key relationship linking to the related Organization object.", + ) + patch_publication_timestamp = models.DateTimeField( + blank=True, + null=True, + help_text="Date when a patch was published for this vulnerability", + ) + cisa_known_exploited = models.DateTimeField(blank=True, null=True, help_text="????") + port = models.IntegerField( + blank=True, + null=True, + help_text="Number of the port that was vulnerability scanned", + ) + port_protocol = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Protocol for the vulnerable port in this scan ('tcp' or 'udp')", + ) + risk_factor = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Risk factor of the detected vulnerability according to the vulnerability scanner", + ) + script_version = models.CharField( + max_length=255, blank=True, null=True, help_text="Script version string" + ) + see_also = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Additional reference(s) for this vulnerability provided by the vulnerability scanner", + ) + service = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Service detected at the vulnerable port in this scan", + ) + severity = models.IntegerField( + blank=True, + null=True, + help_text="CVSS v2.0 severity rating from the vulnerability scanner.", + ) + solution = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Solution to mitigate the detected vulnerability, according to the vulnerability scanner", + ) + source = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Source of the vulnerability scan (e.g. 'nessus').", + ) + synopsis = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Brief overview of the vulnerability.", + ) + vuln_detection_timestamp = models.DateTimeField( + blank=True, + null=True, + help_text="Timestamp indicating when the vulnerability was detected.", + ) + vuln_publication_timestamp = models.DateTimeField( + blank=True, null=True, help_text="Vulnerability publication date." + ) + xref = models.CharField( + max_length=255, blank=True, null=True, help_text="External reference." + ) + cwe = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Common Weakness Enumeration (CWE) id for the weakness or vulnerability.", + ) + bid = models.CharField( + max_length=255, blank=True, null=True, help_text="Bugtraq ID" + ) + exploited_by_malware = models.BooleanField( + default=False, + help_text="A boolean field to flag if the vuln type has been exploited by a known malware.", + ) + thorough_tests = models.BooleanField( + default=False, + help_text="Boolean field to flag if more thorough tests have been run on the vulnerability for confirmation.", + ) + cvss_score_rationale = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Rationale for the cvss score given to the vulnerability.", + ) + cvss_score_source = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="The source that determined the cvss score for this vulnerability.", ) - patch_publication_timestamp = models.DateTimeField(blank=True, null=True) - cisa_known_exploited = models.DateTimeField(blank=True, null=True) - port = models.IntegerField(blank=True, null=True) - port_protocol = models.CharField(max_length=255, blank=True, null=True) - risk_factor = models.CharField(max_length=255, blank=True, null=True) - script_version = models.CharField(max_length=255, blank=True, null=True) - see_also = models.CharField(max_length=255, blank=True, null=True) - service = models.CharField(max_length=255, blank=True, null=True) - severity = models.IntegerField(blank=True, null=True) - solution = models.CharField(max_length=255, blank=True, null=True) - source = models.CharField(max_length=255, blank=True, null=True) - synopsis = models.CharField(max_length=255, blank=True, null=True) - vuln_detection_timestamp = models.DateTimeField(blank=True, null=True) - vuln_publication_timestamp = models.DateTimeField(blank=True, null=True) - xref = models.CharField(max_length=255, blank=True, null=True) - cwe = models.CharField(max_length=255, blank=True, null=True) - bid = models.CharField(max_length=255, blank=True, null=True) - exploited_by_malware = models.BooleanField(default=False) - thorough_tests = models.BooleanField(default=False) - cvss_score_rationale = models.CharField(max_length=255, blank=True, null=True) - cvss_score_source = models.CharField(max_length=255, blank=True, null=True) cvss3_base_score = models.DecimalField( - max_digits=5, decimal_places=2, blank=True, null=True + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="CVSS version 3 base score.", + ) + cvss3_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the set of CVSS version 3 metrics.", + ) + cvss3_temporal_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the metric values used to determine the temporal score.", ) - cvss3_vector = models.CharField(max_length=255, blank=True, null=True) - cvss3_temporal_vector = models.CharField(max_length=255, blank=True, null=True) cvss3_temporal_score = models.DecimalField( - max_digits=5, decimal_places=2, blank=True, null=True - ) - asset_inventory = models.BooleanField(default=False) - plugin_id = models.CharField(max_length=255, blank=True, null=True) - plugin_modification_date = models.DateTimeField(blank=True, null=True) - plugin_publication_date = models.DateTimeField(blank=True, null=True) - plugin_name = models.CharField(max_length=255, blank=True, null=True) - plugin_type = models.CharField(max_length=255, blank=True, null=True) - plugin_family = models.CharField(max_length=255, blank=True, null=True) - f_name = models.CharField(max_length=255, blank=True, null=True) - cisco_bug_id = models.CharField(max_length=255, blank=True, null=True) - cisco_sa = models.CharField(max_length=255, blank=True, null=True) - plugin_output = models.TextField(blank=True, null=True) + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="Score representing a vulnerabilities urgency at specific points in time.", + ) + asset_inventory = models.BooleanField(default=False, help_text="????") + plugin_id = models.CharField( + max_length=255, blank=True, null=True, help_text="ID of the plugin." + ) + plugin_modification_date = models.DateTimeField( + blank=True, + null=True, + help_text="Latest modification date of the vulnerability scanner plugin that detected this vulnerability.", + ) + plugin_publication_date = models.DateTimeField( + blank=True, + null=True, + help_text="Publication date of the vulnerability scanner plugin that detected this vulnerability.", + ) + plugin_name = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Name of the vulnerability scanner plugin that detected this vulnerability.", + ) + plugin_type = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Vulnerability scanner plugin type.", + ) + plugin_family = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Family of the plugin run by the vulnerability scanner that detected this vulnerability.", + ) + f_name = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Filename of the vulnerability scanner plugin that detected this vulnerability.", + ) + cisco_bug_id = models.CharField( + max_length=255, blank=True, null=True, help_text="??????" + ) + cisco_sa = models.CharField( + max_length=255, blank=True, null=True, help_text="??????" + ) + plugin_output = models.TextField( + blank=True, + null=True, + help_text="Plugin-specific output from the vulnerability scanner", + ) # snapshots = models.ManyToManyField(Snapshot, related_name='vuln_scans') # ticket_events = models.ManyToManyField(TicketEvent, related_name='vuln_scans') - other_findings = models.JSONField(default=dict, blank=True) - - -class Meta: - """The Meta class for VulnScan.""" + other_findings = models.JSONField( + default=dict, + blank=True, + help_text="Additional data collected by the VS vuln scan that is not commonly seen.", + ) + + class Meta: + """The Meta class for VulnScan.""" - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "vuln_scan" + app_label = app_label_name + managed = manage_db + db_table = "vuln_scan" class Cidr(models.Model): """The Cidr Model.""" - id = models.UUIDField(primary_key=True, editable=False) - created_date = models.DateTimeField(auto_now_add=True) + id = models.UUIDField( + primary_key=True, + editable=False, + help_text="Unique idenifier for the Cidr object.", + ) + created_date = models.DateTimeField( + auto_now_add=True, help_text="Date the cidr object was added to the database." + ) network = InetAddressField( - null=True, blank=True, unique=True + null=True, blank=True, unique=True, help_text="The cidr block" ) # models.TextField() # This field type is a guess. - start_ip = InetAddressField(null=True, blank=True) - end_ip = InetAddressField(null=True, blank=True) - retired = models.BooleanField(null=True, blank=True) - updated_at = models.DateTimeField(auto_now=True) - insert_alert = models.TextField(blank=True, null=True) - first_seen = models.DateField(blank=True, null=True) - last_seen = models.DateField(blank=True, null=True) - current = models.BooleanField(blank=True, null=True) + start_ip = InetAddressField( + null=True, blank=True, help_text="The first IP address in the cidr block." + ) + end_ip = InetAddressField( + null=True, blank=True, help_text="The last IP address in the cidr block." + ) + retired = models.BooleanField( + null=True, + blank=True, + help_text="A boolean field flagging if the cidr has been retired.", + ) + updated_at = models.DateTimeField( + auto_now=True, + help_text="The last time the cidr object was updated in the database.", + ) + insert_alert = models.TextField( + blank=True, + null=True, + help_text="An alert message specifying any conflicts when inserting the cidr into the database.", + ) + first_seen = models.DateField( + blank=True, null=True, help_text="First time the cidr was seen." + ) + last_seen = models.DateField( + blank=True, null=True, help_text="Last time the cidr was seen." + ) + current = models.BooleanField( + blank=True, + null=True, + help_text="A boolean field flagging if the cidr is current. If it is False it should not be run through any scans.", + ) data_source = models.ForeignKey( "DataSource", on_delete=models.CASCADE, db_column="data_source_uid", blank=True, null=True, + help_text="Foreign key relationship to the data source that inserted the cidr object.", ) organizations = models.ManyToManyField( - Organization, related_name="cidrs", blank=True + Organization, + related_name="cidrs", + blank=True, + help_text="Foreign key relationship to the organization that owns the cidr object.", ) class Meta: """The Meta class for Cidr.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cidr" indexes = [models.Index(fields=["network"])] @@ -1049,21 +1850,54 @@ class Meta: class Location(models.Model): """The Location model.""" - id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) - name = models.CharField(max_length=255, null=True, blank=True) - country_abrv = models.CharField(max_length=255, null=True, blank=True) - country = models.CharField(max_length=255, null=True, blank=True) - county = models.CharField(max_length=255, null=True, blank=True) - county_fips = models.CharField(max_length=255, null=True, blank=True) - gnis_id = models.CharField(max_length=255, null=True, blank=True, unique=True) - state_abrv = models.CharField(max_length=255, null=True, blank=True) - state_fips = models.CharField(max_length=255, null=True, blank=True) - state = models.CharField(max_length=255, null=True, blank=True) + id = models.UUIDField( + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text="Unique identifier for a location object.", + ) + name = models.CharField( + max_length=255, null=True, blank=True, help_text="Name of the location." + ) + country_abrv = models.CharField( + max_length=255, null=True, blank=True, help_text="Country abbreviation." + ) + country = models.CharField( + max_length=255, null=True, blank=True, help_text="Full name of the country." + ) + county = models.CharField( + max_length=255, null=True, blank=True, help_text="Name of the county." + ) + county_fips = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Federal Information Processing Standards code for the US county where the organization is found.", + ) + gnis_id = models.CharField( + max_length=255, + null=True, + blank=True, + unique=True, + help_text="(Geographic Names Information System ID) is a unique identifier assigned to geographic features in the GNIS database.", + ) + state_abrv = models.CharField( + max_length=255, null=True, blank=True, help_text="State abbreviation." + ) + state_fips = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Federal Information Processing Standards code for the US state where the organization is found.", + ) + state = models.CharField( + max_length=255, null=True, blank=True, help_text="Full name of the state." + ) class Meta: """The Meta class for Location.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "location" indexes = [models.Index(fields=["gnis_id"])] @@ -1072,19 +1906,39 @@ class Meta: class Sector(models.Model): """The Sector model.""" - id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) - name = models.CharField(max_length=255, null=True, blank=True) - acronym = models.CharField(max_length=255, null=True, blank=True, unique=True) - retired = models.BooleanField(null=True, blank=True) + id = models.UUIDField( + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text="Unique identifier for a sector object in the database.", + ) + name = models.CharField( + max_length=255, null=True, blank=True, help_text="The name of the sector." + ) + acronym = models.CharField( + max_length=255, + null=True, + blank=True, + unique=True, + help_text="The short name of the sector.", + ) + retired = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field flagging if the sector has been retired.", + ) organizations = models.ManyToManyField( - Organization, related_name="sectors", blank=True + Organization, + related_name="sectors", + blank=True, + help_text="Many to many relationship between sectors and organizations.", ) class Meta: """The Meta class for Sector.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "sector" indexes = [models.Index(fields=["acronym"])] @@ -1093,42 +1947,119 @@ class Meta: class Host(models.Model): """The Host model.""" - id = models.CharField(max_length=255, primary_key=True) - ip_string = models.CharField(max_length=255, null=True, blank=True) + id = models.CharField( + max_length=255, + primary_key=True, + help_text="Unique identifier for a host object in the database.", + ) + ip_string = models.CharField( + max_length=255, null=True, blank=True, help_text="The IP address of the host." + ) ip = models.ForeignKey( - "Ip", related_name="hosts", on_delete=models.SET_NULL, null=True, blank=True - ) - updated_timestamp = models.DateTimeField(null=True, blank=True) - latest_netscan_1_timestamp = models.DateTimeField(null=True, blank=True) - latest_netscan_2_timestamp = models.DateTimeField(null=True, blank=True) - latest_vulnscan_timestamp = models.DateTimeField(null=True, blank=True) - latest_portscan_timestamp = models.DateTimeField(null=True, blank=True) - latest_scan_completion_timestamp = models.DateTimeField(null=True, blank=True) + "Ip", + related_name="hosts", + on_delete=models.SET_NULL, + null=True, + blank=True, + help_text="Foreign key relationship to the related model.", + ) + updated_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp of the last time the host object was updated.", + ) + latest_netscan_1_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the NETSCAN1.", + ) + latest_netscan_2_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the NETSCAN2.", + ) + latest_vulnscan_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the PORTSCAN.", + ) + latest_portscan_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the VULNSCAN.", + ) + latest_scan_completion_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed all scans.", + ) location_longitude = models.DecimalField( - max_digits=10, decimal_places=6, null=True, blank=True + max_digits=10, + decimal_places=6, + null=True, + blank=True, + help_text="Longitude of host, according to geolocation database", ) location_latitude = models.DecimalField( - max_digits=10, decimal_places=6, null=True, blank=True - ) - priority = models.IntegerField(null=True, blank=True) - next_scan_timestamp = models.DateTimeField(null=True, blank=True) - rand = models.DecimalField(max_digits=10, decimal_places=6, null=True, blank=True) - curr_stage = models.CharField(max_length=255, null=True, blank=True) - host_live = models.BooleanField(null=True, blank=True) - host_live_reason = models.CharField(max_length=255, null=True, blank=True) - status = models.CharField(max_length=255, null=True, blank=True) + max_digits=10, + decimal_places=6, + null=True, + blank=True, + help_text="Latitude of host, according to geolocation database", + ) + priority = models.IntegerField( + null=True, + blank=True, + help_text="Scan priority of this host document, from -16 (most urgent) to 1 (least urgent)", + ) + next_scan_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp indicating when this host document is scheduled to be scanned next; a value of null indicates that the host document has a status other than 'DONE' (i.e. currently queued up for a scan or running a scan)", + ) + rand = models.DecimalField( + max_digits=10, + decimal_places=6, + null=True, + blank=True, + help_text="A random number between 0 and 1 used to randomize scan order", + ) + curr_stage = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Current scan stage for this host document", + ) + host_live = models.BooleanField( + null=True, + blank=True, + help_text="Whether or not a live host was detected at this host document’s IP address by the port scanner", + ) + host_live_reason = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Reason given by the port scanner as to whether or not this host document represents a live host", + ) + status = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Current scan status for this host document. (WAITING, READY, RUNNING, DONE)", + ) organization = models.ForeignKey( Organization, related_name="hosts", on_delete=models.CASCADE, null=True, blank=True, + help_text="Foreign key relationship to the organization that owns the host.", ) class Meta: """The Meta class for Host.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "host" indexes = [ @@ -1140,40 +2071,97 @@ class Ip(models.Model): """The Ip model.""" # id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) - ip_hash = models.TextField(primary_key=True) + ip_hash = models.TextField( + primary_key=True, help_text="A hash of the IP used as a unique identifier." + ) organization = models.ForeignKey( - Organization, related_name="ips", on_delete=models.CASCADE - ) - created_timestamp = models.DateTimeField(auto_now_add=True) - updated_timestamp = models.DateTimeField(null=True, blank=True, auto_now=True) - last_seen_timestamp = models.DateTimeField(null=True, blank=True) - ip = models.GenericIPAddressField(null=True, blank=True) - live = models.BooleanField(null=True, blank=True) - false_positive = models.BooleanField(null=True, blank=True) - from_cidr = models.BooleanField(null=True, blank=True) - retired = models.BooleanField(null=True, blank=True) - last_reverse_lookup = models.DateTimeField(blank=True, null=True) - from_cidr = models.BooleanField(blank=True, null=True) - - # domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) - # host_scans = models.ManyToManyField("HostScan", related_name='ips', blank=True) - # hosts = models.ManyToManyField(Host, related_name='ips', blank=True) - # tickets = models.ManyToManyField("Ticket", related_name='ips', blank=True) - # vuln_scans = models.ManyToManyField(VulnScan, related_name='ips', blank=True) - # port_scans = models.ManyToManyField("PortScan", related_name='ips', blank=True) - sub_domains = models.ManyToManyField("SubDomains", related_name="ips", blank=True) - has_shodan_results = models.BooleanField(blank=True, null=True) - origin_cidr = models.ForeignKey( - Cidr, on_delete=models.CASCADE, db_column="origin_cidr", blank=True, null=True + Organization, + related_name="ips", + on_delete=models.CASCADE, + help_text="Foreign key relationship to the organization that owns the IP.", ) - current = models.BooleanField(blank=True, null=True) - - class Meta: - """The Meta class for Ip.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "ip" + created_timestamp = models.DateTimeField( + auto_now_add=True, + help_text="Timestamp the cidr object was added to the database.", + ) + updated_timestamp = models.DateTimeField( + null=True, + blank=True, + auto_now=True, + help_text="Timestamp of the last time the IP object was updated.", + ) + last_seen_timestamp = models.DateTimeField( + null=True, blank=True, help_text="Timestamp of the last time the IP was seen." + ) + ip = models.GenericIPAddressField( + null=True, blank=True, help_text="The IP address." + ) + live = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if the IP is live as of the last scan.", + ) + false_positive = models.BooleanField( + null=True, + blank=True, + help_text="A boolean field that marks if the IP was incorrectly attributed to the stakeholder.", + ) + from_cidr = models.BooleanField( + null=True, blank=True, help_text="The cidr block the IP originated from." + ) + retired = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if the IP is no longer owned by the organization.", + ) + last_reverse_lookup = models.DateTimeField( + blank=True, + null=True, + help_text="Last time a reverse lookup was run against the IP.", + ) + from_cidr = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field that flags if the IP came from a stakeholder provided cidr.", + ) + + # domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) + # host_scans = models.ManyToManyField("HostScan", related_name='ips', blank=True) + # hosts = models.ManyToManyField(Host, related_name='ips', blank=True) + # tickets = models.ManyToManyField("Ticket", related_name='ips', blank=True) + # vuln_scans = models.ManyToManyField(VulnScan, related_name='ips', blank=True) + # port_scans = models.ManyToManyField("PortScan", related_name='ips', blank=True) + sub_domains = models.ManyToManyField( + "SubDomains", + related_name="ips", + blank=True, + help_text="Many to many relationship linking to sub domains that were seen running on the IP.", + ) + has_shodan_results = models.BooleanField( + blank=True, + null=True, + help_text="A boolean field that flags if shodan has findings for the givenn IP", + ) + origin_cidr = models.ForeignKey( + Cidr, + on_delete=models.CASCADE, + db_column="origin_cidr", + blank=True, + null=True, + help_text="Foreign key relationship to the cidr from which the ip was enumerated.", + ) + current = models.BooleanField( + blank=True, + null=True, + help_text="A boolean field that flags if the IP is current.", + ) + + class Meta: + """The Meta class for Ip.""" + + app_label = app_label_name + managed = manage_db + db_table = "ip" indexes = [models.Index(fields=["ip", "organization"])] unique_together = ["ip", "organization"] @@ -1182,59 +2170,152 @@ class Ticket(models.Model): """The Ticket model.""" id = models.CharField( - max_length=255, primary_key=True + max_length=255, + primary_key=True, + help_text="Unique identifier for a ticket object in the database.", ) # Assuming the UUID is represented as a string - cve_string = models.CharField(max_length=255, null=True, blank=True) + cve_string = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="CVE (Common Vulnerabilities and Exposures) id for the vulnerability.", + ) cve = models.ForeignKey( - Cve, related_name="tickets", null=True, blank=True, on_delete=models.CASCADE + Cve, + related_name="tickets", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the related CVE object.", ) cvss_base_score = models.DecimalField( - max_digits=5, decimal_places=2, null=True, blank=True + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="CVSS base score](https://nvd.nist.gov/vuln-metrics)", + ) + cvss_version = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="CVSS version used for the CVSS base score", ) - cvss_version = models.CharField(max_length=255, null=True, blank=True) # kev = models.ForeignKey(Kev, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) - vuln_name = models.CharField(max_length=255, null=True, blank=True) - cvss_score_source = models.CharField(max_length=255, null=True, blank=True) + vuln_name = models.CharField( + max_length=255, null=True, blank=True, help_text="Vulnerability name" + ) + cvss_score_source = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the CVSS base score (e.g. 'nvd' or 'nessus')", + ) cvss_severity = models.DecimalField( - max_digits=5, decimal_places=2, null=True, blank=True + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="[CVSS severity rating](https://nvd.nist.gov/vuln-metrics)", ) vpr_score = models.DecimalField( - max_digits=5, decimal_places=2, null=True, blank=True + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="Tenable [Vulnerability Priority Rating](https://docs.tenable.com/nessus/Content/RiskMetrics.htm)", + ) + false_positive = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if this ticket is marked as a false positive?", + ) + ip_string = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="IP address of the host that was vulnerability scanned", ) - false_positive = models.BooleanField(null=True, blank=True) - ip_string = models.CharField(max_length=255, null=True, blank=True) ip = models.ForeignKey( - Ip, related_name="tickets", null=True, blank=True, on_delete=models.CASCADE + Ip, + related_name="tickets", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the related IP object.", + ) + updated_timestamp = models.DateTimeField( + null=True, blank=True, help_text="Timestamp of when the ticket was last updated" ) - updated_timestamp = models.DateTimeField(null=True, blank=True) location_longitude = models.DecimalField( - max_digits=9, decimal_places=6, null=True, blank=True + max_digits=9, + decimal_places=6, + null=True, + blank=True, + help_text="Longitude of host (according to geolocation database) associated with this ticket", ) location_latitude = models.DecimalField( - max_digits=9, decimal_places=6, null=True, blank=True + max_digits=9, + decimal_places=6, + null=True, + blank=True, + help_text="Latitude of host (according to geolocation database) associated with this ticket", + ) + found_in_latest_host_scan = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if this vulnerability was detected in the latest scan of the associated host?", ) - found_in_latest_host_scan = models.BooleanField(null=True, blank=True) organization = models.ForeignKey( Organization, related_name="tickets", null=True, blank=True, on_delete=models.CASCADE, + help_text="Foreign key relationship to the organization that owns the asset that was scanned.", + ) + vuln_port = models.IntegerField( + null=True, blank=True, help_text="Number of the vulnerable port in this ticket." + ) + port_protocol = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Protocol for the vulnerable port in this ticket ('tcp' or 'udp')", + ) + snapshots_bool = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if there are snapshots that include this ticket", + ) + vuln_source = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the vulnerability scan (e.g. 'nessus' or 'nmap')", + ) + vuln_source_id = models.IntegerField( + null=True, + blank=True, + help_text="Source-specific identifier for the vulnerability scan (e.g. the scanner plugin identifier that detected the vulnerability)", + ) + closed_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp when this ticket was closed (vulnerability was no longer detected); value of null indicates that this ticket is currently open", + ) + opened_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp when this ticket was opened (vulnerability was first detected)", ) - vuln_port = models.IntegerField(null=True, blank=True) - port_protocol = models.CharField(max_length=255, null=True, blank=True) - snapshots_bool = models.BooleanField(null=True, blank=True) - vuln_source = models.CharField(max_length=255, null=True, blank=True) - vuln_source_id = models.IntegerField(null=True, blank=True) - closed_timestamp = models.DateTimeField(null=True, blank=True) - opened_timestamp = models.DateTimeField(null=True, blank=True) # snapshots = models.ManyToManyField(Snapshot, related_name='tickets', blank=True) # ticket_events = models.ManyToManyField(TicketEvent, related_name='tickets', blank=True) class Meta: """The Meta class for Ticket.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "ticket" unique_together = ["id"] @@ -1244,23 +2325,76 @@ class PortScan(models.Model): """The PortScan model.""" id = models.CharField( - max_length=255, primary_key=True + max_length=255, + primary_key=True, + help_text="Unique identifier for the port scan object.", ) # Assuming UUIDs are stored as strings - ip_string = models.CharField(max_length=255, null=True, blank=True) + ip_string = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="IP address of the host that was port scanned", + ) ip = models.ForeignKey( - Ip, related_name="port_scans", null=True, blank=True, on_delete=models.CASCADE - ) - latest = models.BooleanField(default=False) - port = models.IntegerField(null=True, blank=True) - protocol = models.CharField(max_length=255, null=True, blank=True) - reason = models.CharField(max_length=255, null=True, blank=True) - service = models.JSONField(default=dict) # Use JSONField to store JSON objects - service_name = models.CharField(max_length=255, null=True, blank=True) - service_confidence = models.IntegerField(null=True, blank=True) - service_method = models.CharField(max_length=255, null=True, blank=True) - source = models.CharField(max_length=255, null=True, blank=True) - state = models.CharField(max_length=255, null=True, blank=True) - time_scanned = models.DateTimeField(null=True, blank=True) + Ip, + related_name="port_scans", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the related IP.", + ) + latest = models.BooleanField( + default=False, + help_text="Booolean field that flags if this is the latest scan of this port.", + ) + port = models.IntegerField( + null=True, blank=True, help_text="Number of the port that was scanned." + ) + protocol = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Protocol for this port scan ('tcp' or 'udp').", + ) + reason = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Why this port is determined to be open, as reported by the port scanner.", + ) + service = models.JSONField( + default=dict, help_text="Details about this port, as reported by the scanner" + ) # Use JSONField to store JSON objects + service_name = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the scan (e.g. 'nmap')", + ) + service_confidence = models.IntegerField( + null=True, blank=True, help_text="Level of confidence the service is running." + ) + service_method = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="The method that was used to identify the service on the port.", + ) + source = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the scan (e.g. 'nmap')", + ) + state = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="State of the port, as reported by the scanner; see nmap states", + ) + time_scanned = models.DateTimeField( + null=True, blank=True, help_text="Timestamp when the port was scanned" + ) # snapshots = models.ManyToManyField(Snapshot, related_name='port_scans', blank=True) organization = models.ForeignKey( Organization, @@ -1268,12 +2402,13 @@ class PortScan(models.Model): null=True, blank=True, on_delete=models.CASCADE, + help_text="Foreign key relationship to the organization that owns the scanned IP.", ) class Meta: """The Meta class for PortScan.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "port_scan" @@ -1285,33 +2420,68 @@ class WasTrackerCustomerdata(models.Model): """Define WasTrackerCustomerdata model.""" customer_id = models.UUIDField( - db_column="customer_id", primary_key=True, default=uuid.uuid1 - ) - tag = models.TextField() - customer_name = models.TextField() - testing_sector = models.TextField() - ci_type = models.TextField() - jira_ticket = models.TextField() - ticket = models.TextField() - next_scheduled = models.TextField() - last_scanned = models.TextField() - frequency = models.TextField() - comments_notes = models.TextField() - was_report_poc = models.TextField() - was_report_email = models.TextField() - onboarding_date = models.TextField() - no_of_web_apps = models.IntegerField() - no_web_apps_last_updated = models.TextField(blank=True, null=True) - elections = models.BooleanField(blank=False, null=False) - fceb = models.BooleanField(blank=False, null=False) - special_report = models.BooleanField(blank=False, null=False) - report_password = models.TextField() - child_tags = models.TextField() + db_column="customer_id", + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a Was customer.", + ) + tag = models.TextField( + help_text="Short name of the customer used to query reports, ideally shoulud match ServiceNow, PE and VS." + ) + customer_name = models.TextField(help_text="Full name of the WAS customer.") + testing_sector = models.TextField(help_text="The sector the customer falls under.") + ci_type = models.TextField(help_text="Critical infrastructure classification.") + jira_ticket = models.TextField(help_text="???") + ticket = models.TextField(help_text="???") + next_scheduled = models.TextField( + help_text="The next date and time the customer's webapps will be scanned." + ) + last_scanned = models.TextField( + help_text="The last date and time the customer's webapps were scanned." + ) + frequency = models.TextField(help_text="The frequency the WAS reports are run.") + comments_notes = models.TextField( + help_text="Additional comments and notes about how and when to run the report." + ) + was_report_poc = models.TextField(help_text="Customer's point of contact.") + was_report_email = models.TextField( + help_text="Email address(es) that WAS reports are delivered to." + ) + onboarding_date = models.TextField( + help_text="Date that the customer was added to the WAS service." + ) + no_of_web_apps = models.IntegerField( + help_text="Number of webapps the customer has submitted to be scanned." + ) + no_web_apps_last_updated = models.TextField( + blank=True, + null=True, + help_text="The last datetime that the number of apps was updated.", + ) + elections = models.BooleanField( + blank=False, + null=False, + help_text="Boolean field that flags if the customer is an election entity.", + ) + fceb = models.BooleanField( + blank=False, + null=False, + help_text="Boolean field that flags if the customer is an FCEB entity.", + ) + special_report = models.BooleanField( + blank=False, + null=False, + help_text="Boolean field that flags if the customer receives a special report.", + ) + report_password = models.TextField( + help_text="The password used to encrypt the WAS report." + ) + child_tags = models.TextField(help_text="List of tags of any child customers.") class Meta: """Set WasTrackerCustomerdata model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "was_tracker_customer_data" @@ -1346,143 +2516,441 @@ class Meta: class WasFindings(models.Model): """Define WasFindings model.""" - finding_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - finding_type = models.TextField(blank=True, null=True) - webapp_id = models.IntegerField(blank=True, null=True) - was_org_id = models.TextField(blank=True, null=True) - owasp_category = models.TextField(blank=True, null=True) - severity = models.TextField(blank=True, null=True) - times_detected = models.IntegerField(blank=True, null=True) - base_score = models.FloatField(blank=True, null=True) - temporal_score = models.FloatField(blank=True, null=True) - fstatus = models.TextField(blank=True, null=True) - last_detected = models.DateField(blank=True, null=True) - first_detected = models.DateField(blank=True, null=True) - is_remediated = models.BooleanField(blank=True, null=True) - potential = models.BooleanField(blank=True, null=True) - webapp_url = models.TextField(blank=True, null=True) - webapp_name = models.TextField(blank=True, null=True) - name = models.TextField(blank=True, null=True) - cvss_v3_attack_vector = models.TextField(blank=True, null=True) + finding_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a WAS finding object.", + ) + finding_type = models.TextField( + blank=True, + null=True, + help_text="Type of WAS finding. (INFORMATION_GATHERED, SENSITIVE_CONTENT, VULNERABILITY)", + ) + webapp_id = models.IntegerField( + blank=True, + null=True, + help_text="Identifier for the webapp on which the finding was found.", + ) + was_org_id = models.TextField( + blank=True, + null=True, + help_text="Acronym of the customer who owns the scanned webapp.", + ) + owasp_category = models.TextField( + blank=True, + null=True, + help_text="OWASP (Open Web Application Security Project) categorization of the finding.", + ) + severity = models.TextField( + blank=True, null=True, help_text="Severity of the finding, rated 1-5." + ) + times_detected = models.IntegerField( + blank=True, null=True, help_text="How many times the finding has been seen." + ) + base_score = models.FloatField( + blank=True, null=True, help_text="Base CVSS score for the finding." + ) + temporal_score = models.FloatField( + blank=True, null=True, help_text="Temporal CVSS score for the finding." + ) + fstatus = models.TextField( + blank=True, + null=True, + help_text="Status of finding. (NEW, ACTIVE, REOPENED, FIXED)", + ) + last_detected = models.DateField( + blank=True, null=True, help_text="The last time the finding was seen." + ) + first_detected = models.DateField( + blank=True, null=True, help_text="The first time the finding was seen." + ) + is_remediated = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the fiding has been remediated.", + ) + potential = models.BooleanField(blank=True, null=True, help_text="???") + webapp_url = models.TextField( + blank=True, + null=True, + help_text="URL of the webapp where the finding was identified.", + ) + webapp_name = models.TextField( + blank=True, + null=True, + help_text="Name of the webapp where the finding was identified.", + ) + name = models.TextField(blank=True, null=True, help_text="Name of the finding.") + cvss_v3_attack_vector = models.TextField( + blank=True, + null=True, + help_text="Vector of the attack. (Adjacent Network, Local Access, Network, None)", + ) cwe_list = ArrayField( - models.IntegerField(blank=True, null=True), blank=True, null=True + models.IntegerField(blank=True, null=True), + blank=True, + null=True, + help_text="List of CWEs identified in the finding.", + ) + wasc_list = models.JSONField( + blank=True, + null=True, + help_text="List of dictionaries containing links to relevant WASC (Web Application Security Consortium) references.", + ) + last_tested = models.DateField( + blank=True, null=True, help_text="Last time the finding was tested." + ) + fixed_date = models.DateField( + blank=True, null=True, help_text="Date the finding was remediated." + ) + is_ignored = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the customer has decided to ignore the finding.", + ) + url = models.TextField( + blank=True, null=True, help_text="URL where the finding was identified." + ) + qid = models.IntegerField( + blank=True, null=True, help_text="Qualys id for the finding." + ) + response = models.TextField( + blank=True, null=True, help_text="The returned response from the webapp." + ) + + class Meta: + """Set WasFindings model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_findings" + + +class WasHistory(models.Model): + """Define WasHistory model.""" + + was_org_id = models.TextField( + blank=True, null=True, help_text="Unique identifier for the WAS history object." + ) + date_scanned = models.DateField( + help_text="The date a customers webapps were scanned." + ) + vuln_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many vulnerabilities were identified.", + ) + vuln_webapp_cnt = models.IntegerField( + blank=True, null=True, help_text="The count of how many webapps are vulnerable." + ) + web_app_cnt = models.IntegerField( + blank=True, + null=True, + help_text="Count of how many webapps were scanned for the customer.", + ) + high_rem_time = models.IntegerField( + blank=True, + null=True, + help_text="Average time it took to remediate vulnerabilities with a high severity.", + ) + crit_rem_time = models.IntegerField( + blank=True, + null=True, + help_text="Average time it took to remediate vulnerabilities with a critical severity.", + ) + crit_vuln_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many vulnerabilites have a critical severity.", + ) + high_vuln_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many vulnerabilites have a high severity.", + ) + report_period = models.DateField( + blank=True, + null=True, + help_text="The report period these findings were identified within.", + ) + high_rem_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many high severity vulnerabiliteis were remediated.", + ) + crit_rem_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many critical severity vulnerabiliteis were remediated.", + ) + total_potential = models.IntegerField( + blank=True, + null=True, + help_text="A count of all potential vulnerabilities there are across a customer's webapps.", + ) + + class Meta: + """Set WasHistory model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_history" + unique_together = (("was_org_id", "date_scanned"),) + + +class WasMap(models.Model): + """Define WasMap model.""" + + was_org_id = models.TextField( + blank=True, primary_key=True, help_text="WAS customer acronym." + ) + pe_org_id = models.UUIDField( + blank=True, null=True, help_text="Corresponding PE organization acronym" + ) + report_on = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the organization receives a report???", + ) # Not sure if this is a PE or WAS report. + last_scanned = models.DateField( + blank=True, + null=True, + help_text="Last time the organization was scanned by WAS.", + ) + + class Meta: + """Set WasMap model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_map" + + +class WasReport(models.Model): + """The WasReport model.""" + + org_name = models.TextField( + blank=True, null=True, help_text="Name of the WAS customer." + ) + date_pulled = models.DateTimeField( + blank=True, null=True, help_text="Date the was report wasw pulled from Qualys." + ) + last_scan_date = models.DateTimeField( + blank=True, + null=True, + help_text="Last time WAS ran a Qualys scan on the organization's webapps.", + ) + security_risk = models.TextField( + blank=True, null=True, help_text="Security risk the customer's webapps face." + ) + total_info = models.IntegerField( + blank=True, + null=True, + help_text="Number of findings found across the customer's webapps.???", + ) + num_apps = models.IntegerField( + blank=True, null=True, help_text="Number of webapps scanned in this report." + ) + risk_color = models.TextField( + blank=True, + null=True, + help_text="Color code for the risk level presented in the reports.", + ) + sensitive_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of sensitive findings included inn the report.???", + ) + sensitive_color = models.TextField( + blank=True, + null=True, + help_text="Color code for the sensitivity level presented in the report.", + ) + max_days_open_urgent = models.IntegerField( + blank=True, + null=True, + help_text="The maximum days an urgent finding has remained open.", + ) + max_days_open_critical = models.IntegerField( + blank=True, + null=True, + help_text="The maximum days a critical finding has remained open.", + ) + urgent_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display urgent details in the report.", + ) + critical_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display critical details in the report.", + ) + org_was_acronym = models.TextField( + blank=True, + null=True, + help_text="Acronym or short name of the organization receiving the report.", + ) + name_len = models.TextField( + blank=True, + null=True, + help_text="Number of characters in the organization's name.", + ) + vuln_csv_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary containing counts of each type of vulnerability for each webapp", + ) + ssn_cc_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary containing counts of credit card and social security data found on the customer's webapps.", + ) + app_overview_csv_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary containing an overview of the apps urls and findings.", + ) + details_csv = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of additional details regarding the findings in the report.???", + ) + info_csv = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of Finding information for each of the findings in the report.", + ) + links_crawled = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of links crawled including duration of time and depth of the crawl.", + ) + links_rejected = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of rejecting links and which webapp they were from.???", + ) + emails_found = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of emails found in each webapp.", + ) + owasp_count_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary that counts each of the OWASP categories for each webapp.???", + ) + group_count_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary counting the sums of each OWASP category for all webapps.", + ) + fixed = models.IntegerField( + blank=True, + null=True, + help_text="Count of fixed vulns in all webapps owned by the organization.", + ) + total = models.IntegerField( + blank=True, + null=True, + help_text="Total vulnerability count across all webapps owned by the organization.", + ) + vulns_monthly_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary summing each of the findings by month they were found.", + ) + path_disc = models.IntegerField(blank=True, null=True, help_text="???") + info_disc = models.IntegerField(blank=True, null=True, help_text="???") + cross_site = models.IntegerField( + blank=True, + null=True, + help_text="Count of cross site scripting vulnerabilities.", + ) + burp = models.IntegerField( + blank=True, null=True, help_text="Vulnerabilities detected by BURP.???" + ) + sql_inj = models.IntegerField( + blank=True, null=True, help_text="Count of SQL injection vulnerabilities." + ) + bugcrowd = models.IntegerField( + blank=True, null=True, help_text="Vulnerabilities detected by Bugcrowd." + ) + reopened = models.IntegerField( + blank=True, null=True, help_text="Count of reopened vulnerabilities." + ) + reopened_color = models.TextField( + blank=True, null=True, help_text="Color used to display the reopened count." + ) + new_vulns = models.IntegerField( + blank=True, null=True, help_text="Count of new vulnerablities" + ) + new_vulns_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display count of new vulnerabilities.", + ) + tot_vulns = models.IntegerField( + blank=True, null=True, help_text="Total count of vulnerabilities." + ) + tot_vulns_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display count of new vulnerabilities.", + ) + lev1 = models.IntegerField( + blank=True, null=True, help_text="Count of level 1 vulnerabilities." + ) + lev2 = models.IntegerField( + blank=True, null=True, help_text="Count of level 2 vulnerabilities." + ) + lev3 = models.IntegerField( + blank=True, null=True, help_text="Count of level 3 vulnerabilities." + ) + lev4 = models.IntegerField( + blank=True, null=True, help_text="Count of level 4 vulnerabilities." + ) + lev5 = models.IntegerField( + blank=True, null=True, help_text="Count of level 5 vulnerabilities." + ) + severities = ArrayField( + models.IntegerField(), + blank=True, + null=True, + default=list, + help_text="List of the severities assigned to each of the vulnerabilities.", + ) + ages = ArrayField( + models.IntegerField(), + blank=True, + null=True, + default=list, + help_text="List of ages of all the vulnerabilities.", + ) + pdf_obj = models.BinaryField( + blank=True, + null=True, + help_text="PDF binary or the full pdf generated by Qualys.", ) - wasc_list = models.JSONField(blank=True, null=True) - last_tested = models.DateField(blank=True, null=True) - fixed_date = models.DateField(blank=True, null=True) - is_ignored = models.BooleanField(blank=True, null=True) - url = models.TextField(blank=True, null=True) - qid = models.IntegerField(blank=True, null=True) - response = models.TextField(blank=True, null=True) - - class Meta: - """Set WasFindings model metadata.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "was_findings" - - -class WasHistory(models.Model): - """Define WasHistory model.""" - - was_org_id = models.TextField(blank=True, null=True) - date_scanned = models.DateField() - vuln_cnt = models.IntegerField(blank=True, null=True) - vuln_webapp_cnt = models.IntegerField(blank=True, null=True) - web_app_cnt = models.IntegerField(blank=True, null=True) - high_rem_time = models.IntegerField(blank=True, null=True) - crit_rem_time = models.IntegerField(blank=True, null=True) - crit_vuln_cnt = models.IntegerField(blank=True, null=True) - high_vuln_cnt = models.IntegerField(blank=True, null=True) - report_period = models.DateField(blank=True, null=True) - high_rem_cnt = models.IntegerField(blank=True, null=True) - crit_rem_cnt = models.IntegerField(blank=True, null=True) - total_potential = models.IntegerField(blank=True, null=True) - - class Meta: - """Set WasHistory model metadata.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "was_history" - unique_together = (("was_org_id", "date_scanned"),) - - -class WasMap(models.Model): - """Define WasMap model.""" - - was_org_id = models.TextField(blank=True, primary_key=True) - pe_org_id = models.UUIDField(blank=True, null=True) - report_on = models.BooleanField(blank=True, null=True) - last_scanned = models.DateField(blank=True, null=True) - - class Meta: - """Set WasMap model metadata.""" - - app_label = "dmz_mini_dl" - managed = manage_db - db_table = "was_map" - - -class WasReport(models.Model): - """The WasReport model.""" - - org_name = models.TextField(blank=True, null=True) - date_pulled = models.DateTimeField(blank=True, null=True) - last_scan_date = models.DateTimeField(blank=True, null=True) - security_risk = models.TextField(blank=True, null=True) - total_info = models.IntegerField(blank=True, null=True) - num_apps = models.IntegerField(blank=True, null=True) - risk_color = models.TextField(blank=True, null=True) - sensitive_count = models.IntegerField(blank=True, null=True) - sensitive_color = models.TextField(blank=True, null=True) - max_days_open_urgent = models.IntegerField(blank=True, null=True) - max_days_open_critical = models.IntegerField(blank=True, null=True) - urgent_color = models.TextField(blank=True, null=True) - critical_color = models.TextField(blank=True, null=True) - org_was_acronym = models.TextField(blank=True, null=True) - name_len = models.TextField(blank=True, null=True) - vuln_csv_dict = models.JSONField(blank=True, null=True, default=dict) - ssn_cc_dict = models.JSONField(blank=True, null=True, default=dict) - app_overview_csv_dict = models.JSONField(blank=True, null=True, default=dict) - details_csv = models.JSONField(blank=True, null=True, default=list) - info_csv = models.JSONField(blank=True, null=True, default=list) - links_crawled = models.JSONField(blank=True, null=True, default=list) - links_rejected = models.JSONField(blank=True, null=True, default=list) - emails_found = models.JSONField(blank=True, null=True, default=list) - owasp_count_dict = models.JSONField(blank=True, null=True, default=dict) - group_count_dict = models.JSONField(blank=True, null=True, default=dict) - fixed = models.IntegerField(blank=True, null=True) - total = models.IntegerField(blank=True, null=True) - vulns_monthly_dict = models.JSONField(blank=True, null=True, default=dict) - path_disc = models.IntegerField(blank=True, null=True) - info_disc = models.IntegerField(blank=True, null=True) - cross_site = models.IntegerField(blank=True, null=True) - burp = models.IntegerField(blank=True, null=True) - sql_inj = models.IntegerField(blank=True, null=True) - bugcrowd = models.IntegerField(blank=True, null=True) - reopened = models.IntegerField(blank=True, null=True) - reopened_color = models.TextField(blank=True, null=True) - new_vulns = models.IntegerField(blank=True, null=True) - new_vulns_color = models.TextField(blank=True, null=True) - tot_vulns = models.IntegerField(blank=True, null=True) - tot_vulns_color = models.TextField(blank=True, null=True) - lev1 = models.IntegerField(blank=True, null=True) - lev2 = models.IntegerField(blank=True, null=True) - lev3 = models.IntegerField(blank=True, null=True) - lev4 = models.IntegerField(blank=True, null=True) - lev5 = models.IntegerField(blank=True, null=True) - severities = ArrayField(models.IntegerField(), blank=True, null=True, default=list) - ages = ArrayField(models.IntegerField(), blank=True, null=True, default=list) - pdf_obj = models.BinaryField(blank=True, null=True) class Meta: """The Meta class for WasReport.""" db_table = "was_report" unique_together = ("last_scan_date", "org_was_acronym") - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db @@ -1490,18 +2958,51 @@ class Meta: class PeUsers(models.Model): """Define Users model.""" - id = models.UUIDField(primary_key=True) - email = models.CharField(unique=True, max_length=64, blank=True, null=True) - username = models.CharField(unique=True, max_length=64, blank=True, null=True) - admin = models.IntegerField(blank=True, null=True) - role = models.IntegerField(blank=True, null=True) - password_hash = models.CharField(max_length=128, blank=True, null=True) - api_key = models.CharField(unique=True, max_length=128, blank=True, null=True) + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a PE user object." + ) + email = models.CharField( + unique=True, + max_length=64, + blank=True, + null=True, + help_text="Email address of the user.", + ) + username = models.CharField( + unique=True, + max_length=64, + blank=True, + null=True, + help_text="Username of the user.", + ) + admin = models.IntegerField( + blank=True, + null=True, + help_text="Django generated field that determines the admin permissions for the user.", + ) + role = models.IntegerField( + blank=True, + null=True, + help_text="Django generated field that determines the role permissions for the user.", + ) + password_hash = models.CharField( + max_length=128, + blank=True, + null=True, + help_text="Cryptographic hash of the user's password.", + ) + api_key = models.CharField( + unique=True, + max_length=128, + blank=True, + null=True, + help_text="The user's API key.", + ) class Meta: """Set User model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "pe_users" @@ -1510,12 +3011,16 @@ class Meta: class AlembicVersion(models.Model): """Define AlembicVersion model.""" - version_num = models.CharField(primary_key=True, max_length=32) + version_num = models.CharField( + primary_key=True, + max_length=32, + help_text="A unique identifier assigned to each database migration script in Alembic, this may not be used currently.", + ) class Meta: """Set AlembicVersion model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "alembic_version" @@ -1523,34 +3028,82 @@ class Meta: class SixgillAlerts(models.Model): """Define Alerts model.""" - alerts_uid = models.UUIDField(primary_key=True) - alert_name = models.TextField(blank=True, null=True) - content = models.TextField(blank=True, null=True) - date = models.DateField(blank=True, null=True) - sixgill_id = models.TextField(unique=True, blank=True, null=True) - read = models.TextField(blank=True, null=True) - severity = models.TextField(blank=True, null=True) - site = models.TextField(blank=True, null=True) - threat_level = models.TextField(blank=True, null=True) - threats = models.TextField(blank=True, null=True) - title = models.TextField(blank=True, null=True) - user_id = models.TextField(blank=True, null=True) - category = models.TextField(blank=True, null=True) - lang = models.TextField(blank=True, null=True) + alerts_uid = models.UUIDField( + primary_key=True, + help_text="Unique identifier for the cyber sixgill alert object.", + ) + alert_name = models.TextField( + blank=True, null=True, help_text="Name of the alert provided by Cybersixgill." + ) + content = models.TextField( + blank=True, + null=True, + help_text="Content of the post or website that triggered the alert.", + ) + date = models.DateField( + blank=True, null=True, help_text="Date the alert was created." + ) + sixgill_id = models.TextField( + unique=True, + blank=True, + null=True, + help_text="Cybersixgill ID associated with alert.", + ) + read = models.TextField( + blank=True, + null=True, + help_text="Boolean field that flags if the alert was read in the Cybersixgill portal.", + ) + severity = models.TextField( + blank=True, null=True, help_text="Severity ranking of alert from 1 - 10." + ) + site = models.TextField( + blank=True, null=True, help_text="Site associated with the alert." + ) + threat_level = models.TextField( + blank=True, + null=True, + help_text="Threat level of alert either 'imminent' or 'emerging'.", + ) + threats = models.TextField( + blank=True, null=True, help_text="Type of threat for alert" + ) + title = models.TextField(blank=True, null=True, help_text="Title of alert post") + user_id = models.TextField( + blank=True, null=True, help_text="Id of user that made the API call" + ) + category = models.TextField(blank=True, null=True, help_text="Category of alert") + lang = models.TextField( + blank=True, null=True, help_text="Language of alert content" + ) organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization_uid" + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="Foreign Key to the related organization", ) data_source = models.ForeignKey( - "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="Foreign Key to the data_source.", + ) + content_snip = models.TextField( + blank=True, + null=True, + help_text="100 character snippet of the post content. 50 characters before/after the specific mention", + ) + asset_mentioned = models.TextField( + blank=True, null=True, help_text="Asset mentioned in alert" + ) + asset_type = models.TextField( + blank=True, null=True, help_text="Type of asset mentioned in alert" ) - content_snip = models.TextField(blank=True, null=True) - asset_mentioned = models.TextField(blank=True, null=True) - asset_type = models.TextField(blank=True, null=True) class Meta: """Set Alerts model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "sixgill_alerts" @@ -1558,16 +3111,21 @@ class Meta: class Alias(models.Model): """Define Alias model.""" - alias_uid = models.UUIDField(primary_key=True) + alias_uid = models.UUIDField( + primary_key=True, help_text="Unique identifier for an alias." + ) organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization_uid" + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organizations", ) - alias = models.TextField(unique=True) + alias = models.TextField(unique=True, help_text="Alias for an organization") class Meta: """Set Alias model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "alias" @@ -1577,24 +3135,51 @@ class AssetHeaders(models.Model): """Define AssetHeaders model.""" field_id = models.UUIDField( - db_column="_id", primary_key=True + db_column="_id", + primary_key=True, + help_text="Unique identifier for the asset header object.", ) # Field renamed because it started with '_'. organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization_uid" + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="Foreign key relationship to the organization that owns the asset.", + ) + sub_url = models.TextField(help_text="URL to the subdomain that was scanned.") + tech_detected = models.TextField( + help_text="List of technologies identified running on the subdomain." + ) # This field type is a guess. + interesting_header = models.TextField( + help_text="List of headers that potentially have relevant findings." + ) # This field type is a guess. + ssl2 = models.TextField( + blank=True, + null=True, + help_text="Evidence that the subdomain is running the outdateed SSL2 protocol", + ) # This field type is a guess. + tls1 = models.TextField( + blank=True, + null=True, + help_text="Evidence that the subdomain is running the outdateed TLS1 protocol", + ) # This field type is a guess. + certificate = models.TextField( + blank=True, null=True, help_text="Certificate details of the subdomain." + ) # This field type is a guess. + scanned = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the suubdomain has been scanned.", + ) + ssl_scanned = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if an SSL scan has been run against the subdomain.", ) - sub_url = models.TextField() - tech_detected = models.TextField() # This field type is a guess. - interesting_header = models.TextField() # This field type is a guess. - ssl2 = models.TextField(blank=True, null=True) # This field type is a guess. - tls1 = models.TextField(blank=True, null=True) # This field type is a guess. - certificate = models.TextField(blank=True, null=True) # This field type is a guess. - scanned = models.BooleanField(blank=True, null=True) - ssl_scanned = models.BooleanField(blank=True, null=True) class Meta: """Set AssetHeaders model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "asset_headers" unique_together = (("organization", "sub_url"),) @@ -1699,70 +3284,76 @@ class Meta: # db_table = "auth_user_user_permissions" # unique_together = (("user", "permission"),) -# needs merging merged into cidr table -# class Cidrs(models.Model): -# """Define Cidrs model.""" - -# cidr_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) -# network = models.TextField() # This field type is a guess. -# organization_uid = models.ForeignKey( -# "Organizations", -# on_delete=models.CASCADE, -# db_column="organization_uid", -# blank=True, -# null=True, -# ) -# data_source_uid = models.ForeignKey( -# "DataSource", -# on_delete=models.CASCADE, -# db_column="data_source_uid", -# blank=True, -# null=True, -# ) -# insert_alert = models.TextField(blank=True, null=True) -# first_seen = models.DateField(blank=True, null=True) -# last_seen = models.DateField(blank=True, null=True) -# current = models.BooleanField(blank=True, null=True) - -# class Meta: -# """Set Cidrs model metadata.""" - -# managed = False -# db_table = "cidrs" -# unique_together = (("organization_uid", "network"),) - class CredentialBreaches(models.Model): """Define CredentialBreaches model.""" - credential_breaches_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - breach_name = models.TextField(unique=True) - description = models.TextField(blank=True, null=True) - exposed_cred_count = models.BigIntegerField(blank=True, null=True) - breach_date = models.DateField(blank=True, null=True) - added_date = models.DateTimeField(blank=True, null=True) - modified_date = models.DateTimeField(blank=True, null=True) + credential_breaches_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for credential_breaches.", + ) + breach_name = models.TextField(unique=True, help_text="Name of breach.") + description = models.TextField( + blank=True, null=True, help_text="Description of breach." + ) + exposed_cred_count = models.BigIntegerField( + blank=True, null=True, help_text="Number of credentials exposed in breach." + ) + breach_date = models.DateField( + blank=True, null=True, help_text="Date when breach occured." + ) + added_date = models.DateTimeField( + blank=True, null=True, help_text="Date breach was added by the source." + ) + modified_date = models.DateTimeField( + blank=True, + null=True, + help_text="Date breach information was last modified/updated.", + ) data_classes = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of types of data identified in the breach.", ) # This field type is a guess. - password_included = models.BooleanField(blank=True, null=True) - is_verified = models.BooleanField(blank=True, null=True) - is_fabricated = models.BooleanField(blank=True, null=True) - is_sensitive = models.BooleanField(blank=True, null=True) - is_retired = models.BooleanField(blank=True, null=True) - is_spam_list = models.BooleanField(blank=True, null=True) + password_included = models.BooleanField( + blank=True, + null=True, + help_text="T/F Were passwords included with the credentials?", + ) + is_verified = models.BooleanField( + blank=True, null=True, help_text="T/F Is breach verified?" + ) + is_fabricated = models.BooleanField( + blank=True, null=True, help_text="T/F Is the breach fabricated?" + ) + is_sensitive = models.BooleanField( + blank=True, + null=True, + help_text="T/F Does the breach contain sensitive content?", + ) + is_retired = models.BooleanField( + blank=True, + null=True, + help_text="T/F Has the breach been retired? (I believe the means it is no longer posted", + ) + is_spam_list = models.BooleanField( + blank=True, null=True, help_text="T/F Is the breach a spam list?" + ) data_source = models.ForeignKey( "DataSource", on_delete=models.CASCADE, db_column="data_source_uid", blank=True, null=True, + help_text="FK: Foreign Key to data_source", ) class Meta: """Set CredentialBreaches model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "credential_breaches" @@ -1770,19 +3361,41 @@ class Meta: class CredentialExposures(models.Model): """Define CredentialExposures model.""" - credential_exposures_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - email = models.TextField() + credential_exposures_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for credential_exposures", + ) + email = models.TextField(help_text="Email found in the breach") organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization_uid" + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + root_domain = models.TextField( + blank=True, + null=True, + help_text="The root domain for the email found in the breach", + ) + sub_domain = models.TextField( + blank=True, + null=True, + help_text="The sub domain for thee email found in the breach", + ) + breach_name = models.TextField( + blank=True, null=True, help_text="Name of breach where credentials were exposed" + ) + modified_date = models.DateTimeField( + blank=True, + null=True, + help_text="Date credential exposure information was last modified/updated", ) - root_domain = models.TextField(blank=True, null=True) - sub_domain = models.TextField(blank=True, null=True) - breach_name = models.TextField(blank=True, null=True) - modified_date = models.DateTimeField(blank=True, null=True) credential_breaches = models.ForeignKey( CredentialBreaches, on_delete=models.CASCADE, db_column="credential_breaches_uid", + help_text="FK: Foreign Key to credential_breaches", ) data_source = models.ForeignKey( "DataSource", @@ -1790,71 +3403,79 @@ class CredentialExposures(models.Model): db_column="data_source_uid", blank=True, null=True, + help_text="FK: Foreign Key to data_source", + ) + name = models.TextField( + blank=True, null=True, help_text="Name of person whose credentials were exposed" + ) + login_id = models.TextField( + blank=True, + null=True, + help_text="Login ID of person whose credentials were exposed", + ) + phone = models.TextField( + blank=True, + null=True, + help_text="Phone number of person whose credentials were exposed", + ) + password = models.TextField( + blank=True, + null=True, + help_text="Password of person whose credentials were exposed", + ) + hash_type = models.TextField( + blank=True, null=True, help_text="The method used to hash the password" + ) + intelx_system_id = models.TextField( + blank=True, null=True, help_text="Id of the Exposure in the intelx system." ) - name = models.TextField(blank=True, null=True) - login_id = models.TextField(blank=True, null=True) - phone = models.TextField(blank=True, null=True) - password = models.TextField(blank=True, null=True) - hash_type = models.TextField(blank=True, null=True) - intelx_system_id = models.TextField(blank=True, null=True) class Meta: """Set CredentialExposures model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "credential_exposures" unique_together = (("breach_name", "email"),) -# needs merging -# class CveInfo(models.Model): -# """Define CveInfo model.""" - -# cve_uuid = models.UUIDField(primary_key=True, default=uuid.uuid1) -# cve_name = models.TextField(unique=True, blank=True, null=True) -# cvss_2_0 = models.DecimalField( -# max_digits=1000, decimal_places=1000, blank=True, null=True -# ) -# cvss_2_0_severity = models.TextField(blank=True, null=True) -# cvss_2_0_vector = models.TextField(blank=True, null=True) -# cvss_3_0 = models.DecimalField( -# max_digits=1000, decimal_places=1000, blank=True, null=True -# ) -# cvss_3_0_severity = models.TextField(blank=True, null=True) -# cvss_3_0_vector = models.TextField(blank=True, null=True) -# dve_score = models.DecimalField( -# max_digits=1000, decimal_places=1000, blank=True, null=True -# ) - -# class Meta: -# """Set CveInfo model metadata.""" - -# managed = False -# db_table = "cve_info" - - class CyhyContacts(models.Model): """Define CyhyContacts model.""" field_id = models.UUIDField( - db_column="_id", primary_key=True, default=uuid.uuid1 + db_column="_id", + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for cyhy contacts", ) # Field renamed because it started with '_'. - org_id = models.TextField() + org_id = models.TextField(help_text="Organization abbreviated name") organization = models.ForeignKey( - "Organization", models.DO_NOTHING, db_column="organization_uid" + "Organization", + models.DO_NOTHING, + db_column="organization_uid", + help_text="FK: Foreign key to the organization", + ) + org_name = models.TextField(help_text="Organization full name") + phone = models.TextField( + blank=True, null=True, help_text="Phone number for organization contact" + ) + contact_type = models.TextField(help_text="Type of contact") + email = models.TextField( + blank=True, null=True, help_text="Email for organization contact" + ) + name = models.TextField( + blank=True, null=True, help_text="Name of organization contact" + ) + date_pulled = models.DateField( + blank=True, + null=True, + help_text="The date we pulled the contact from the cyhy database more recently", ) - org_name = models.TextField() - phone = models.TextField(blank=True, null=True) - contact_type = models.TextField() - email = models.TextField(blank=True, null=True) - name = models.TextField(blank=True, null=True) - date_pulled = models.DateField(blank=True, null=True) class Meta: """Set CyhyContacts model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cyhy_contacts" unique_together = (("org_id", "contact_type", "email", "name"),) @@ -1864,55 +3485,57 @@ class CyhyDbAssets(models.Model): """Define CyhyDbAssets model.""" field_id = models.UUIDField( - db_column="_id", primary_key=True, default=uuid.uuid1 + db_column="_id", + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for cyhy db assets", ) # Field renamed because it started with '_'. - org_id = models.TextField(blank=True, null=True) + org_id = models.TextField( + blank=True, null=True, help_text="Organization abbreviated name" + ) organization = models.ForeignKey( - "Organization", models.DO_NOTHING, db_column="organization_uid" + "Organization", + models.DO_NOTHING, + db_column="organization_uid", + help_text="FK: Foreign key to the organization", + ) + org_name = models.TextField( + blank=True, null=True, help_text="Organization full name" + ) + contact = models.TextField( + blank=True, null=True, help_text="Organization contact information" + ) + network = models.GenericIPAddressField( + blank=True, + null=True, + help_text="Cidr range or IP address owned by the organization", + ) + type = models.TextField(blank=True, null=True, help_text="Network type") + first_seen = models.DateField( + blank=True, + null=True, + help_text="First date and time the asset was associated with the cyhy customer.", + ) + last_seen = models.DateField( + blank=True, + null=True, + help_text="Last date and time the asset was associated with the cyhy customer.", + ) + currently_in_cyhy = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the cidr was seen in the last pull from cyhy", ) - org_name = models.TextField(blank=True, null=True) - contact = models.TextField(blank=True, null=True) - network = models.GenericIPAddressField(blank=True, null=True) - type = models.TextField(blank=True, null=True) - first_seen = models.DateField(blank=True, null=True) - last_seen = models.DateField(blank=True, null=True) - currently_in_cyhy = models.BooleanField(blank=True, null=True) class Meta: """Set CyhyDbAssets model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cyhy_db_assets" unique_together = (("org_id", "network"),) -# Probably included in VS models -# class CyhyPortScans(models.Model): -# """Define CyhyPortScans model.""" - -# cyhy_port_scans_uid = models.UUIDField(primary_key=True) -# organization_uid = models.ForeignKey( -# "Organization", models.DO_NOTHING, db_column="organization_uid" -# ) -# cyhy_id = models.TextField(unique=True, blank=True, null=True) -# cyhy_time = models.DateTimeField(blank=True, null=True) -# service_name = models.TextField(blank=True, null=True) -# port = models.TextField(blank=True, null=True) -# product = models.TextField(blank=True, null=True) -# cpe = models.TextField(blank=True, null=True) -# first_seen = models.DateField(blank=True, null=True) -# last_seen = models.DateField(blank=True, null=True) -# ip = models.TextField(blank=True, null=True) -# state = models.TextField(blank=True, null=True) -# agency_type = models.TextField(blank=True, null=True) - -# class Meta: -# """Set CyhyPortScans model metadata.""" - -# managed = False -# db_table = "cyhy_port_scans" - # TODO determine if we want user logic on both databases # class PEDataapiApiuser(models.Model): # """Define DataapiApiuser model.""" @@ -1936,15 +3559,19 @@ class Meta: class DataSource(models.Model): """Define DataSource model.""" - data_source_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - name = models.TextField() - description = models.TextField() - last_run = models.DateField() + data_source_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for data_sources", + ) + name = models.TextField(help_text="Name of data source") + description = models.TextField(help_text="Description of data source") + last_run = models.DateField(help_text="Date that data source was last ran") class Meta: """Set DataSource model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "data_source" @@ -2020,90 +3647,94 @@ class Meta: class DnsRecords(models.Model): """Define DnsRecords model.""" - dns_record_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - domain_name = models.TextField(blank=True, null=True) - domain_type = models.TextField(blank=True, null=True) - created_date = models.DateTimeField(blank=True, null=True) - updated_date = models.DateTimeField(blank=True, null=True) - expiration_date = models.DateTimeField(blank=True, null=True) + dns_record_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a DNS record.", + ) + domain_name = models.TextField(blank=True, null=True, help_text="") + domain_type = models.TextField(blank=True, null=True, help_text="") + created_date = models.DateTimeField(blank=True, null=True, help_text="") + updated_date = models.DateTimeField(blank=True, null=True, help_text="") + expiration_date = models.DateTimeField(blank=True, null=True, help_text="") name_servers = models.TextField( - blank=True, null=True + blank=True, null=True, help_text="" ) # This field type is a guess. - whois_server = models.TextField(blank=True, null=True) - registrar_name = models.TextField(blank=True, null=True) - status = models.TextField(blank=True, null=True) - clean_text = models.TextField(blank=True, null=True) - raw_text = models.TextField(blank=True, null=True) - registrant_name = models.TextField(blank=True, null=True) - registrant_organization = models.TextField(blank=True, null=True) - registrant_street = models.TextField(blank=True, null=True) - registrant_city = models.TextField(blank=True, null=True) - registrant_state = models.TextField(blank=True, null=True) - registrant_post_code = models.TextField(blank=True, null=True) - registrant_country = models.TextField(blank=True, null=True) - registrant_email = models.TextField(blank=True, null=True) - registrant_phone = models.TextField(blank=True, null=True) - registrant_phone_ext = models.TextField(blank=True, null=True) - registrant_fax = models.TextField(blank=True, null=True) - registrant_fax_ext = models.TextField(blank=True, null=True) - registrant_raw_text = models.TextField(blank=True, null=True) - administrative_name = models.TextField(blank=True, null=True) - administrative_organization = models.TextField(blank=True, null=True) - administrative_street = models.TextField(blank=True, null=True) - administrative_city = models.TextField(blank=True, null=True) - administrative_state = models.TextField(blank=True, null=True) - administrative_post_code = models.TextField(blank=True, null=True) - administrative_country = models.TextField(blank=True, null=True) - administrative_email = models.TextField(blank=True, null=True) - administrative_phone = models.TextField(blank=True, null=True) - administrative_phone_ext = models.TextField(blank=True, null=True) - administrative_fax = models.TextField(blank=True, null=True) - administrative_fax_ext = models.TextField(blank=True, null=True) - administrative_raw_text = models.TextField(blank=True, null=True) - technical_name = models.TextField(blank=True, null=True) - technical_organization = models.TextField(blank=True, null=True) - technical_street = models.TextField(blank=True, null=True) - technical_city = models.TextField(blank=True, null=True) - technical_state = models.TextField(blank=True, null=True) - technical_post_code = models.TextField(blank=True, null=True) - technical_country = models.TextField(blank=True, null=True) - technical_email = models.TextField(blank=True, null=True) - technical_phone = models.TextField(blank=True, null=True) - technical_phone_ext = models.TextField(blank=True, null=True) - technical_fax = models.TextField(blank=True, null=True) - technical_fax_ext = models.TextField(blank=True, null=True) - technical_raw_text = models.TextField(blank=True, null=True) - billing_name = models.TextField(blank=True, null=True) - billing_organization = models.TextField(blank=True, null=True) - billing_street = models.TextField(blank=True, null=True) - billing_city = models.TextField(blank=True, null=True) - billing_state = models.TextField(blank=True, null=True) - billing_post_code = models.TextField(blank=True, null=True) - billing_country = models.TextField(blank=True, null=True) - billing_email = models.TextField(blank=True, null=True) - billing_phone = models.TextField(blank=True, null=True) - billing_phone_ext = models.TextField(blank=True, null=True) - billing_fax = models.TextField(blank=True, null=True) - billing_fax_ext = models.TextField(blank=True, null=True) - billing_raw_text = models.TextField(blank=True, null=True) - zone_name = models.TextField(blank=True, null=True) - zone_organization = models.TextField(blank=True, null=True) - zone_street = models.TextField(blank=True, null=True) - zone_city = models.TextField(blank=True, null=True) - zone_state = models.TextField(blank=True, null=True) - zone_post_code = models.TextField(blank=True, null=True) - zone_country = models.TextField(blank=True, null=True) - zone_email = models.TextField(blank=True, null=True) - zone_phone = models.TextField(blank=True, null=True) - zone_phone_ext = models.TextField(blank=True, null=True) - zone_fax = models.TextField(blank=True, null=True) - zone_fax_ext = models.TextField(blank=True, null=True) - zone_raw_text = models.TextField(blank=True, null=True) + whois_server = models.TextField(blank=True, null=True, help_text="") + registrar_name = models.TextField(blank=True, null=True, help_text="") + status = models.TextField(blank=True, null=True, help_text="") + clean_text = models.TextField(blank=True, null=True, help_text="") + raw_text = models.TextField(blank=True, null=True, help_text="") + registrant_name = models.TextField(blank=True, null=True, help_text="") + registrant_organization = models.TextField(blank=True, null=True, help_text="") + registrant_street = models.TextField(blank=True, null=True, help_text="") + registrant_city = models.TextField(blank=True, null=True, help_text="") + registrant_state = models.TextField(blank=True, null=True, help_text="") + registrant_post_code = models.TextField(blank=True, null=True, help_text="") + registrant_country = models.TextField(blank=True, null=True, help_text="") + registrant_email = models.TextField(blank=True, null=True, help_text="") + registrant_phone = models.TextField(blank=True, null=True, help_text="") + registrant_phone_ext = models.TextField(blank=True, null=True, help_text="") + registrant_fax = models.TextField(blank=True, null=True, help_text="") + registrant_fax_ext = models.TextField(blank=True, null=True, help_text="") + registrant_raw_text = models.TextField(blank=True, null=True, help_text="") + administrative_name = models.TextField(blank=True, null=True, help_text="") + administrative_organization = models.TextField(blank=True, null=True, help_text="") + administrative_street = models.TextField(blank=True, null=True, help_text="") + administrative_city = models.TextField(blank=True, null=True, help_text="") + administrative_state = models.TextField(blank=True, null=True, help_text="") + administrative_post_code = models.TextField(blank=True, null=True, help_text="") + administrative_country = models.TextField(blank=True, null=True, help_text="") + administrative_email = models.TextField(blank=True, null=True, help_text="") + administrative_phone = models.TextField(blank=True, null=True, help_text="") + administrative_phone_ext = models.TextField(blank=True, null=True, help_text="") + administrative_fax = models.TextField(blank=True, null=True, help_text="") + administrative_fax_ext = models.TextField(blank=True, null=True, help_text="") + administrative_raw_text = models.TextField(blank=True, null=True, help_text="") + technical_name = models.TextField(blank=True, null=True, help_text="") + technical_organization = models.TextField(blank=True, null=True, help_text="") + technical_street = models.TextField(blank=True, null=True, help_text="") + technical_city = models.TextField(blank=True, null=True, help_text="") + technical_state = models.TextField(blank=True, null=True, help_text="") + technical_post_code = models.TextField(blank=True, null=True, help_text="") + technical_country = models.TextField(blank=True, null=True, help_text="") + technical_email = models.TextField(blank=True, null=True, help_text="") + technical_phone = models.TextField(blank=True, null=True, help_text="") + technical_phone_ext = models.TextField(blank=True, null=True, help_text="") + technical_fax = models.TextField(blank=True, null=True, help_text="") + technical_fax_ext = models.TextField(blank=True, null=True, help_text="") + technical_raw_text = models.TextField(blank=True, null=True, help_text="") + billing_name = models.TextField(blank=True, null=True, help_text="") + billing_organization = models.TextField(blank=True, null=True, help_text="") + billing_street = models.TextField(blank=True, null=True, help_text="") + billing_city = models.TextField(blank=True, null=True, help_text="") + billing_state = models.TextField(blank=True, null=True, help_text="") + billing_post_code = models.TextField(blank=True, null=True, help_text="") + billing_country = models.TextField(blank=True, null=True, help_text="") + billing_email = models.TextField(blank=True, null=True, help_text="") + billing_phone = models.TextField(blank=True, null=True, help_text="") + billing_phone_ext = models.TextField(blank=True, null=True, help_text="") + billing_fax = models.TextField(blank=True, null=True, help_text="") + billing_fax_ext = models.TextField(blank=True, null=True, help_text="") + billing_raw_text = models.TextField(blank=True, null=True, help_text="") + zone_name = models.TextField(blank=True, null=True, help_text="") + zone_organization = models.TextField(blank=True, null=True, help_text="") + zone_street = models.TextField(blank=True, null=True, help_text="") + zone_city = models.TextField(blank=True, null=True, help_text="") + zone_state = models.TextField(blank=True, null=True, help_text="") + zone_post_code = models.TextField(blank=True, null=True, help_text="") + zone_country = models.TextField(blank=True, null=True, help_text="") + zone_email = models.TextField(blank=True, null=True, help_text="") + zone_phone = models.TextField(blank=True, null=True, help_text="") + zone_phone_ext = models.TextField(blank=True, null=True, help_text="") + zone_fax = models.TextField(blank=True, null=True, help_text="") + zone_fax_ext = models.TextField(blank=True, null=True, help_text="") + zone_raw_text = models.TextField(blank=True, null=True, help_text="") class Meta: """Set DnsRecords model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "dns_records" @@ -2112,24 +3743,42 @@ class Meta: class DomainAlerts(models.Model): """Define DomainAlerts model.""" - domain_alert_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + domain_alert_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for domain_alerts", + ) sub_domain = models.ForeignKey( - "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid" + "SubDomains", + on_delete=models.CASCADE, + db_column="sub_domain_uid", + help_text="FK: Foreign Key to sub_domains", ) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + organization_uid = models.UUIDField(help_text="FK: Foreign Key to organizations") + alert_type = models.TextField( + blank=True, null=True, help_text="Type of domain alert" ) - organization_uid = models.UUIDField() - alert_type = models.TextField(blank=True, null=True) - message = models.TextField(blank=True, null=True) - previous_value = models.TextField(blank=True, null=True) - new_value = models.TextField(blank=True, null=True) - date = models.DateField(blank=True, null=True) + message = models.TextField( + blank=True, null=True, help_text="Message description associated with alert" + ) + previous_value = models.TextField( + blank=True, null=True, help_text="Previous value associated with alert" + ) + new_value = models.TextField( + blank=True, null=True, help_text="New updated value associated with alert" + ) + date = models.DateField(blank=True, null=True, help_text="Date of alert") class Meta: """Set DomainAlerts model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "domain_alerts" unique_together = (("alert_type", "sub_domain", "date", "new_value"),) @@ -2138,23 +3787,59 @@ class Meta: class DomainPermutations(models.Model): """Define DomainPermutations model.""" - suspected_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + suspected_domain_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a DNSTwist domain permutation.", + ) organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization_uid" - ) - domain_permutation = models.TextField(blank=True, null=True) - ipv4 = models.TextField(blank=True, null=True) - ipv6 = models.TextField(blank=True, null=True) - mail_server = models.TextField(blank=True, null=True) - name_server = models.TextField(blank=True, null=True) - fuzzer = models.TextField(blank=True, null=True) - date_observed = models.DateField(blank=True, null=True) - ssdeep_score = models.TextField(blank=True, null=True) - malicious = models.BooleanField(blank=True, null=True) - blocklist_attack_count = models.IntegerField(blank=True, null=True) - blocklist_report_count = models.IntegerField(blank=True, null=True) + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="Foreign Key for a linked Organization", + ) + domain_permutation = models.TextField( + blank=True, + null=True, + help_text="Domain that has been flagged as a possible spoof of a domain owned by the stakeholder.", + ) + ipv4 = models.TextField( + blank=True, null=True, help_text="IPv4 associated with the identified domain" + ) + ipv6 = models.TextField( + blank=True, null=True, help_text="IPv6 associated with the identified domain" + ) + mail_server = models.TextField( + blank=True, null=True, help_text="Mail server seen on the domain." + ) + name_server = models.TextField( + blank=True, null=True, help_text="Name server seen on the domain." + ) + fuzzer = models.TextField(blank=True, null=True, help_text="Fuzzing technique used") + date_observed = models.DateField( + blank=True, null=True, help_text="Date domain permutation was observed" + ) + ssdeep_score = models.TextField( + blank=True, null=True, help_text="HTML similarity with fuzzy hashes" + ) + malicious = models.BooleanField( + blank=True, null=True, help_text="T/F Is subdomain malicious?" + ) + blocklist_attack_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of attacks reported in the Blocklist.de database", + ) + blocklist_report_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of reports reported in the Blocklist.de database", + ) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", ) sub_domain = models.ForeignKey( "SubDomains", @@ -2162,15 +3847,26 @@ class DomainPermutations(models.Model): db_column="sub_domain_uid", blank=True, null=True, + help_text="FK: Foreign Key to sub_domains", + ) + dshield_record_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of records reported in the DSheild database", + ) + dshield_attack_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of attacks reported in the DSHeild databse", + ) + date_active = models.DateField( + blank=True, null=True, help_text="Last known date permutation was active" ) - dshield_record_count = models.IntegerField(blank=True, null=True) - dshield_attack_count = models.IntegerField(blank=True, null=True) - date_active = models.DateField(blank=True, null=True) class Meta: """Set DomainPermutations model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "domain_permutations" unique_together = (("domain_permutation", "organization"),) @@ -2179,19 +3875,33 @@ class Meta: class DotgovDomains(models.Model): """Define DotgovDomains model.""" - dotgov_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - domain_name = models.TextField(unique=True) - domain_type = models.TextField(blank=True, null=True) - agency = models.TextField(blank=True, null=True) - organization = models.TextField(blank=True, null=True) - city = models.TextField(blank=True, null=True) - state = models.TextField(blank=True, null=True) - security_contact_email = models.TextField(blank=True, null=True) + dotgov_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for dotgov_domains", + ) + domain_name = models.TextField(unique=True, help_text="Name of the dotgov domain") + domain_type = models.TextField( + blank=True, null=True, help_text="Branch of govt. for dotgov domain" + ) + agency = models.TextField( + blank=True, null=True, help_text="Name of agency domain is associated with" + ) + organization = models.TextField( + blank=True, + null=True, + help_text="Name of organization domain is associated with", + ) + city = models.TextField(blank=True, null=True, help_text="City of organization") + state = models.TextField(blank=True, null=True, help_text="State of organization") + security_contact_email = models.TextField( + blank=True, null=True, help_text="Email of organization's security contact" + ) class Meta: """Set DotgovDomains model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "dotgov_domains" @@ -2199,96 +3909,111 @@ class Meta: class Executives(models.Model): """Define Executives model.""" - executives_uid = models.UUIDField(primary_key=True) + executives_uid = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for executives" + ) organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization" + "Organization", + on_delete=models.CASCADE, + db_column="organization", + help_text="FK: Foreign Key to organizations", ) - executives = models.TextField() + executives = models.TextField(help_text="Executive's name") class Meta: """Set Executives model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "executives" -# merged with vs's IP table -# class Ips(models.Model): -# """Define Ips model.""" - -# ip_hash = models.TextField(primary_key=True) -# ip = models.GenericIPAddressField(unique=True) -# origin_cidr = models.ForeignKey( -# Cidr, on_delete=models.CASCADE, db_column="origin_cidr", blank=True, null=True -# ) -# shodan_results = models.BooleanField(blank=True, null=True) -# live = models.BooleanField(blank=True, null=True) -# date_last_live = models.DateTimeField(blank=True, null=True) -# last_reverse_lookup = models.DateTimeField(blank=True, null=True) -# first_seen = models.DateField(blank=True, null=True) -# last_seen = models.DateField(blank=True, null=True) -# current = models.BooleanField(blank=True, null=True) -# from_cidr = models.BooleanField(blank=True, null=True) # varchar type in db??? -# organization_uid = models.UUIDField(blank=True, null=True) - -# class Meta: -# """Set Ips model metadata.""" - -# managed = False -# db_table = "ips" - -# replaced with manny to many field in ip table -# class IpsSubs(models.Model): -# """Define IpsSubs model.""" - -# ips_subs_uid = models.UUIDField(primary_key=True) -# ip_hash = models.ForeignKey(Ips, on_delete=models.CASCADE, db_column="ip_hash") -# sub_domain_uid = models.ForeignKey( -# "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid" -# ) - -# class Meta: -# """Set IpsSubs model metadata.""" - -# managed = False -# # db_table = 'ips_subs' -# unique_together = (("ip_hash", "sub_domain_uid"),) - - class Mentions(models.Model): """Define Mentions model.""" - mentions_uid = models.UUIDField(primary_key=True) - category = models.TextField(blank=True, null=True) - collection_date = models.TextField(blank=True, null=True) - content = models.TextField(blank=True, null=True) - creator = models.TextField(blank=True, null=True) - date = models.DateField(blank=True, null=True) - sixgill_mention_id = models.TextField(unique=True, blank=True, null=True) - post_id = models.TextField(blank=True, null=True) - lang = models.TextField(blank=True, null=True) - rep_grade = models.TextField(blank=True, null=True) - site = models.TextField(blank=True, null=True) - site_grade = models.TextField(blank=True, null=True) - title = models.TextField(blank=True, null=True) - type = models.TextField(blank=True, null=True) - url = models.TextField(blank=True, null=True) - comments_count = models.TextField(blank=True, null=True) - sub_category = models.TextField(blank=True, null=True) - tags = models.TextField(blank=True, null=True) - organization_uid = models.UUIDField() + mentions_uid = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for cyber sixgill mentions" + ) + category = models.TextField(blank=True, null=True, help_text="Category of mention") + collection_date = models.TextField( + blank=True, null=True, help_text="Date that mention was recorded" + ) + content = models.TextField( + blank=True, null=True, help_text="Content of mention incident" + ) + creator = models.TextField( + blank=True, null=True, help_text="User who created the mention" + ) + date = models.DateField( + blank=True, null=True, help_text="Date the mention was posted" + ) + sixgill_mention_id = models.TextField( + unique=True, + blank=True, + null=True, + help_text="Cybersixgill mention ID associated with mention incident", + ) + post_id = models.TextField( + blank=True, + null=True, + help_text="Cybersixgill post ID associated with mention incident", + ) + lang = models.TextField( + blank=True, null=True, help_text="Language of the mention post" + ) + rep_grade = models.TextField( + blank=True, + null=True, + help_text="Threat actors reputation score determined by cyber sixgill", + ) + site = models.TextField( + blank=True, null=True, help_text="Site were the mention occured" + ) + site_grade = models.TextField( + blank=True, null=True, help_text="Grade of site where mention occured 0 - 5" + ) + title = models.TextField( + blank=True, null=True, help_text="Title of post where mention occured" + ) + type = models.TextField( + blank=True, null=True, help_text="Type of post where mention occured" + ) + url = models.TextField(blank=True, null=True, help_text="URL of mention post") + comments_count = models.TextField( + blank=True, null=True, help_text="Number of comments on the mention post" + ) + sub_category = models.TextField( + blank=True, null=True, help_text="Subcategory of mention" + ) + tags = models.TextField( + blank=True, null=True, help_text="Tags associated with mention alert" + ) + organization_uid = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organizations", + ) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + title_translated = models.TextField( + blank=True, null=True, help_text="Title of mention post translated to english" + ) + content_translated = models.TextField( + blank=True, null=True, help_text="Content of mention post translated to english" + ) + detected_lang = models.TextField( + blank=True, null=True, help_text="Detected language of metion post" ) - title_translated = models.TextField(blank=True, null=True) - content_translated = models.TextField(blank=True, null=True) - detected_lang = models.TextField(blank=True, null=True) class Meta: """Set Mentions model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "mentions" @@ -2297,14 +4022,22 @@ class Meta: class OrgIdMap(models.Model): """Define OrgIdMap model.""" - cyhy_id = models.TextField(blank=True, null=True) - pe_org_id = models.TextField(blank=True, null=True) - merge_orgs = models.BooleanField(blank=True, null=True) + cyhy_id = models.TextField( + blank=True, null=True, help_text="Cyber Hygiene organization ID" + ) + pe_org_id = models.TextField( + blank=True, null=True, help_text="Posture & Exposure organization ID" + ) + merge_orgs = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field to flag if the orgs should be merged", + ) class Meta: """Set OrgIdMap model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "org_id_map" unique_together = (("cyhy_id", "pe_org_id"),) @@ -2313,13 +4046,17 @@ class Meta: class OrgType(models.Model): """Define OrgType model.""" - org_type_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) - org_type = models.TextField(blank=True, null=True) + org_type_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for organization type object.", + ) + org_type = models.TextField(blank=True, null=True, help_text="Organization type.") class Meta: """Set OrgType model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "org_type" @@ -2383,90 +4120,106 @@ class Meta: class PshttResults(models.Model): """Define PshttResults model.""" - pshtt_results_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + pshtt_results_uid = models.UUIDField( + primary_key=True, default=uuid.uuid1, help_text="" + ) organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, db_column="organization_uid" + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="", ) sub_domain = models.ForeignKey( - "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid" + "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid", help_text="" ) data_source = models.ForeignKey( - "DataSource", on_delete=models.CASCADE, db_column="data_source_uid" - ) - sub_domain = models.TextField() - date_scanned = models.DateField(blank=True, null=True) - base_domain = models.TextField(blank=True, null=True) - base_domain_hsts_preloaded = models.BooleanField(blank=True, null=True) - canonical_url = models.TextField(blank=True, null=True) - defaults_to_https = models.BooleanField(blank=True, null=True) - domain = models.TextField(blank=True, null=True) - domain_enforces_https = models.BooleanField(blank=True, null=True) - domain_supports_https = models.BooleanField(blank=True, null=True) - domain_uses_strong_hsts = models.BooleanField(blank=True, null=True) - downgrades_https = models.BooleanField(blank=True, null=True) - htss = models.BooleanField(blank=True, null=True) - hsts_entire_domain = models.BooleanField(blank=True, null=True) - hsts_header = models.TextField(blank=True, null=True) + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="", + ) + sub_domain = models.TextField(help_text="") + date_scanned = models.DateField(blank=True, null=True, help_text="") + base_domain = models.TextField(blank=True, null=True, help_text="") + base_domain_hsts_preloaded = models.BooleanField( + blank=True, null=True, help_text="" + ) + canonical_url = models.TextField(blank=True, null=True, help_text="") + defaults_to_https = models.BooleanField(blank=True, null=True, help_text="") + domain = models.TextField(blank=True, null=True, help_text="") + domain_enforces_https = models.BooleanField(blank=True, null=True, help_text="") + domain_supports_https = models.BooleanField(blank=True, null=True, help_text="") + domain_uses_strong_hsts = models.BooleanField(blank=True, null=True, help_text="") + downgrades_https = models.BooleanField(blank=True, null=True, help_text="") + htss = models.BooleanField(blank=True, null=True, help_text="") + hsts_entire_domain = models.BooleanField(blank=True, null=True, help_text="") + hsts_header = models.TextField(blank=True, null=True, help_text="") hsts_max_age = models.DecimalField( - max_digits=1000, decimal_places=1000, blank=True, null=True - ) - hsts_preload_pending = models.BooleanField(blank=True, null=True) - hsts_preload_ready = models.BooleanField(blank=True, null=True) - hsts_preloaded = models.BooleanField(blank=True, null=True) - https_bad_chain = models.BooleanField(blank=True, null=True) - https_bad_hostname = models.BooleanField(blank=True, null=True) - https_cert_chain_length = models.IntegerField(blank=True, null=True) - https_client_auth_required = models.BooleanField(blank=True, null=True) - https_custom_truststore_trusted = models.BooleanField(blank=True, null=True) - https_expired_cert = models.BooleanField(blank=True, null=True) - https_full_connection = models.BooleanField(blank=True, null=True) - https_live = models.BooleanField(blank=True, null=True) + max_digits=1000, decimal_places=1000, blank=True, null=True, help_text="" + ) + hsts_preload_pending = models.BooleanField(blank=True, null=True, help_text="") + hsts_preload_ready = models.BooleanField(blank=True, null=True, help_text="") + hsts_preloaded = models.BooleanField(blank=True, null=True, help_text="") + https_bad_chain = models.BooleanField(blank=True, null=True, help_text="") + https_bad_hostname = models.BooleanField(blank=True, null=True, help_text="") + https_cert_chain_length = models.IntegerField(blank=True, null=True, help_text="") + https_client_auth_required = models.BooleanField( + blank=True, null=True, help_text="" + ) + https_custom_truststore_trusted = models.BooleanField( + blank=True, null=True, help_text="" + ) + https_expired_cert = models.BooleanField(blank=True, null=True, help_text="") + https_full_connection = models.BooleanField(blank=True, null=True, help_text="") + https_live = models.BooleanField(blank=True, null=True, help_text="") https_probably_missing_intermediate_cert = models.BooleanField( - blank=True, null=True - ) - https_publicly_trusted = models.BooleanField(blank=True, null=True) - https_self_signed_cert = models.BooleanField(blank=True, null=True) - https_leaf_cert_expiration_date = models.DateField(blank=True, null=True) - https_leaf_cert_issuer = models.TextField(blank=True, null=True) - https_leaf_cert_subject = models.TextField(blank=True, null=True) - https_root_cert_issuer = models.TextField(blank=True, null=True) - ip = models.GenericIPAddressField(blank=True, null=True) - live = models.BooleanField(blank=True, null=True) - notes = models.TextField(blank=True, null=True) - redirect = models.BooleanField(blank=True, null=True) - redirect_to = models.TextField(blank=True, null=True) - server_header = models.TextField(blank=True, null=True) - server_version = models.TextField(blank=True, null=True) - strictly_forces_https = models.BooleanField(blank=True, null=True) - unknown_error = models.BooleanField(blank=True, null=True) - valid_https = models.BooleanField(blank=True, null=True) + blank=True, null=True, help_text="" + ) + https_publicly_trusted = models.BooleanField(blank=True, null=True, help_text="") + https_self_signed_cert = models.BooleanField(blank=True, null=True, help_text="") + https_leaf_cert_expiration_date = models.DateField( + blank=True, null=True, help_text="" + ) + https_leaf_cert_issuer = models.TextField(blank=True, null=True, help_text="") + https_leaf_cert_subject = models.TextField(blank=True, null=True, help_text="") + https_root_cert_issuer = models.TextField(blank=True, null=True, help_text="") + ip = models.GenericIPAddressField(blank=True, null=True, help_text="") + live = models.BooleanField(blank=True, null=True, help_text="") + notes = models.TextField(blank=True, null=True, help_text="") + redirect = models.BooleanField(blank=True, null=True, help_text="") + redirect_to = models.TextField(blank=True, null=True, help_text="") + server_header = models.TextField(blank=True, null=True, help_text="") + server_version = models.TextField(blank=True, null=True, help_text="") + strictly_forces_https = models.BooleanField(blank=True, null=True, help_text="") + unknown_error = models.BooleanField(blank=True, null=True, help_text="") + valid_https = models.BooleanField(blank=True, null=True, help_text="") ep_http_headers = models.TextField( - blank=True, null=True + blank=True, null=True, help_text="" ) # This field type is a guess. - ep_http_server_header = models.TextField(blank=True, null=True) - ep_http_server_version = models.TextField(blank=True, null=True) + ep_http_server_header = models.TextField(blank=True, null=True, help_text="") + ep_http_server_version = models.TextField(blank=True, null=True, help_text="") ep_https_headers = models.TextField( - blank=True, null=True + blank=True, null=True, help_text="" ) # This field type is a guess. - ep_https_hsts_header = models.TextField(blank=True, null=True) - ep_https_server_header = models.TextField(blank=True, null=True) - ep_https_server_version = models.TextField(blank=True, null=True) + ep_https_hsts_header = models.TextField(blank=True, null=True, help_text="") + ep_https_server_header = models.TextField(blank=True, null=True, help_text="") + ep_https_server_version = models.TextField(blank=True, null=True, help_text="") ep_httpswww_headers = models.TextField( - blank=True, null=True + blank=True, null=True, help_text="" ) # This field type is a guess. - ep_httpswww_hsts_header = models.TextField(blank=True, null=True) - ep_httpswww_server_header = models.TextField(blank=True, null=True) - ep_httpswww_server_version = models.TextField(blank=True, null=True) + ep_httpswww_hsts_header = models.TextField(blank=True, null=True, help_text="") + ep_httpswww_server_header = models.TextField(blank=True, null=True, help_text="") + ep_httpswww_server_version = models.TextField(blank=True, null=True, help_text="") ep_httpwww_headers = models.TextField( - blank=True, null=True + blank=True, null=True, help_text="" ) # This field type is a guess. - ep_httpwww_server_header = models.TextField(blank=True, null=True) - ep_httpwww_server_version = models.TextField(blank=True, null=True) + ep_httpwww_server_header = models.TextField(blank=True, null=True, help_text="") + ep_httpwww_server_version = models.TextField(blank=True, null=True, help_text="") class Meta: """Set PshttResults model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "pshtt_results" unique_together = (("organization", "sub_domain"),) @@ -2475,44 +4228,53 @@ class Meta: class PeReportSummaryStats(models.Model): """Define ReportSummaryStats model.""" - report_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + report_uid = models.UUIDField(primary_key=True, default=uuid.uuid1, help_text="") organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid" - ) - start_date = models.DateField() - end_date = models.DateField(blank=True, null=True) - ip_count = models.IntegerField(blank=True, null=True) - root_count = models.IntegerField(blank=True, null=True) - sub_count = models.IntegerField(blank=True, null=True) - ports_count = models.IntegerField(blank=True, null=True) - creds_count = models.IntegerField(blank=True, null=True) - breach_count = models.IntegerField(blank=True, null=True) - cred_password_count = models.IntegerField(blank=True, null=True) - domain_alert_count = models.IntegerField(blank=True, null=True) - suspected_domain_count = models.IntegerField(blank=True, null=True) - insecure_port_count = models.IntegerField(blank=True, null=True) - verified_vuln_count = models.IntegerField(blank=True, null=True) - suspected_vuln_count = models.IntegerField(blank=True, null=True) - suspected_vuln_addrs_count = models.IntegerField(blank=True, null=True) - threat_actor_count = models.IntegerField(blank=True, null=True) - dark_web_alerts_count = models.IntegerField(blank=True, null=True) - dark_web_mentions_count = models.IntegerField(blank=True, null=True) - dark_web_executive_alerts_count = models.IntegerField(blank=True, null=True) - dark_web_asset_alerts_count = models.IntegerField(blank=True, null=True) - pe_number_score = models.TextField(blank=True, null=True) - pe_letter_grade = models.TextField(blank=True, null=True) + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="", + ) + start_date = models.DateField(help_text="") + end_date = models.DateField(blank=True, null=True, help_text="") + ip_count = models.IntegerField(blank=True, null=True, help_text="") + root_count = models.IntegerField(blank=True, null=True, help_text="") + sub_count = models.IntegerField(blank=True, null=True, help_text="") + ports_count = models.IntegerField(blank=True, null=True, help_text="") + creds_count = models.IntegerField(blank=True, null=True, help_text="") + breach_count = models.IntegerField(blank=True, null=True, help_text="") + cred_password_count = models.IntegerField(blank=True, null=True, help_text="") + domain_alert_count = models.IntegerField(blank=True, null=True, help_text="") + suspected_domain_count = models.IntegerField(blank=True, null=True, help_text="") + insecure_port_count = models.IntegerField(blank=True, null=True, help_text="") + verified_vuln_count = models.IntegerField(blank=True, null=True, help_text="") + suspected_vuln_count = models.IntegerField(blank=True, null=True, help_text="") + suspected_vuln_addrs_count = models.IntegerField( + blank=True, null=True, help_text="" + ) + threat_actor_count = models.IntegerField(blank=True, null=True, help_text="") + dark_web_alerts_count = models.IntegerField(blank=True, null=True, help_text="") + dark_web_mentions_count = models.IntegerField(blank=True, null=True, help_text="") + dark_web_executive_alerts_count = models.IntegerField( + blank=True, null=True, help_text="" + ) + dark_web_asset_alerts_count = models.IntegerField( + blank=True, null=True, help_text="" + ) + pe_number_score = models.TextField(blank=True, null=True, help_text="") + pe_letter_grade = models.TextField(blank=True, null=True, help_text="") pe_percent_score = models.DecimalField( - max_digits=1000, decimal_places=1000, blank=True, null=True + max_digits=1000, decimal_places=1000, blank=True, null=True, help_text="" ) - cidr_count = models.IntegerField(blank=True, null=True) - port_protocol_count = models.IntegerField(blank=True, null=True) - software_count = models.IntegerField(blank=True, null=True) - foreign_ips_count = models.IntegerField(blank=True, null=True) + cidr_count = models.IntegerField(blank=True, null=True, help_text="") + port_protocol_count = models.IntegerField(blank=True, null=True, help_text="") + software_count = models.IntegerField(blank=True, null=True, help_text="") + foreign_ips_count = models.IntegerField(blank=True, null=True, help_text="") class Meta: """Set ReportSummaryStats model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "pe_report_summary_stats" unique_together = (("organization", "start_date"),) @@ -2521,21 +4283,37 @@ class Meta: class RootDomains(models.Model): """Define RootDomains model.""" - root_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + root_domain_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for root domains", + ) organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid" + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + root_domain = models.TextField(help_text="Root domain") + ip_address = models.TextField( + blank=True, null=True, help_text="IP address of root domain" ) - root_domain = models.TextField() - ip_address = models.TextField(blank=True, null=True) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + enumerate_subs = models.BooleanField( + blank=True, + null=True, + help_text="T/F should we identify subdomains for this root domain? (We don't enumerate for Cloud provider roots)", ) - enumerate_subs = models.BooleanField(blank=True, null=True) class Meta: """Set RootDomains model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "root_domains" unique_together = (("root_domain", "organization"),) @@ -2544,19 +4322,35 @@ class Meta: class PeTeamMembers(models.Model): """Define TeamMembers model.""" - team_member_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - team_member_fname = models.TextField(blank=False, null=False) - team_member_lname = models.TextField(blank=False, null=False) - team_member_email = models.TextField(blank=False, null=False) - team_member_ghID = models.TextField(blank=False, null=False) - team_member_phone = models.TextField(blank=True, null=True) - team_member_role = models.TextField(blank=True, null=True) - team_member_notes = models.TextField(blank=True, null=True) + team_member_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a PE Team Member object.", + ) + team_member_fname = models.TextField( + blank=False, null=False, help_text="First name." + ) + team_member_lname = models.TextField(blank=False, null=False, help_text="Last Name") + team_member_email = models.TextField( + blank=False, null=False, help_text="Team member's email address." + ) + team_member_ghID = models.TextField( + blank=False, null=False, help_text="Team member's github ID." + ) + team_member_phone = models.TextField( + blank=True, null=True, help_text="Team member's phone number." + ) + team_member_role = models.TextField( + blank=True, null=True, help_text="Team member's role." + ) + team_member_notes = models.TextField( + blank=True, null=True, help_text="Notes about the team member." + ) class Meta: """Set TeamMembers model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "pe_team_members" @@ -2564,47 +4358,71 @@ class Meta: class ShodanAssets(models.Model): """Define ShodanAssets model.""" - shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + shodan_asset_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for shodan assets", + ) organization = models.ForeignKey( Organization, on_delete=models.CASCADE, db_column="organization_uid", blank=True, null=True, + help_text="FK: Foreign Key to organizations", ) # If you still need to store the organization name or acronym, use a separate field for that organization_name = models.TextField( - blank=True, null=True + blank=True, null=True, help_text="Organization name" ) # New field to store the name or acronym - ip = models.TextField(blank=True, null=True) - port = models.IntegerField(blank=True, null=True) - protocol = models.TextField(blank=True, null=True) - timestamp = models.DateTimeField(blank=True, null=True) - product = models.TextField(blank=True, null=True) - server = models.TextField(blank=True, null=True) - tags = models.JSONField(blank=True, null=True) # Store tags as a list (JSON format) + ip = models.TextField(blank=True, null=True, help_text="IP address") + port = models.IntegerField(blank=True, null=True, help_text="Port number") + protocol = models.TextField( + blank=True, null=True, help_text="Protocol running on the port" + ) + timestamp = models.DateTimeField( + blank=True, null=True, help_text="Time the asset was last seen by Shodan" + ) + product = models.TextField( + blank=True, null=True, help_text="What product is running on the asset" + ) + server = models.TextField( + blank=True, null=True, help_text="What server is running on the asset" + ) + tags = models.JSONField( + blank=True, + null=True, + help_text="shodan tags associated with the asset (ex. self-signed, vpn, starttls, cloud, etc.)", + ) # Store tags as a list (JSON format) domains = models.JSONField( - blank=True, null=True + blank=True, null=True, help_text="domains associated with the asset" ) # Store domains as a list (JSON format) hostnames = models.JSONField( - blank=True, null=True + blank=True, null=True, help_text="hostnames associated with the asset" ) # Store hostnames as a list (JSON format) - isn = models.TextField(blank=True, null=True) - asn = models.IntegerField(blank=True, null=True) + isp = models.TextField(blank=True, null=True, help_text="Internet service provider") + asn = models.IntegerField( + blank=True, null=True, help_text="Autonomous system number" + ) data_source = models.ForeignKey( DataSource, on_delete=models.CASCADE, db_column="data_source_uid", blank=True, null=True, + help_text="FK: Foreign Key to data_source", + ) + country_code = models.TextField( + blank=True, null=True, help_text="Country code where the IP was located." + ) + location = models.TextField( + blank=True, null=True, help_text="Location where the IP hosted." ) - country_code = models.TextField(blank=True, null=True) - location = models.TextField(blank=True, null=True) class Meta: """Set ShodanAssets model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "shodan_assets" unique_together = (("organization", "ip", "port", "protocol", "timestamp"),) @@ -2652,63 +4470,148 @@ class Meta: class ShodanVulns(models.Model): """Define ShodanVulns model.""" - shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) + shodan_vuln_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a shodan vulnerability object.", + ) organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid" - ) - organization_name = models.TextField(blank=True, null=True) - ip = models.TextField(blank=True, null=True) - port = models.TextField(blank=True, null=True) - protocol = models.TextField(blank=True, null=True) - timestamp = models.DateTimeField(blank=True, null=True) - cve = models.TextField(blank=True, null=True) - severity = models.TextField(blank=True, null=True) + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + organization_name = models.TextField( + blank=True, null=True, help_text="Organization name" + ) + ip = models.TextField(blank=True, null=True, help_text="IP address") + port = models.TextField(blank=True, null=True, help_text="Port number") + protocol = models.TextField(blank=True, null=True, help_text="Protocol") + timestamp = models.DateTimeField( + blank=True, + null=True, + help_text="Timestamp when unverified vulnerability was found", + ) + cve = models.TextField( + blank=True, null=True, help_text="CVE associated with vulnerability" + ) + severity = models.TextField( + blank=True, + null=True, + help_text="Severity of vulnerability (medium, high, critical)", + ) cvss = models.DecimalField( - max_digits=1000, decimal_places=1000, blank=True, null=True - ) - summary = models.TextField(blank=True, null=True) - product = models.TextField(blank=True, null=True) - attack_vector = models.TextField(blank=True, null=True) - av_description = models.TextField(blank=True, null=True) - attack_complexity = models.TextField(blank=True, null=True) - ac_description = models.TextField(blank=True, null=True) - confidentiality_impact = models.TextField(blank=True, null=True) - ci_description = models.TextField(blank=True, null=True) - integrity_impact = models.TextField(blank=True, null=True) - ii_description = models.TextField(blank=True, null=True) - availability_impact = models.TextField(blank=True, null=True) - ai_description = models.TextField(blank=True, null=True) + max_digits=1000, + decimal_places=1000, + blank=True, + null=True, + help_text="Common Vulnerability Scoring System Score", + ) + summary = models.TextField( + blank=True, null=True, help_text="Summary of vulnerability" + ) + product = models.TextField( + blank=True, null=True, help_text="Product associated with vulnerability" + ) + attack_vector = models.TextField( + blank=True, null=True, help_text="Attack vector of vulnerability" + ) + av_description = models.TextField( + blank=True, null=True, help_text="Description of attack vector" + ) + attack_complexity = models.TextField( + blank=True, null=True, help_text="Complexity of attack (low, medium, high)" + ) + ac_description = models.TextField( + blank=True, null=True, help_text="Description of attack complexity" + ) + confidentiality_impact = models.TextField( + blank=True, + null=True, + help_text="Impact on confidentiality (none, partial, complete)", + ) + ci_description = models.TextField( + blank=True, null=True, help_text="Description of confidentiality impact" + ) + integrity_impact = models.TextField( + blank=True, null=True, help_text="Impact on integrity (none, partial complete)" + ) + ii_description = models.TextField( + blank=True, null=True, help_text="Description of integrity impact" + ) + availability_impact = models.TextField( + blank=True, + null=True, + help_text="Impact on availability (none, partial, complete)", + ) + ai_description = models.TextField( + blank=True, null=True, help_text="Description of availability impact" + ) tags = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of tags associated with vulnerability", ) # This field type is a guess. domains = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of domains associated with vulnerability", ) # This field type is a guess. hostnames = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="Host names associated with vulnerability", ) # This field type is a guess. - isn = models.TextField(blank=True, null=True) - asn = models.IntegerField(blank=True, null=True) + isp = models.TextField(blank=True, null=True, help_text="Internet service provider") + asn = models.IntegerField( + blank=True, null=True, help_text="Autonomous system number" + ) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + null=True, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", ) - type = models.TextField(blank=True, null=True) - name = models.TextField(blank=True, null=True) + type = models.TextField(blank=True, null=True, help_text="Type of vulnerability") + name = models.TextField(blank=True, null=True, help_text="Name of vulnerability") potential_vulns = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of potential vulnerabilities associated with vulnerability", ) # This field type is a guess. - mitigation = models.TextField(blank=True, null=True) - server = models.TextField(blank=True, null=True) - is_verified = models.BooleanField(blank=True, null=True) - banner = models.TextField(blank=True, null=True) - version = models.TextField(blank=True, null=True) - mitigation = models.TextField(blank=True, null=True) - cpe = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + mitigation = models.TextField( + blank=True, null=True, help_text="Information on how to mitigate vulnerability" + ) + server = models.TextField( + blank=True, null=True, help_text="Server associated with vulnerability" + ) + is_verified = models.BooleanField( + blank=True, null=True, help_text="T/F Is this a verified vulnerability?" + ) + banner = models.TextField( + blank=True, + null=True, + help_text="Snippet of information retrieved by Shodan about a service or device, typically revealing details like the software, version, and configuration of a system exposed to the internet.", + ) + version = models.TextField( + blank=True, null=True, help_text="Version of the server running.???" + ) + cpe = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="Common Platform Enumeration (CPE) id for the product the vulnerability was found on.", + ) class Meta: """Set ShodanVulns model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "shodan_vulns" unique_together = (("organization", "ip", "port", "protocol", "timestamp"),) @@ -2717,13 +4620,25 @@ class Meta: class SubDomains(models.Model): """Define SubDomains model.""" - sub_domain_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) - sub_domain = models.TextField() # Crossfeed Domains name field + sub_domain_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for subdomains", + ) + sub_domain = models.TextField( + help_text="Subdomain name" + ) # Crossfeed Domains name field root_domain = models.ForeignKey( - RootDomains, on_delete=models.CASCADE, db_column="root_domain_uid" + RootDomains, + on_delete=models.CASCADE, + db_column="root_domain_uid", + help_text="FK: Foreign Key to root domains", ) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", ) dns_record = models.ForeignKey( DnsRecords, @@ -2731,51 +4646,115 @@ class SubDomains(models.Model): db_column="dns_record_uid", blank=True, null=True, + help_text="FK: Foreign Key to dns record", + ) + status = models.BooleanField( + blank=True, + null=True, + help_text="T/F: Boolean field flagging if the status is active.???", + ) + first_seen = models.DateField( + blank=True, + null=True, + help_text="Date and time of the first time teh subdomain was seen.", + ) + last_seen = models.DateField( + blank=True, null=True, help_text="Date of the last time the subdomain was seen." + ) + created_at = models.DateTimeField( + db_column="created_at", help_text="Datetime the subdomain object was created." + ) + updated_at = models.DateTimeField( + db_column="updated_at", help_text="Datetime the subdomain was last updated." + ) + current = models.BooleanField( + blank=True, + null=True, + help_text="T/F is this sub domain still live and linked to the organization", + ) + identified = models.BooleanField( + blank=True, + null=True, + help_text="T/F was this subdomain identified via an IP lookup.???", ) - status = models.BooleanField(blank=True, null=True) - first_seen = models.DateField(blank=True, null=True) - last_seen = models.DateField(blank=True, null=True) - created_at = models.DateTimeField(db_column="created_at") - updated_at = models.DateTimeField(db_column="updated_at") - current = models.BooleanField(blank=True, null=True) - identified = models.BooleanField(blank=True, null=True) - ip_address = models.TextField(blank=True, null=True) # XFD column + ip_address = models.TextField( + blank=True, null=True, help_text="IP address linked to the subdomain" + ) # XFD column synced_at = models.DateTimeField( - db_column="synced_at", blank=True, null=True + db_column="synced_at", + blank=True, + null=True, + help_text="Date the subdomain was last synced", ) # XFD column from_root_domain = models.TextField( - db_column="from_root_domain", blank=True, null=True + db_column="from_root_domain", + blank=True, + null=True, + help_text="Root domain associated with the subdomain", ) # XFD column subdomain_source = models.TextField( - db_column="subdomain_source", max_length=255, blank=True, null=True + db_column="subdomain_source", + max_length=255, + blank=True, + null=True, + help_text="Where teh subdomain originated from.", ) # XFD column organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid" + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", ) - ip_only = models.BooleanField(db_column="ip_only", default=False) # XFD column + ip_only = models.BooleanField( + db_column="ip_only", + default=False, + help_text="T/F if there is no subdomain but just an IP", + ) # XFD column reverse_name = models.CharField( - db_column="reverse_name", max_length=512 + db_column="reverse_name", + max_length=512, + help_text="DNS reverse lookup of a subdomain", ) # XFD column screenshot = models.CharField( - max_length=512, blank=True, null=True + max_length=512, + blank=True, + null=True, + help_text="link to the screenshot of the subdomain site.???", ) # XFD Crossfeed Domains screenshot field - country = models.CharField(max_length=255, blank=True, null=True) # XFD column - asn = models.CharField(max_length=255, blank=True, null=True) # XFD column + country = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Country where the subdomain is hosted.", + ) # XFD column + asn = models.CharField( + max_length=255, blank=True, null=True, help_text="Autonomous system number" + ) # XFD column cloud_hosted = models.BooleanField( - db_column="cloud_hosted", default=False + db_column="cloud_hosted", + default=False, + help_text="T/F is this subdomain cloud hosted", ) # XFD column - ssl = models.JSONField(blank=True, null=True) # XFD columnv + ssl = models.JSONField( + blank=True, + null=True, + help_text="SSL (Secure Sockets Layer) or TLS (Transport Layer Security) connection, certificate, or related security features.", + ) # XFD columnv censys_certificates_results = models.JSONField( - db_column="censys_certificates_results", default=dict + db_column="censys_certificates_results", + default=dict, + help_text="Results from the censys certificate scan.", ) # XFD column trustymail_results = models.JSONField( - db_column="trustymail_results", default=dict + db_column="trustymail_results", + default=dict, + help_text="Results from the trustymail scan.", ) # XFD column class Meta: """Set SubDomains model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "sub_domains" unique_together = (("sub_domain", "root_domain"),) @@ -2784,20 +4763,41 @@ class Meta: class TopCves(models.Model): """Define TopCves model.""" - top_cves_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) - cve_id = models.TextField(blank=True, null=True) - dynamic_rating = models.TextField(blank=True, null=True) - nvd_base_score = models.TextField(blank=True, null=True) - date = models.DateField(blank=True, null=True) - summary = models.TextField(blank=True, null=True) + top_cves_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: unique identifier for top cves", + ) + cve_id = models.TextField( + blank=True, null=True, help_text="CVE identifier ex. CVE-202-20038" + ) + dynamic_rating = models.TextField( + blank=True, + null=True, + help_text="CyberSixGill’s Dynamic Vulnerability Exploit (DVE) Score", + ) + nvd_base_score = models.TextField( + blank=True, + null=True, + help_text="Base CVE score from National vulnerability Databse", + ) + date = models.DateField( + blank=True, + null=True, + help_text="Date the CVE was fetched from the Cybersixgill API", + ) + summary = models.TextField(blank=True, null=True, help_text="Summary of the CVE") data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", ) class Meta: """Set TopCves model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "top_cves" unique_together = (("cve_id", "date"),) @@ -2807,15 +4807,23 @@ class Meta: class TopicTotals(models.Model): """Define TopicTotals model.""" - count_uuid = models.UUIDField(primary_key=True, default=uuid.uuid1()) - organization_uid = models.UUIDField() - content_count = models.IntegerField() - count_date = models.TextField(blank=True, null=True) + count_uuid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for topic_totals", + ) + organization_uid = models.UUIDField(help_text="FK: Foreign Key to organizations") + content_count = models.IntegerField( + help_text="Number dark web mentions that fit into a NLP topic" + ) + count_date = models.TextField( + blank=True, null=True, help_text="Date the count was taken" + ) class Meta: """Set TopicTotals model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "topic_totals" @@ -2825,14 +4833,19 @@ class UniqueSoftware(models.Model): """Define UniqueSoftware model.""" field_id = models.UUIDField( - db_column="_id", primary_key=True, default=uuid.uuid1() + db_column="_id", + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a unique software object.", ) # Field renamed because it started with '_'. - software_name = models.TextField(blank=False, null=False) + software_name = models.TextField( + blank=False, null=False, help_text="Name of the software." + ) class Meta: """Set UniqueSoftware model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "unique_software" @@ -2840,26 +4853,50 @@ class Meta: class WebAssets(models.Model): """Define WebAssets model.""" - asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) - asset_type = models.TextField() - asset = models.TextField() - ip_type = models.TextField(blank=True, null=True) - verified = models.BooleanField(blank=True, null=True) + asset_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifer for a web asset object.", + ) + asset_type = models.TextField(help_text="Type of web asset.") + asset = models.TextField(help_text="The web asset owned by the organization.") + ip_type = models.TextField( + blank=True, null=True, help_text="Type of IP if the asset is an IP" + ) + verified = models.BooleanField( + blank=True, null=True, help_text="T/F if the asset is verified or not." + ) organization = models.ForeignKey( - Organization, on_delete=models.CASCADE, db_column="organization_uid" + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to the organization", + ) + asset_origin = models.TextField( + blank=True, null=True, help_text="Where the asset originated from." + ) + report_on = models.BooleanField( + blank=True, + null=True, + help_text="Whetherr or not PE should report on findings related to this asset.", + ) + last_scanned = models.DateTimeField( + blank=True, null=True, help_text="Last date the asset was scanned." + ) + report_status_reason = models.TextField( + blank=True, null=True, help_text="Reason the asset is not being reported on." ) - asset_origin = models.TextField(blank=True, null=True) - report_on = models.BooleanField(blank=True, null=True) - last_scanned = models.DateTimeField(blank=True, null=True) - report_status_reason = models.TextField(blank=True, null=True) data_source = models.ForeignKey( - DataSource, on_delete=models.CASCADE, db_column="data_source_uid" + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to the data source that created the web asset object.", ) class Meta: """Set WebAssets model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "web_assets" unique_together = (("asset", "organization"),) @@ -2868,25 +4905,49 @@ class Meta: class WeeklyStatusesMdl(models.Model): """Define WeeklyStatusesMdl model.""" - weekly_status_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - user_status = models.TextField(blank=True) - key_accomplishments = models.TextField(blank=True, null=True) - ongoing_task = models.TextField() - upcoming_task = models.TextField() - obstacles = models.TextField(blank=True, null=True) - non_standard_meeting = models.TextField(blank=True, null=True) - deliverables = models.TextField(blank=True, null=True) - pto = models.TextField(blank=True, null=True) - week_ending = models.DateField() - notes = models.TextField(blank=True, null=True) - statusComplete = models.IntegerField(blank=True, null=True) + weekly_status_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier to the weekly status object", + ) + user_status = models.TextField(blank=True, help_text="Name of the user.???") + key_accomplishments = models.TextField( + blank=True, + null=True, + help_text="Key accomplishments the user has made this week.", + ) + ongoing_task = models.TextField(help_text="Ongoing tasks the user is working on.") + upcoming_task = models.TextField( + help_text="Tasks the user has planned on starting." + ) + obstacles = models.TextField( + blank=True, + null=True, + help_text="Obstacles that the user is currently facing in accomplishing their tasks.", + ) + non_standard_meeting = models.TextField( + blank=True, + null=True, + help_text="Any non standard meetings during the last week.", + ) + deliverables = models.TextField( + blank=True, null=True, help_text="Key deliverables turned in by the user." + ) + pto = models.TextField(blank=True, null=True, help_text="Any upcoming PTO.") + week_ending = models.DateField(help_text="Last day of the week.") + notes = models.TextField(blank=True, null=True, help_text="additional notes") + statusComplete = models.IntegerField( + blank=True, + null=True, + help_text="T/F if the user has completed the status report.", + ) class Meta: """Set WeeklyStatusesMdl model metadata.""" # unique_together = (('week_ending', 'user_status'),) - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "weekly_statuses_mdl" @@ -2895,15 +4956,23 @@ class Meta: class CyhyKevs(models.Model): """Define CyhyKevs model.""" - cyhy_kevs_uid = models.UUIDField(primary_key=True) - kev = models.CharField(blank=True, null=True, max_length=255) - first_seen = models.DateField(blank=True, null=True) - last_seen = models.DateField(blank=True, null=True) + cyhy_kevs_uid = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier of the cyhy kev object." + ) + kev = models.CharField( + blank=True, null=True, max_length=255, help_text="CVE id of the KEV." + ) + first_seen = models.DateField( + blank=True, null=True, help_text="First time the KEV was seen." + ) + last_seen = models.DateField( + blank=True, null=True, help_text="Last time the KEV was seen." + ) class Meta: """Set CyhyKevs model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cyhy_kevs" @@ -2911,8 +4980,14 @@ class Meta: class XpanseBusinessUnits(models.Model): """Define XpanseBusinessUnits model.""" - xpanse_business_unit_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - entity_name = models.TextField(unique=True, blank=True, null=True) + xpanse_business_unit_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a xpanse business unit object.", + ) + entity_name = models.TextField( + unique=True, blank=True, null=True, help_text="Name of the business unit." + ) cyhy_db_name = models.ForeignKey( "Organization", on_delete=models.CASCADE, @@ -2920,19 +4995,34 @@ class XpanseBusinessUnits(models.Model): to_field="acronym", null=True, # Allow NULL values blank=True, + help_text="Acronym of the organization associated with the business unit.", + ) + state = models.TextField( + blank=True, null=True, help_text="State where the business unit is based." + ) + county = models.TextField( + blank=True, null=True, help_text="County where the business unit is based." + ) + city = models.TextField( + blank=True, null=True, help_text="City where the business unit is based." + ) + sector = models.TextField( + blank=True, null=True, help_text="Business unit's sector." + ) + entity_type = models.TextField( + blank=True, null=True, help_text="Type of business unit." + ) + region = models.TextField( + blank=True, null=True, help_text="Region where the business unit is based." + ) + rating = models.IntegerField( + blank=True, null=True, help_text="Xpanse rating of the business unit." ) - state = models.TextField(blank=True, null=True) - county = models.TextField(blank=True, null=True) - city = models.TextField(blank=True, null=True) - sector = models.TextField(blank=True, null=True) - entity_type = models.TextField(blank=True, null=True) - region = models.TextField(blank=True, null=True) - rating = models.IntegerField(blank=True, null=True) class Meta: """Set XpanseBusinessUnits metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "xpanse_business_units" @@ -2940,52 +5030,117 @@ class Meta: class XpanseAssetsMdl(models.Model): """Define XpanseAssetsMdl model.""" - xpanse_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - asm_id = models.TextField(unique=True, blank=False, null=False) - asset_name = models.TextField(blank=True, null=True) - asset_type = models.TextField(blank=True, null=True) - last_observed = models.DateTimeField(blank=True, null=True) - first_observed = models.DateTimeField(blank=True, null=True) + xpanse_asset_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for an Xpanse Asset object.", + ) + asm_id = models.TextField( + unique=True, blank=False, null=False, help_text="Xpanse ID for the asset" + ) + asset_name = models.TextField(blank=True, null=True, help_text="Name of the asset") + asset_type = models.TextField(blank=True, null=True, help_text="Type of asset") + last_observed = models.DateTimeField( + blank=True, null=True, help_text="Last datetime that the asset was observed" + ) + first_observed = models.DateTimeField( + blank=True, null=True, help_text="First datetime the asset was observed" + ) externally_detected_providers = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of externally detected providers.", + ) + created = models.DateTimeField( + blank=True, null=True, help_text="Datetime the asset was created" + ) + ips = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of Ips associated with the asset.", ) - created = models.DateTimeField(blank=True, null=True) - ips = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) active_external_services_types = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of active external services running on the asset.", + ) + domain = models.TextField( + blank=True, null=True, help_text="Domain associated with the asset." + ) + certificate_issuer = models.TextField( + blank=True, null=True, help_text="Certificate issuer." + ) + certificate_algorithm = models.TextField( + blank=True, null=True, help_text="Certificate algorithm" ) - domain = models.TextField(blank=True, null=True) - certificate_issuer = models.TextField(blank=True, null=True) - certificate_algorithm = models.TextField(blank=True, null=True) certificate_classifications = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of certificate classifications", + ) + resolves = models.BooleanField( + blank=True, null=True, help_text="T/F does the domain resolve to an live site." ) - resolves = models.BooleanField(blank=True, null=True) # details - top_level_asset_mapper_domain = models.TextField(blank=True, null=True) - domain_asset_type = models.JSONField(blank=True, null=True) - is_paid_level_domain = models.BooleanField(blank=True, null=True) - domain_details = models.JSONField(blank=True, null=True) - dns_zone = models.TextField(blank=True, null=True) - latest_sampled_ip = models.IntegerField(blank=True, null=True) - - recent_ips = models.JSONField(blank=True, null=True) - external_services = models.JSONField(blank=True, null=True) + top_level_asset_mapper_domain = models.TextField( + blank=True, null=True, help_text="The top level domain the subdomain maps to" + ) + domain_asset_type = models.JSONField( + blank=True, null=True, help_text="Type of the domain asset" + ) + is_paid_level_domain = models.BooleanField( + blank=True, null=True, help_text="T/F is the asset a paid level domain" + ) + domain_details = models.JSONField( + blank=True, null=True, help_text="Details about the domain" + ) + dns_zone = models.TextField( + blank=True, null=True, help_text="What zone does the dns resolve to." + ) + latest_sampled_ip = models.IntegerField( + blank=True, null=True, help_text="Latest IP seen on the domain" + ) + + recent_ips = models.JSONField( + blank=True, null=True, help_text="List of recent IPs linked to the domain" + ) + external_services = models.JSONField( + blank=True, null=True, help_text="External services running on the asset" + ) externally_inferred_vulnerability_score = models.DecimalField( - max_digits=5, decimal_places=2, blank=True, null=True + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="Externally inferred vulnerability score", ) externally_inferred_cves = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Externally inferred CVEs", ) explainers = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Explainer text", + ) + tags = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Tags associated with the asset", ) - tags = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) class Meta: """Set XpanseAssetsMdl metdata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "xpanse_assets_mdl" @@ -2993,21 +5148,40 @@ class Meta: class XpanseCvesMdl(models.Model): """Define XpanseCvesMdl model.""" - xpanse_cve_uid = models.UUIDField(unique=True, primary_key=True, default=uuid.uuid1) - cve_id = models.TextField(unique=True, blank=True, null=True) + xpanse_cve_uid = models.UUIDField( + unique=True, + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for an Xpanse CVE objct.", + ) + cve_id = models.TextField( + unique=True, blank=True, null=True, help_text="CVE identifier." + ) cvss_score_v2 = models.DecimalField( - max_digits=5, decimal_places=2, blank=True, null=True + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="CVVS Score version 2", + ) + cve_severity_v2 = models.TextField( + blank=True, null=True, help_text="CVSS Severity Score version 2" ) - cve_severity_v2 = models.TextField(blank=True, null=True) cvss_score_v3 = models.DecimalField( - max_digits=5, decimal_places=2, blank=True, null=True + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="CVSS Score version 3", + ) + cve_severity_v3 = models.TextField( + blank=True, null=True, help_text="CVSS Severity Score version 3" ) - cve_severity_v3 = models.TextField(blank=True, null=True) class Meta: """Set XpanseCvesMdl metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "xpanse_cves_mdl" @@ -3015,44 +5189,107 @@ class Meta: class XpanseServicesMdl(models.Model): """Define XpanseServicesMdl model.""" - xpanse_service_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - service_id = models.TextField(unique=True, blank=True, null=True) - service_name = models.TextField(blank=True, null=True) - service_type = models.TextField(blank=True, null=True) + xpanse_service_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a Xpanse Service object.", + ) + service_id = models.TextField( + unique=True, + blank=True, + null=True, + help_text="Xpanse Identifier for the service.", + ) + service_name = models.TextField( + blank=True, null=True, help_text="Name of the service." + ) + service_type = models.TextField(blank=True, null=True, help_text="Type of service") ip_address = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of IP addresses where the service is hosted, if applicable.", + ) + domain = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of domains where the service is hosted, if applicable.", ) - domain = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) externally_detected_providers = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of externally detected providers.", + ) + is_active = models.TextField( + blank=True, null=True, help_text="State of the service (Active, Inactive)." + ) + first_observed = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the service was first observed by Xpanse", + ) + last_observed = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the service was last observed by Xpanse", + ) + port = models.IntegerField( + blank=True, + null=True, + help_text="Number of the port where the service is running.", + ) + protocol = models.TextField( + blank=True, null=True, help_text="Protocol running on the port." ) - is_active = models.TextField(blank=True, null=True) - first_observed = models.DateTimeField(blank=True, null=True) - last_observed = models.DateTimeField(blank=True, null=True) - port = models.IntegerField(blank=True, null=True) - protocol = models.TextField(blank=True, null=True) active_classifications = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Current, actively detected and recognized software, technologies, or behaviors observed on a service based on the most recent data collected", ) inactive_classifications = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="previously detected and recognized software, technologies, or behaviors observed on a service previously, but not on the most recent data collected", + ) + discovery_type = models.TextField( + blank=True, null=True, help_text="How the service was detected." ) - discovery_type = models.TextField(blank=True, null=True) externally_inferred_vulnerability_score = models.DecimalField( - max_digits=5, decimal_places=2, blank=True, null=True + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="vulnerability score assigned to a service based on publicly available information about its product name and version, compared against known vulnerabilities in the National Vulnerability Database (NVD)", ) externally_inferred_cves = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="potential vulnerabilities identified by comparing the publicly visible version information of a service discovered on an organization's external attack surface with known vulnerabilities listed in the National Vulnerability Database (NVD)", + ) + service_key = models.TextField( + blank=True, + null=True, + help_text="identifier associated with a specific service that allows for access and interaction with that service within the Xpanse environment", + ) + service_key_type = models.TextField( + blank=True, null=True, help_text="Type of service key." ) - service_key = models.TextField(blank=True, null=True) - service_key_type = models.TextField(blank=True, null=True) - cves = models.ManyToManyField(XpanseCvesMdl, through="XpanseCveServiceMdl") + cves = models.ManyToManyField( + XpanseCvesMdl, + through="XpanseCveServiceMdl", + help_text="Many to many linking table to the cve table.", + ) class Meta: """Set XpanseServicesMdl metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "xpanse_services_mdl" @@ -3060,21 +5297,57 @@ class Meta: class XpanseCveServiceMdl(models.Model): """Define XpanseCves-Service linking table model.""" - xpanse_inferred_cve = models.ForeignKey(XpanseCvesMdl, on_delete=models.CASCADE) - xpanse_service = models.ForeignKey(XpanseServicesMdl, on_delete=models.CASCADE) - inferred_cve_match_type = models.TextField(blank=True, null=True) - product = models.TextField(blank=True, null=True) - confidence = models.TextField(blank=True, null=True) - vendor = models.TextField(blank=True, null=True) - version_number = models.TextField(blank=True, null=True) - activity_status = models.TextField(blank=True, null=True) - first_observed = models.DateTimeField(blank=True, null=True) - last_observed = models.DateTimeField(blank=True, null=True) + xpanse_inferred_cve = models.ForeignKey( + XpanseCvesMdl, + on_delete=models.CASCADE, + help_text="FK: Foreign key to the CVE associated with the service.", + ) + xpanse_service = models.ForeignKey( + XpanseServicesMdl, + on_delete=models.CASCADE, + help_text="FK: Foreign key to the service associated with the CVEs.", + ) + inferred_cve_match_type = models.TextField( + blank=True, + null=True, + help_text="If the match between service and CVE is approximate or exact.", + ) + product = models.TextField( + blank=True, + null=True, + help_text="Vulnerable product on the service that triggered the finding.", + ) + confidence = models.TextField( + blank=True, + null=True, + help_text="How confident Xpanse is the vulnerability is present.", + ) + vendor = models.TextField( + blank=True, null=True, help_text="Vendor who makes the compromised product." + ) + version_number = models.TextField( + blank=True, null=True, help_text="Version number of the compromised product." + ) + activity_status = models.TextField( + blank=True, + null=True, + help_text="Current activity status of the vulnerable product. (Inactive, Active)", + ) + first_observed = models.DateTimeField( + blank=True, + null=True, + help_text="First time the vulnerable product was seen running on the service.", + ) + last_observed = models.DateTimeField( + blank=True, + null=True, + help_text="Last time the vulnerable product was seen running on the service.", + ) class Meta: """Set XpanseCveServiceMdl metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "xpanse_cve_services_mdl" unique_together = (("xpanse_inferred_cve", "xpanse_service"),) @@ -3083,26 +5356,51 @@ class Meta: class XpanseAlerts(models.Model): """Define XpanseAlerts model.""" - xpanse_alert_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - time_pulled_from_xpanse = models.DateTimeField(blank=True, null=True) - alert_id = models.TextField(unique=True, blank=False, null=False) - detection_timestamp = models.DateTimeField(blank=True, null=True) - alert_name = models.TextField(blank=True, null=True) + xpanse_alert_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for an Xpanse alert object.", + ) + time_pulled_from_xpanse = models.DateTimeField( + blank=True, + null=True, + help_text="Time the alert was pulled from the Xpanse API.", + ) + alert_id = models.TextField( + unique=True, blank=False, null=False, help_text="Xpanse alert id." + ) + detection_timestamp = models.DateTimeField( + blank=True, null=True, help_text="Datetime the alert was detected by Xpanse." + ) + alert_name = models.TextField(blank=True, null=True, help_text="Name of the alert.") # endpoint_id ???, - description = models.TextField(blank=True, null=True) - host_name = models.TextField(blank=True, null=True) - alert_action = models.TextField(blank=True, null=True) + description = models.TextField( + blank=True, null=True, help_text="Description of the alert." + ) + host_name = models.TextField( + blank=True, null=True, help_text="IP or domain where the alert points." + ) + alert_action = models.TextField( + blank=True, + null=True, + help_text="a specific response or remediation step that is automatically taken when an alert is triggered.", + ) # user_name ??? null, # mac_addresses ??? null, # source ??? null, - action_pretty = models.TextField(blank=True, null=True) + action_pretty = models.TextField( + blank=True, null=True, help_text="Human readable version of the alert action." + ) # category ??? null, # project ??? null, # cloud_provider ??? null, # resource_sub_type ??? null, # resource_type ??? null, action_country = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Country where the action was performed, if applicable.", ) # event_type ??? null, # is_whitelisted ??? null, @@ -3112,57 +5410,129 @@ class XpanseAlerts(models.Model): # action_external_hostname ??? null, # action_remote_ip ??? null, action_remote_port = ArrayField( - models.IntegerField(blank=True, null=False), blank=True, null=True + models.IntegerField(blank=True, null=False), + blank=True, + null=True, + help_text="Port number.", ) # "matching_service_rule_id ??? null, - starred = models.BooleanField(blank=True, null=True) - external_id = models.TextField(blank=True, null=True) - related_external_id = models.TextField(blank=True, null=True) - alert_occurrence = models.IntegerField(blank=True, null=True) - severity = models.TextField(blank=True, null=True) - matching_status = models.TextField(blank=True, null=True) + starred = models.BooleanField( + blank=True, + null=True, + help_text="T/F if the user has starred the alert in the Xpanse system.", + ) + external_id = models.TextField( + blank=True, + null=True, + help_text="unique identifier that is used to reference a specific asset or record from the Xpanse system", + ) + related_external_id = models.TextField( + blank=True, + null=True, + help_text="Alert external id of the same alert being issued multiple times in the Xpanse system", + ) + alert_occurrence = models.IntegerField( + blank=True, + null=True, + help_text="Number of times the alert has been made in the Xpanse system for the same entity.", + ) + severity = models.TextField( + blank=True, null=True, help_text="Severity of the alert." + ) + matching_status = models.TextField( + blank=True, null=True, help_text="Status of the matching attempt." + ) # end_match_attempt_ts ??? null, - local_insert_ts = models.DateTimeField(blank=True, null=True) - last_modified_ts = models.DateTimeField(blank=True, null=True) - case_id = models.IntegerField(blank=True, null=True) + local_insert_ts = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the alert was inserted into the mini data lake", + ) + last_modified_ts = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the alert was modified in the Xpanse system", + ) + case_id = models.IntegerField( + blank=True, null=True, help_text="Case id in the Xpanse system." + ) # deduplicate_tokens ??? null, # filter_rule_id ??? null, # event_id ??? null, event_timestamp = ArrayField( - models.DateTimeField(blank=True, null=False), blank=True, null=True + models.DateTimeField(blank=True, null=False), + blank=True, + null=True, + help_text="List of event timestamps associated with the alert.", ) # action_local_ip_v6 ??? null, # action_remote_ip_v6 ??? null, - alert_type = models.TextField(blank=True, null=True) - resolution_status = models.TextField(blank=True, null=True) - resolution_comment = models.TextField(blank=True, null=True) + alert_type = models.TextField(blank=True, null=True, help_text="Type of alert") + resolution_status = models.TextField( + blank=True, null=True, help_text="Current resolution status of the alert." + ) + resolution_comment = models.TextField( + blank=True, null=True, help_text="Comment about the resolution." + ) # dynamic_fields ??? null, - tags = ArrayField(models.TextField(blank=True, null=False), blank=True, null=True) + tags = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of tags associated with the alert.", + ) # malicious_urls ??? null, - last_observed = models.DateTimeField(blank=True, null=True) + last_observed = models.DateTimeField( + blank=True, null=True, help_text="Last time the issue was observed" + ) country_codes = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Country code associated with the alert", ) cloud_providers = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of cloud providers associated with the assets in the alert.", ) ipv4_addresses = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of IPs associated with the alert.", ) # ipv6_addresses ??? null, domain_names = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of domains associated with the alert", ) service_ids = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of services ids associated with the alert", ) website_ids = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of websites associated with the alert.", ) asset_ids = ArrayField( - models.TextField(blank=True, null=False), blank=True, null=True + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of asset's ids that are associated with the alert.", + ) + certificate = models.JSONField( + blank=True, + null=True, + help_text="Dictionary containing certificate data assocated with the alert.", ) - certificate = models.JSONField(blank=True, null=True) # { # issuerName": "IOS-Self-Signed-Certificate-782645061", # subjectName": "IOS-Self-Signed-Certificate-782645061", @@ -3170,20 +5540,44 @@ class XpanseAlerts(models.Model): # validNotAfter": 1577836800000, # serialNumber": "1" # }, - port_protocol = models.TextField(blank=True, null=True) + port_protocol = models.TextField( + blank=True, null=True, help_text="Port protocol associated with the alert." + ) # business_unit_hierarchies - attack_surface_rule_name = models.TextField(blank=True, null=True) - remediation_guidance = models.TextField(blank=True, null=True) - asset_identifiers = models.JSONField(blank=True, null=True) + attack_surface_rule_name = models.TextField( + blank=True, + null=True, + help_text="Attack surface rule that was triggered to create the alert", + ) + remediation_guidance = models.TextField( + blank=True, null=True, help_text="Guidance to remediate the alert." + ) + asset_identifiers = models.JSONField( + blank=True, + null=True, + help_text="List of dictionaries containg asset data associated with the alert", + ) - business_units = models.ManyToManyField(XpanseBusinessUnits, related_name="alerts") - services = models.ManyToManyField(XpanseServicesMdl, related_name="alerts") - assets = models.ManyToManyField(XpanseAssetsMdl, related_name="alerts") + business_units = models.ManyToManyField( + XpanseBusinessUnits, + related_name="alerts", + help_text="Many to many relationship to the related business units.", + ) + services = models.ManyToManyField( + XpanseServicesMdl, + related_name="alerts", + help_text="Many to many relationsthip to the services associated with the alert.", + ) + assets = models.ManyToManyField( + XpanseAssetsMdl, + related_name="alerts", + help_text="Many to many relationship to the assets associated with the alert.", + ) class Meta: """Set XpanseAlerts model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "xpanse_alerts_mdl" @@ -3191,13 +5585,19 @@ class Meta: class CpeVender(models.Model): """Define CpeVender model.""" - cpe_vender_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - vender_name = models.TextField(unique=True, blank=True, null=True) + cpe_vender_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique ID of the vender object", + ) + vender_name = models.TextField( + unique=True, blank=True, null=True, help_text="Vender name" + ) class Meta: """Set CpeVender model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cpe_vender" @@ -3205,20 +5605,36 @@ class Meta: class CpeProduct(models.Model): """Define CpeProduct model.""" - cpe_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) - cpe_product_name = models.TextField(blank=True, null=True) - version_number = models.TextField(blank=True, null=True) + cpe_product_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for the Product (CPE)", + ) + cpe_product_name = models.TextField( + blank=True, null=True, help_text="Name of the product" + ) + version_number = models.TextField( + blank=True, null=True, help_text="Version of the product" + ) cpe_vender = models.ForeignKey( - "CpeVender", on_delete=models.CASCADE, db_column="cpe_vender_uid", default=None + "CpeVender", + on_delete=models.CASCADE, + db_column="cpe_vender_uid", + default=None, + help_text="FK: Foreign key to the related vender object.", ) # Create linking table for many to many relationship - cves = models.ManyToManyField(Cve, related_name="products") + cves = models.ManyToManyField( + Cve, + related_name="products", + help_text="Many to many relationship to the CVEs associated with the product", + ) class Meta: """Set CpeProduct model metadata.""" - app_label = "dmz_mini_dl" + app_label = app_label_name managed = manage_db db_table = "cpe_product_mdl" unique_together = (("cpe_product_name", "version_number"),) diff --git a/src/pe_reports/pe_reports_django_project/home/models.py b/src/pe_reports/pe_reports_django_project/home/models.py index 5ef93222..647ddc6e 100644 --- a/src/pe_reports/pe_reports_django_project/home/models.py +++ b/src/pe_reports/pe_reports_django_project/home/models.py @@ -2525,3 +2525,53 @@ class Meta: managed = False db_table = "was_findings" + +class WasReport(models.Model): + org_name = models.TextField(blank=True, null=True) + date_pulled = models.DateTimeField(blank=True, null=True) + last_scan_date = models.DateTimeField(blank=True, null=True) + security_risk = models.TextField(blank=True, null=True) + total_info = models.IntegerField(blank=True, null=True) + num_apps = models.IntegerField(blank=True, null=True) + risk_color = models.TextField(blank=True, null=True) + sensitive_count = models.IntegerField(blank=True, null=True) + sensitive_color = models.TextField(blank=True, null=True) + max_days_open_urgent = models.IntegerField(blank=True, null=True) + max_days_open_critical = models.IntegerField(blank=True, null=True) + urgent_color = models.TextField(blank=True, null=True) + critical_color = models.TextField(blank=True, null=True) + org_was_acronym = models.TextField(blank=True, null=True) + name_len = models.TextField(blank=True, null=True) + vuln_csv_dict = models.JSONField(blank=True, null=True, default=dict) + ssn_cc_dict = models.JSONField(blank=True, null=True, default=dict) + app_overview_csv_dict = models.JSONField(blank=True, null=True, default=dict) + details_csv = models.JSONField(blank=True, null=True, default=list) + info_csv = models.JSONField(blank=True, null=True, default=list) + links_crawled = models.JSONField(blank=True, null=True, default=list) + links_rejected = models.JSONField(blank=True, null=True, default=list) + emails_found = models.JSONField(blank=True, null=True, default=list) + owasp_count_dict = models.JSONField(blank=True, null=True, default=dict) + group_count_dict = models.JSONField(blank=True, null=True, default=dict) + fixed = models.IntegerField(blank=True, null=True) + total = models.IntegerField(blank=True, null=True) + vulns_monthly_dict = models.JSONField(blank=True, null=True, default=dict) + path_disc = models.IntegerField(blank=True, null=True) + info_disc = models.IntegerField(blank=True, null=True) + cross_site = models.IntegerField(blank=True, null=True) + burp = models.IntegerField(blank=True, null=True) + sql_inj = models.IntegerField(blank=True, null=True) + bugcrowd = models.IntegerField(blank=True, null=True) + reopened = models.IntegerField(blank=True, null=True) + reopened_color = models.TextField(blank=True, null=True) + new_vulns = models.IntegerField(blank=True, null=True) + new_vulns_color = models.TextField(blank=True, null=True) + tot_vulns = models.IntegerField(blank=True, null=True) + tot_vulns_color = models.TextField(blank=True, null=True) + lev1 = models.IntegerField(blank=True, null=True) + lev2 = models.IntegerField(blank=True, null=True) + lev3 = models.IntegerField(blank=True, null=True) + lev4 = models.IntegerField(blank=True, null=True) + lev5 = models.IntegerField(blank=True, null=True) + severities = ArrayField(models.IntegerField(), blank=True, null=True, default=list) + ages = ArrayField(models.IntegerField(), blank=True, null=True, default=list) + pdf_obj = models.BinaryField(blank=True, null=True) diff --git a/src/pe_source/cybersixgill.py b/src/pe_source/cybersixgill.py index e2268bc7..c8b7f802 100644 --- a/src/pe_source/cybersixgill.py +++ b/src/pe_source/cybersixgill.py @@ -3,11 +3,14 @@ # Standard Python Libraries from datetime import date, datetime, timedelta import logging -import pandas as pd -import sys -import time + +# import sys +# import time import traceback +# Third-Party Libraries +import pandas as pd + from .data.pe_db.db_query_source import ( get_breaches, get_data_source_uid, @@ -19,18 +22,17 @@ insert_sixgill_topCVEs, ) from .data.sixgill.api import get_sixgill_organizations -from .data.sixgill.source import ( +from .data.sixgill.source import ( # cve_summary,; get_alerts_content, alerts, alias_organization, all_assets_list, creds, - cve_summary, - get_alerts_content, mentions, root_domains, top_cves, ) -from .data.helpers.redact_pii import redact_pii, redact_pii_new + +# from .data.helpers.redact_pii import redact_pii # Set todays date formatted YYYY-MM-DD and the start_date 30 days prior TODAY = date.today() @@ -48,7 +50,7 @@ LOGGER = logging.getLogger(__name__) # Suppress output from presidio-analyzer import (PII filter) -logging.getLogger('presidio-analyzer').setLevel(logging.CRITICAL+1) +logging.getLogger("presidio-analyzer").setLevel(logging.CRITICAL + 1) class Cybersixgill: @@ -116,7 +118,7 @@ def run_cybersixgill(self): # Get sixgill_org_id associated with the PE org try: sixgill_org_id = sixgill_orgs[org_id][0] - except KeyError as err: + except KeyError: LOGGER.warning(f"{org_id} is not registered in Cybersixgill") # print(err, file=sys.stderr) # failed.append("%s not in sixgill" % org_id) @@ -124,7 +126,9 @@ def run_cybersixgill(self): # Run alerts if "alerts" in method_list: - LOGGER.info(f"Fetching alert data for {org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Fetching alert data for {org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if ( self.get_alerts( org_id, @@ -138,7 +142,9 @@ def run_cybersixgill(self): failed.append("%s alerts" % org_id) # Run mentions if "mentions" in method_list: - LOGGER.info(f"Fetching mention data for {org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Fetching mention data for {org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if ( self.get_mentions( org_id, @@ -152,7 +158,9 @@ def run_cybersixgill(self): failed.append("%s mentions" % org_id) # Run credentials if "credentials" in method_list: - LOGGER.info(f"Fetching credential data for {org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Fetching credential data for {org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if ( self.get_credentials( org_id, sixgill_org_id, pe_org_uid, source_uid @@ -386,20 +394,28 @@ def get_credentials(self, org_id, sixgill_org_id, pe_org_uid, source_uid): # Catch no root assets situation if len(roots) == 0: - LOGGER.warning(f"{org_id} does not have any root domain assets in Cybersixgill") + LOGGER.warning( + f"{org_id} does not have any root domain assets in Cybersixgill" + ) return 0 # Fetch credential data if len(roots) > 100: # Catch situation where an org has >100 roots in sixgill - LOGGER.info(f"{org_id} has more than 100 root assets in cybersixgill, breaking into chunks of 100...") - root_chunks = [roots[i:i + 100] for i in range(0, len(roots), 100)] + LOGGER.info( + f"{org_id} has more than 100 root assets in cybersixgill, breaking into chunks of 100..." + ) + root_chunks = [roots[i : i + 100] for i in range(0, len(roots), 100)] creds_df = pd.DataFrame() for idx, chunk in enumerate(root_chunks): try: - LOGGER.info(f"On chunk {idx+1} of {len(root_chunks)} for {org_id} credentials") + LOGGER.info( + f"On chunk {idx+1} of {len(root_chunks)} for {org_id} credentials" + ) chunk_creds_df = creds(chunk, START_DATE_TIME, END_DATE_TIME) - LOGGER.info("Found %s credentials for this chunk", len(chunk_creds_df.index)) + LOGGER.info( + "Found %s credentials for this chunk", len(chunk_creds_df.index) + ) chunk_creds_df["organizations_uid"] = pe_org_uid chunk_creds_df["data_source_uid"] = source_uid creds_df = creds_df.append(chunk_creds_df, ignore_index=True) @@ -535,7 +551,7 @@ def get_topCVEs(self, source_uid): # except Exception: # summary = "" # top_cve_df.at[cve_index, "summary"] = summary - + except Exception as e: LOGGER.error("Failed fetching top CVEs.") LOGGER.error(e)