From b4a77d598653b13f81d803778e9384db42aeddc7 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Mon, 14 Oct 2024 18:11:57 +0200 Subject: [PATCH 1/5] Add value limits to GPS coordinates for Configurations --- publicdb/histograms/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publicdb/histograms/models.py b/publicdb/histograms/models.py index bfc4cf27..29c26c13 100644 --- a/publicdb/histograms/models.py +++ b/publicdb/histograms/models.py @@ -102,8 +102,8 @@ def get_absolute_url(self): class Configuration(models.Model): summary = models.ForeignKey(Summary, models.CASCADE, related_name='configurations') timestamp = models.DateTimeField() - gps_latitude = models.FloatField() - gps_longitude = models.FloatField() + gps_latitude = models.FloatField(min_value=-90, max_value=90) + gps_longitude = models.FloatField(min_value=-360, max_value=360) gps_altitude = models.FloatField() mas_version = models.CharField(max_length=40) slv_version = models.CharField(max_length=40) From 9faa5b7451fa11688a54733130e35ede2ed53a17 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sun, 27 Oct 2024 14:50:11 +0100 Subject: [PATCH 2/5] Add missing django_get_or_create for factories --- tests/factories/histograms_factories.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/factories/histograms_factories.py b/tests/factories/histograms_factories.py index e81360b4..45c74304 100644 --- a/tests/factories/histograms_factories.py +++ b/tests/factories/histograms_factories.py @@ -168,6 +168,7 @@ class NetworkHistogramFactory(factory.django.DjangoModelFactory): class Meta: model = models.NetworkHistogram + django_get_or_create = ('network_summary', 'type') class CoincidencetimeHistogramFactory(NetworkHistogramFactory): @@ -190,6 +191,7 @@ class DailyHistogramFactory(factory.django.DjangoModelFactory): class Meta: model = models.DailyHistogram + django_get_or_create = ('summary', 'type') class EventtimeHistogramFactory(DailyHistogramFactory): @@ -249,6 +251,7 @@ class DailyDatasetFactory(factory.django.DjangoModelFactory): class Meta: model = models.DailyDataset + django_get_or_create = ('summary', 'type') class TemperatureDatasetFactory(DailyDatasetFactory): @@ -301,3 +304,4 @@ class StationTimingOffsetFactory(factory.django.DjangoModelFactory): class Meta: model = models.StationTimingOffset + django_get_or_create = ('ref_summary', 'summary') From 4b09382ad9d9b9c9fc96d7e0210040eacd4591b2 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sun, 27 Oct 2024 14:51:39 +0100 Subject: [PATCH 3/5] Update expected datastore HOST for local requests --- publicdb/inforecords/views.py | 4 +--- publicdb/settings_docker.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/publicdb/inforecords/views.py b/publicdb/inforecords/views.py index 155aeaab..798f9df9 100644 --- a/publicdb/inforecords/views.py +++ b/publicdb/inforecords/views.py @@ -1,5 +1,3 @@ -import socket - from django.conf import settings from django.core.exceptions import PermissionDenied from django.shortcuts import render @@ -11,7 +9,7 @@ def create_datastore_config(request): """Create the datastore configuration""" # Limit access to only allow access from the Datastore server - if socket.gethostbyaddr(request.META['REMOTE_ADDR'])[0] != settings.DATASTORE_HOST: + if request.META['REMOTE_HOST'] != settings.DATASTORE_HOST: raise PermissionDenied return render( diff --git a/publicdb/settings_docker.py b/publicdb/settings_docker.py index 43a47189..5932a017 100644 --- a/publicdb/settings_docker.py +++ b/publicdb/settings_docker.py @@ -26,4 +26,4 @@ DATASTORE_PROXY = 'http://datastore:8002' # Datastore host name -DATASTORE_HOST = 'publicdb_datastore_1.publicdb_default' +DATASTORE_HOST = 'datastore' From c113041ac2993c17931ea5d5998c6934be3b8cd2 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sun, 27 Oct 2024 14:52:08 +0100 Subject: [PATCH 4/5] Sort creation of fake station data --- .../management/commands/createfakedata.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/publicdb/default/management/commands/createfakedata.py b/publicdb/default/management/commands/createfakedata.py index 509a7f83..919dbd3e 100644 --- a/publicdb/default/management/commands/createfakedata.py +++ b/publicdb/default/management/commands/createfakedata.py @@ -62,16 +62,16 @@ def handle(*args, **options): ] for summary in summaries: - histograms_factories.EventtimeHistogramFactory(summary=summary) histograms_factories.AzimuthHistogramFactory(summary=summary) - histograms_factories.ZenithHistogramFactory(summary=summary) + histograms_factories.BarometerDatasetFactory(summary=summary) + histograms_factories.ConfigurationFactory(summary=summary) + histograms_factories.DetectorTimingOffsetFactory(summary=summary) + histograms_factories.EventtimeHistogramFactory(summary=summary) histograms_factories.PulseheightHistogramFactory(summary=summary) histograms_factories.PulseintegralHistogramFactory(summary=summary) - histograms_factories.SingleslowHistogramFactory(summary=summary) histograms_factories.SingleshighHistogramFactory(summary=summary) - histograms_factories.TemperatureDatasetFactory(summary=summary) - histograms_factories.BarometerDatasetFactory(summary=summary) - histograms_factories.SinglesratelowDatasetFactory(summary=summary) + histograms_factories.SingleslowHistogramFactory(summary=summary) histograms_factories.SinglesratehighDatasetFactory(summary=summary) - histograms_factories.ConfigurationFactory(summary=summary) - histograms_factories.DetectorTimingOffsetFactory(summary=summary) + histograms_factories.SinglesratelowDatasetFactory(summary=summary) + histograms_factories.TemperatureDatasetFactory(summary=summary) + histograms_factories.ZenithHistogramFactory(summary=summary) From ccf25c56e32fdaa29770cff7aee276f8262a3efa Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sun, 27 Oct 2024 15:10:57 +0100 Subject: [PATCH 5/5] Enforce latitude and longitude limits in jobs Set value to 0 if is outside valid range. --- publicdb/histograms/jobs.py | 29 ++++++++++++++++++++--------- publicdb/histograms/models.py | 4 ++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/publicdb/histograms/jobs.py b/publicdb/histograms/jobs.py index d4618acb..e254dacd 100644 --- a/publicdb/histograms/jobs.py +++ b/publicdb/histograms/jobs.py @@ -581,16 +581,27 @@ def update_config(summary): for config in configs[summary.num_config :]: new_config = Configuration(summary=summary) - for var in vars(new_config): - if var in ['summary', 'id', 'summary_id'] or var[0] == '_': - pass - elif var in ['mas_version', 'slv_version']: - vars(new_config)[var] = blobs[config[var]] - elif var == 'timestamp': - ts = datetime.datetime.utcfromtimestamp(config[var]) - vars(new_config)[var] = ts + for field in new_config._meta.get_fields(): + field_name = field.name + if field_name in ['summary', 'id']: + continue + elif field_name in ['mas_version', 'slv_version']: + value = blobs[config[field_name]] + elif field_name == 'timestamp': + value = datetime.datetime.utcfromtimestamp(config[field_name]) + elif field_name == 'gps_latitude': + if -90 <= config[field_name] <= 90: + value = config[field_name] + else: + value = 0 + elif field_name == 'gps_longitude': + if -360 <= config[field_name] <= 360: + value = config[field_name] + else: + value = 0 else: - vars(new_config)[var] = config[var] + value = config[field_name] + setattr(new_config, field_name, value) django.db.close_old_connections() new_config.save() diff --git a/publicdb/histograms/models.py b/publicdb/histograms/models.py index 29c26c13..bfc4cf27 100644 --- a/publicdb/histograms/models.py +++ b/publicdb/histograms/models.py @@ -102,8 +102,8 @@ def get_absolute_url(self): class Configuration(models.Model): summary = models.ForeignKey(Summary, models.CASCADE, related_name='configurations') timestamp = models.DateTimeField() - gps_latitude = models.FloatField(min_value=-90, max_value=90) - gps_longitude = models.FloatField(min_value=-360, max_value=360) + gps_latitude = models.FloatField() + gps_longitude = models.FloatField() gps_altitude = models.FloatField() mas_version = models.CharField(max_length=40) slv_version = models.CharField(max_length=40)