From 64b6ade1b9cbe55d4d4473f078cb3369cd00fc98 Mon Sep 17 00:00:00 2001 From: John Tordoff <> Date: Thu, 23 Jun 2022 14:10:14 -0400 Subject: [PATCH 1/5] remove storage region configuration from the migration stream and add to config file --- .gitignore | 1 + osf/apps.py | 6 ++++++ osf/migrations/__init__.py | 15 +++++++++++++++ storage-regions-local.yaml | 13 +++++++++++++ website/settings/defaults.py | 3 +++ website/settings/local-dist.py | 2 ++ website/settings/local-travis.py | 2 ++ 7 files changed, 42 insertions(+) create mode 100644 storage-regions-local.yaml diff --git a/.gitignore b/.gitignore index 0f83ae23f86..631436b1d17 100644 --- a/.gitignore +++ b/.gitignore @@ -204,3 +204,4 @@ ssl/ # pyenv .python-version +/storage-regions.yaml # contains secrets diff --git a/osf/apps.py b/osf/apps.py index 339e451375f..c76685c669d 100644 --- a/osf/apps.py +++ b/osf/apps.py @@ -9,6 +9,7 @@ update_blocked_email_domains, update_license, update_permission_groups, + update_storage_regions, update_waffle_flags, ) @@ -53,3 +54,8 @@ def ready(self): update_blocked_email_domains, dispatch_uid='osf.apps.update_blocked_email_domains' ) + + post_migrate.connect( + update_storage_regions, + dispatch_uid='osf.apps.update_storage_regions' + ) diff --git a/osf/migrations/__init__.py b/osf/migrations/__init__.py index c68beea4073..9d77c16a6ff 100644 --- a/osf/migrations/__init__.py +++ b/osf/migrations/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import sys +import yaml import logging from django.db.utils import ProgrammingError @@ -162,3 +163,17 @@ def update_blocked_email_domains(sender, verbosity=0, **kwargs): domain=domain, defaults={'note': NotableEmailDomain.Note.EXCLUDE_FROM_ACCOUNT_CREATION}, ) + + +def update_storage_regions(sender, verbosity=0, **kwargs): + from django.apps import apps + + if getattr(sender, 'label', None) == 'osf': + with open(osf_settings.STORAGE_REGION_CONFIG_PATH, 'r') as stream: + features = yaml.safe_load(stream) + Region = apps.get_model('addons_osfstorage', 'Region') + for region in features['storage_regions']: + Region.objects.update_or_create(_id=region['_id'], defaults=region) + + if 'pytest' not in sys.modules: # Allows for isolated tests without Regions + assert not Region.objects.all(), 'No storage regions found.' diff --git a/storage-regions-local.yaml b/storage-regions-local.yaml new file mode 100644 index 00000000000..e8544edb623 --- /dev/null +++ b/storage-regions-local.yaml @@ -0,0 +1,13 @@ +# This is used to populate fake storage region's for use in development, urls and file paths in this file should point +# to urls and file paths in test environments and services. +storage_regions: + - _id: us # default + name: United States + waterbutler_credentials: + storage: {} + waterbutler_url: http://localhost:7777 + mfr_url: http://localhost:7778 + waterbutler_settings: + storage: + folder: /code/website/osfstoragecache + provider: filesystem \ No newline at end of file diff --git a/website/settings/defaults.py b/website/settings/defaults.py index dabaf1b0b66..f8a987af051 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -2109,3 +2109,6 @@ def from_node_usage(cls, usage_bytes, private_limit=None, public_limit=None): PREPRINT_METRICS_START_DATE = datetime.datetime(2019, 1, 1) WAFFLE_VALUES_YAML = 'osf/features.yaml' + +STORAGE_REGION_CONFIG_PATH = '/code/storage-regions.yaml' + diff --git a/website/settings/local-dist.py b/website/settings/local-dist.py index e12d27feec6..502614dfdf9 100644 --- a/website/settings/local-dist.py +++ b/website/settings/local-dist.py @@ -142,3 +142,5 @@ class CeleryConfig(defaults.CeleryConfig): DATACITE_ENABLED = False OOPSPAM_CHECK_IP = False + +STORAGE_REGION_CONFIG_PATH = 'storage-regions-local.yaml' diff --git a/website/settings/local-travis.py b/website/settings/local-travis.py index b1e44aa9725..85ddcd918f8 100644 --- a/website/settings/local-travis.py +++ b/website/settings/local-travis.py @@ -107,3 +107,5 @@ class CeleryConfig(defaults.CeleryConfig): SHARE_ENABLED = False DATACITE_ENABLED = False IA_ARCHIVE_ENABLED = False + +STORAGE_REGION_CONFIG_PATH = 'storage-regions-local.yaml' From 47639a406890b1604c914dc1a5e4ef689b199a51 Mon Sep 17 00:00:00 2001 From: John Tordoff <> Date: Mon, 27 Jun 2022 20:35:30 -0400 Subject: [PATCH 2/5] only ensure default provider and remove yaml config --- .gitignore | 1 - osf/migrations/__init__.py | 28 +++++++++++++++++----------- storage-regions-local.yaml | 13 ------------- website/settings/defaults.py | 4 +--- website/settings/local-dist.py | 2 -- website/settings/local-travis.py | 2 -- 6 files changed, 18 insertions(+), 32 deletions(-) delete mode 100644 storage-regions-local.yaml diff --git a/.gitignore b/.gitignore index 631436b1d17..0f83ae23f86 100644 --- a/.gitignore +++ b/.gitignore @@ -204,4 +204,3 @@ ssl/ # pyenv .python-version -/storage-regions.yaml # contains secrets diff --git a/osf/migrations/__init__.py b/osf/migrations/__init__.py index 9d77c16a6ff..b2c40f4f391 100644 --- a/osf/migrations/__init__.py +++ b/osf/migrations/__init__.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- import sys -import yaml import logging +from django.apps import apps from django.db.utils import ProgrammingError from django.core.management import call_command +from addons.osfstorage.settings import DEFAULT_REGION_ID, DEFAULT_REGION_NAME from api.base import settings as api_settings from osf.management.commands.manage_switch_flags import manage_waffle from osf.utils.migrations import ensure_schemas, map_schemas_to_schemablocks @@ -166,14 +167,19 @@ def update_blocked_email_domains(sender, verbosity=0, **kwargs): def update_storage_regions(sender, verbosity=0, **kwargs): - from django.apps import apps - if getattr(sender, 'label', None) == 'osf': - with open(osf_settings.STORAGE_REGION_CONFIG_PATH, 'r') as stream: - features = yaml.safe_load(stream) - Region = apps.get_model('addons_osfstorage', 'Region') - for region in features['storage_regions']: - Region.objects.update_or_create(_id=region['_id'], defaults=region) - - if 'pytest' not in sys.modules: # Allows for isolated tests without Regions - assert not Region.objects.all(), 'No storage regions found.' + ensure_default_storage_region() + + +def ensure_default_storage_region(): + osfstorage_config = apps.get_app_config('addons_osfstorage') + Region = apps.get_model('addons_osfstorage', 'Region') + Region.objects.update_or_create( + _id=DEFAULT_REGION_ID, + defaults={ + 'name': DEFAULT_REGION_NAME, + 'waterbutler_credentials': osfstorage_config.WATERBUTLER_CREDENTIALS, + 'waterbutler_settings': osfstorage_config.WATERBUTLER_SETTINGS, + 'waterbutler_url': settings.WATERBUTLER_URL + } + ) diff --git a/storage-regions-local.yaml b/storage-regions-local.yaml deleted file mode 100644 index e8544edb623..00000000000 --- a/storage-regions-local.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# This is used to populate fake storage region's for use in development, urls and file paths in this file should point -# to urls and file paths in test environments and services. -storage_regions: - - _id: us # default - name: United States - waterbutler_credentials: - storage: {} - waterbutler_url: http://localhost:7777 - mfr_url: http://localhost:7778 - waterbutler_settings: - storage: - folder: /code/website/osfstoragecache - provider: filesystem \ No newline at end of file diff --git a/website/settings/defaults.py b/website/settings/defaults.py index f8a987af051..7220447344c 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -2108,7 +2108,5 @@ def from_node_usage(cls, usage_bytes, private_limit=None, public_limit=None): CAS_LOG_LEVEL = 3 # ERROR PREPRINT_METRICS_START_DATE = datetime.datetime(2019, 1, 1) -WAFFLE_VALUES_YAML = 'osf/features.yaml' - -STORAGE_REGION_CONFIG_PATH = '/code/storage-regions.yaml' +WAFFLE_VALUES_YAML = 'osf/features.yaml' diff --git a/website/settings/local-dist.py b/website/settings/local-dist.py index 502614dfdf9..e12d27feec6 100644 --- a/website/settings/local-dist.py +++ b/website/settings/local-dist.py @@ -142,5 +142,3 @@ class CeleryConfig(defaults.CeleryConfig): DATACITE_ENABLED = False OOPSPAM_CHECK_IP = False - -STORAGE_REGION_CONFIG_PATH = 'storage-regions-local.yaml' diff --git a/website/settings/local-travis.py b/website/settings/local-travis.py index 85ddcd918f8..b1e44aa9725 100644 --- a/website/settings/local-travis.py +++ b/website/settings/local-travis.py @@ -107,5 +107,3 @@ class CeleryConfig(defaults.CeleryConfig): SHARE_ENABLED = False DATACITE_ENABLED = False IA_ARCHIVE_ENABLED = False - -STORAGE_REGION_CONFIG_PATH = 'storage-regions-local.yaml' From c9e5eeb5d0e1f72e02a1ec6d3652f2fa06ab3b6c Mon Sep 17 00:00:00 2001 From: John Tordoff <> Date: Sun, 3 Jul 2022 16:53:38 -0400 Subject: [PATCH 3/5] remove default storage region backfill --- osf/migrations/0099_add_default_storage_region.py | 1 - 1 file changed, 1 deletion(-) diff --git a/osf/migrations/0099_add_default_storage_region.py b/osf/migrations/0099_add_default_storage_region.py index 5d3b1b7e37b..8b475a24506 100644 --- a/osf/migrations/0099_add_default_storage_region.py +++ b/osf/migrations/0099_add_default_storage_region.py @@ -75,5 +75,4 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(add_osfstorage_addon, remove_osfstorage_addon), ] From ef033edb47aea6204905b727b384f7f9751d2376 Mon Sep 17 00:00:00 2001 From: John Tordoff <> Date: Wed, 13 Jul 2022 09:26:41 -0400 Subject: [PATCH 4/5] region gets or creates, but not updates --- osf/migrations/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osf/migrations/__init__.py b/osf/migrations/__init__.py index b2c40f4f391..283a6f4a9e7 100644 --- a/osf/migrations/__init__.py +++ b/osf/migrations/__init__.py @@ -174,7 +174,7 @@ def update_storage_regions(sender, verbosity=0, **kwargs): def ensure_default_storage_region(): osfstorage_config = apps.get_app_config('addons_osfstorage') Region = apps.get_model('addons_osfstorage', 'Region') - Region.objects.update_or_create( + Region.objects.get_or_create( _id=DEFAULT_REGION_ID, defaults={ 'name': DEFAULT_REGION_NAME, From 4cbcf633cb55cdca52bb2c2cd570e9f4add97f14 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 19 Jul 2022 16:15:54 -0400 Subject: [PATCH 5/5] Use _id and name as positional arguments and fix osf_settings --- osf/migrations/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osf/migrations/__init__.py b/osf/migrations/__init__.py index 283a6f4a9e7..288f4990fe3 100644 --- a/osf/migrations/__init__.py +++ b/osf/migrations/__init__.py @@ -176,10 +176,10 @@ def ensure_default_storage_region(): Region = apps.get_model('addons_osfstorage', 'Region') Region.objects.get_or_create( _id=DEFAULT_REGION_ID, + name=DEFAULT_REGION_NAME, defaults={ - 'name': DEFAULT_REGION_NAME, 'waterbutler_credentials': osfstorage_config.WATERBUTLER_CREDENTIALS, 'waterbutler_settings': osfstorage_config.WATERBUTLER_SETTINGS, - 'waterbutler_url': settings.WATERBUTLER_URL + 'waterbutler_url': osf_settings.WATERBUTLER_URL } )