[ENG-3836] Investigate waffle flags (#9950)

* Create Waffle Flags and Switches as post_migrate_signal (with configurable values)
Co-authored-by: John Tordoff <>

api_tests/providers/registrations/views/test_registration_provider_schemas.py

@@ -106,7 +106,7 @@ def provider_with_reg(self, osf_reg_schema, egap_schema, schema, out_dated_schem
     def egap_admin(self):
         user = AuthUserFactory()
         user.save()
-        flag = Flag.objects.get(name=EGAP_ADMINS)
+        flag = Flag.objects.create(name=EGAP_ADMINS)
         group = Group.objects.create(name=EGAP_ADMINS)  # Just using the same name for convenience
         flag.groups.add(group)
         group.user_set.add(user)

api_tests/schemas/views/test_registration_schemas_list.py

@@ -31,7 +31,7 @@ def url(self):
     def egap_admin(self):
         user = AuthUserFactory()
         user.save()
-        flag = Flag.objects.get(name=features.EGAP_ADMINS)
+        flag = Flag.objects.create(name=features.EGAP_ADMINS)
         group = Group.objects.create(name=features.EGAP_ADMINS)  # Just using the same name for convenience
         flag.groups.add(group)
         group.user_set.add(user)

osf/apps.py

@@ -1,8 +1,13 @@
-from __future__ import unicode_literals
-
+import logging
 from django.apps import AppConfig as BaseAppConfig
 from django.db.models.signals import post_migrate
-from osf.migrations import update_permission_groups, create_cache_table
+from osf.migrations import (
+    update_permission_groups,
+    update_waffle_flags,
+    create_cache_table
+)
+
+logger = logging.getLogger(__file__)
 
 
 class AppConfig(BaseAppConfig):
@@ -16,6 +21,10 @@ def ready(self):
             update_permission_groups,
             dispatch_uid='osf.apps.update_permissions_groups'
         )
+        post_migrate.connect(
+            update_waffle_flags,
+            dispatch_uid='osf.apps.update_waffle_flags'
+        )
         post_migrate.connect(
             create_cache_table,
             dispatch_uid='osf.apps.create_cache_table'

osf/features.py

@@ -1,49 +1,11 @@
-flags = {
-    'STORAGE_USAGE': 'storage_usage',
-    'INSTITUTIONAL_LANDING_FLAG': 'institutions_nav_bar',
-    'STORAGE_I18N': 'storage_i18n',
-    'OSF_GROUPS': 'osf_groups',
-    'ENABLE_CHRONOS': 'enable_chronos',
-    'EGAP_ADMINS': 'egap_admins',
-    'EMBER_AB_TESTING_HOME_PAGE_VERSION_B': 'ab_testing_home_page_version_b',
-    'EMBER_AB_TESTING_HOME_PAGE_HERO_TEXT_VERSION_B': 'ab_testing_home_page_hero_text_version_b',
-    'EMBER_AUTH_REGISTER': 'ember_auth_register',
-    'EMBER_CREATE_DRAFT_REGISTRATION': 'ember_create_draft_registration_page',
-    'EMBER_EDIT_DRAFT_REGISTRATION': 'ember_edit_draft_registration_page',
-    'EMBER_FILE_REGISTRATION_DETAIL': 'ember_file_registration_detail_page',
-    'EMBER_FILE_PROJECT_DETAIL': 'ember_file_project_detail_page',
-    'EMBER_MEETINGS': 'ember_meetings_page',
-    'EMBER_MEETING_DETAIL': 'ember_meeting_detail_page',
-    'EMBER_MY_PROJECTS': 'ember_my_projects_page',
-    'EMBER_PROJECT_ANALYTICS': 'ember_project_analytics_page',
-    'EMBER_PROJECT_CONTRIBUTORS': 'ember_project_contributors_page',
-    'EMBER_PROJECT_DETAIL': 'ember_project_detail_page',
-    'EMBER_REGISTRATION_FILES': 'ember_registration_files_page',
-    'EMBER_PROJECT_FILES': 'ember_project_files_page',
-    'EMBER_PROJECT_FORKS': 'ember_project_forks_page',
-    'EMBER_PROJECT_REGISTRATIONS': 'ember_project_registrations_page',
-    'EMBER_PROJECT_SETTINGS': 'ember_project_settings_page',
-    'EMBER_PROJECT_WIKI': 'ember_project_wiki_page',
-    'EMBER_REGISTRATION_FORM_DETAIL': 'ember_registration_form_detail_page',
-    'EMBER_REGISTRIES_DETAIL_PAGE': 'ember_registries_detail_page',
-    'EMBER_SEARCH_PAGE': 'ember_search_page',
-    'EMBER_USER_PROFILE': 'ember_user_profile_page',
-    'EMBER_USER_SETTINGS': 'ember_user_settings_page',
-    'EMBER_USER_SETTINGS_ACCOUNTS': 'ember_user_settings_account_page',
-    'EMBER_USER_SETTINGS_ADDONS': 'ember_user_settings_addons_page',
-    'EMBER_USER_SETTINGS_APPS': 'ember_user_settings_apps_page',
-    'EMBER_USER_SETTINGS_NOTIFICATIONS': 'ember_user_settings_notifications_page',
-    'EMBER_USER_SETTINGS_TOKENS': 'ember_user_settings_tokens_page',
-}
+import yaml
+from website import settings
 
-switches = {
-    'ENABLE_INACTIVE_SCHEMAS': 'enable_inactive_schemas',
-    'OSF_PREREGISTRATION': 'osf_preregistration',
-    'DISABLE_ENGAGEMENT_EMAILS': 'disable_engagement_emails',
-    'ELASTICSEARCH_METRICS': 'elasticsearch_metrics',
-    'ENFORCE_CSRF': 'enforce_csrf',
-    'ENABLE_RAW_METRICS': 'enable_raw_metrics',
-}
+with open(settings.WAFFLE_VALUES_YAML, 'r') as stream:
+    features = yaml.safe_load(stream)
 
-locals().update(flags)
-locals().update(switches)
+for flag in features['flags']:
+    locals()[flag.pop('flag_name')] = flag['name']
+
+for switch in features['switches']:
+    locals()[switch.pop('flag_name')] = switch['name']

osf/features.yaml

@@ -0,0 +1,216 @@
+# This file contains the default configuration for our feature flipping using waffle flags and switches, this represents
+# the intended configuration of flags at the time of release, if you plan to  switch the activity status of a feature in
+# after release (during normal operation) please make sure that is reflected in this document by not explicitly setting
+# it here. However features not intended to be after a release flipped should explicitly stated.
+
+# Workflow:
+# 1. Add a flag/switch with a name and value to this document
+# 2. Run `manage_waffle` to create the flag(s)
+# 3. Use the admin app to enable/disable the flag/switch at your convenience
+# 4. When feature is complete add the activity status (active or everyone) to the value it will have in production until
+# the old flipping code can be removed.
+# 5. When a flag name is no longer referenced anywhere in this repo or in the Ember app remove it from this list.
+flags:
+  - flag_name: EMBER_FILE_PROJECT_DETAIL
+    name: ember_file_project_detail_page
+    note: This is part of the upcoming files page redesign
+    everyone: true
+
+  - flag_name: EMBER_PROJECT_FILES
+    name: ember_project_files_page
+    note: This is part of the upcoming files page redesign
+    everyone: true
+
+  - flag_name: STORAGE_USAGE
+    name: storage_usage
+    note: Indicates whether we display the file storage usage for each node on the project overview page.
+    everyone: true
+
+  - flag_name: INSTITUTIONAL_LANDING_FLAG
+    name: institutions_nav_bar
+    note: Indicates whether we display the institutions navbar.
+    everyone: true
+
+  - flag_name: STORAGE_I18N
+    name: storage_i18n
+    note: Indicates whether region based storage is enabled.
+    everyone: true
+
+  - flag_name: OSF_GROUPS
+    name: osf_groups
+    note: Indicates whether certain parts of the OSF groups feature are enabled.
+    everyone: true
+
+  - flag_name: EGAP_ADMINS
+    name: egap_admins
+    note: Indicates whether EGAP admins have special access to custom schemas
+    everyone: true
+
+  - flag_name: EMBER_AUTH_REGISTER
+    name: ember_auth_register
+    note: This indicates whether this view is routed for OSF register, redirect or go to `auth_logout`
+    everyone: true
+
+  - flag_name: EMBER_PROJECT_DETAIL
+    name: ember_project_detail_page
+    note: This flag controls wheter the project overview page is routed to the ember app
+    everyone: false
+
+  - flag_name: EMBER_CREATE_DRAFT_REGISTRATION
+    name: ember_create_draft_registration_page
+    note: This flag controls wheter POST requests to /project/<pid>/registrations/ and
+          /project/<pid>/node/<nid>/registrations/ are routed to the ember app
+    everyone: false
+
+  - flag_name: EMBER_MEETING_DETAIL
+    name: ember_meeting_detail_page
+    note: This flag controls wheter the `conference_results` view routes to the Ember app
+    everyone: true
+
+  - flag_name: EMBER_MY_PROJECTS
+    name: ember_my_projects_page
+    note: This flag controls wheter the `My Projects Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_PROJECT_CONTRIBUTORS
+    name: ember_project_contributors_page
+    note: This flag controls wheter the `Node Contributor Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_PROJECT_SETTINGS
+    name: ember_project_settings_page
+    note: This flag controls wheter the `Node Settings Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_PROJECT_WIKI
+    name: ember_project_wiki_page
+    note: This flag controls wheter the `Project Wiki Home Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_REGISTRATION_FORM_DETAIL
+    name: ember_registration_form_detail_page
+    note: This flag controls wheter the `Node Register Template Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_SEARCH_PAGE
+    name: ember_search_page
+    note: This flag controls wheter the `Search Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_USER_PROFILE
+    name: ember_user_profile_page
+    note: This flag controls wheter the `User Profile Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_USER_SETTINGS
+    name: ember_user_settings_page
+    note: This flag controls wheter the `User Settings Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_USER_SETTINGS_ADDONS
+    name: ember_user_settings_addons_page
+    note: This flag controls wheter the `User Addons Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_USER_SETTINGS_NOTIFICATIONS
+    name: ember_user_settings_notifications_page
+    note: This flag controls wheter the `User Notifications Page` view routes to the Ember app
+    everyone: false
+
+  - flag_name: EMBER_MEETINGS
+    name: ember_meetings_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_EDIT_DRAFT_REGISTRATION
+    name: ember_edit_draft_registration_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_FILE_REGISTRATION_DETAIL
+    name: ember_file_registration_detail_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_REGISTRATION_FILES
+    name: ember_registration_files_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_REGISTRIES_DETAIL_PAGE
+    name: ember_registries_detail_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_USER_SETTINGS_ACCOUNTS
+    name: ember_user_settings_account_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_USER_SETTINGS_APPS
+    name: ember_user_settings_apps_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_USER_SETTINGS_TOKENS
+    name: ember_user_settings_tokens_page
+    note: This is complete and should be permanently on.
+    everyone: true
+
+  - flag_name: EMBER_AB_TESTING_HOME_PAGE_VERSION_B
+    name: ab_testing_home_page_version_b
+    note: This is no longer used.
+
+  - flag_name: EMBER_AB_TESTING_HOME_PAGE_HERO_TEXT_VERSION_B
+    name: ab_testing_home_page_hero_text_version_b
+    note: This is no longer used.
+    everyone: true
+
+  - flag_name: EMBER_PROJECT_ANALYTICS
+    name: ember_project_analytics_page
+    note: This is no longer used.
+    everyone: false
+
+  - flag_name: EMBER_PROJECT_FORKS
+    name: ember_project_forks_page
+    note: This is no longer used.
+    everyone: false
+
+  - flag_name: EMBER_PROJECT_REGISTRATIONS
+    name: ember_project_registrations_page
+    note: This is no longer used.
+    everyone: false
+
+  - flag_name: ENABLE_CHRONOS
+    name: enable_chronos
+    note: This is not used
+    everyone: true
+
+switches:
+  - flag_name: DISABLE_ENGAGEMENT_EMAILS
+    name: disable_engagement_emails
+    note: if set to true, prevents engagment emails from being sent
+    active: false
+
+  - flag_name: ELASTICSEARCH_METRICS
+    name: elasticsearch_metrics
+    note: enables ES metrics server
+    active: true
+
+  - flag_name: ENFORCE_CSRF
+    name: enforce_csrf
+    note: enforces csrf for OSF session authentication
+    active: true
+
+  - flag_name: ENABLE_RAW_METRICS
+    name: enable_raw_metrics
+    note: allows for raw queries againest our ES metrics database
+    active: false
+
+  - flag_name: OSF_PREREGISTRATION
+    name: osf_preregistration
+    note: This is no longer used
+
+  - flag_name: ENABLE_INACTIVE_SCHEMAS
+    name: enable_inactive_schemas
+    note: This is no longer used

osf/management/commands/manage_switch_flags.py

@@ -1,38 +1,36 @@
 # -*- coding: utf-8 -*-
 import logging
+import yaml
 
 from django.core.management.base import BaseCommand
-
-from osf.features import switches, flags
-from waffle.models import Flag, Switch
+from django.db import transaction
+from website import settings
 
 logger = logging.getLogger(__name__)
 
+
 def manage_waffle(delete_waffle=False):
-    file_switches = list(switches.values())
-    current_switches = Switch.objects.values_list('name', flat=True)
-
-    add_switches = set(file_switches) - set(current_switches)
-    for switch in add_switches:
-        Switch.objects.get_or_create(name=switch, defaults={'active': False})
-        logger.info('Adding switch: {}'.format(switch))
-
-    file_flags = list(flags.values())
-    current_flags = Flag.objects.values_list('name', flat=True)
-
-    add_flags = set(file_flags) - set(current_flags)
-    for flag_name in add_flags:
-        Flag.objects.get_or_create(name=flag_name, defaults={'everyone': False})
-        logger.info('Adding flag: {}'.format(flag_name))
-
-    if delete_waffle:
-        delete_switches = set(current_switches) - set(file_switches)
-        Switch.objects.filter(name__in=delete_switches).delete()
-        logger.info('Deleting switches: {}'.format(delete_switches))
-
-        delete_flags = set(current_flags) - set(file_flags)
-        Flag.objects.filter(name__in=delete_flags).delete()
-        logger.info('Deleting flags: {}'.format(delete_flags))
+    # Inline importation of models is done to so for use in post migrate signal.
+    from django.apps import apps
+    Flag = apps.get_model('waffle.Flag')
+    Switch = apps.get_model('waffle.Switch')
+
+    with transaction.atomic():
+        if delete_waffle:
+            results = Switch.objects.all().delete()
+            logger.info(f'Deleting switches: {results}')
+            results = Flag.objects.all().delete()
+            logger.info(f'Deleting flags: {results}')
+
+        with open(settings.WAFFLE_VALUES_YAML, 'r') as stream:
+            features = yaml.safe_load(stream)
+        for flag in features['flags']:
+            flag.pop('flag_name')
+            Flag.objects.update_or_create(name=flag['name'], defaults=flag)
+        for switch in features['switches']:
+            switch.pop('flag_name')
+            Switch.objects.update_or_create(name=switch['name'], defaults=switch)
+
 
 class Command(BaseCommand):
     """Ensure all features and switches are updated with the switch and flag files

osf/migrations/0083_add_ember_waffle_flags.py

@@ -56,5 +56,4 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(add_ember_waffle_flags, reverse_func)
     ]

osf/migrations/0121_remove_support_page_waffle_flag.py

@@ -20,5 +20,4 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(remove_support_page_waffle_flags, reverse_func)
     ]