Merge branch 'feature/django_upgrade' of https://github.com/CenterFor…

…OpenScience/osf.io into signal-storage-region

* 'feature/django_upgrade' of https://github.com/CenterForOpenScience/osf.io:
  [ENG-3866] Move citation style population out of migration stream (#9966)
  [ENG-3868] Move blocked email domains to post-migrate signal (#9958)
  [ENG-3863] Move schema ensuring and schema blocks update to post-migrate signals (#9974)
  Instrument the ORCiD SSO affiliation flow * Existing user with verified ORCiD ID * Existing user confirmation of linking ORCiD ID * New user confirmation of account creation with ORCiD ID
  Add a django command script to handle instn sso email domain changes
  Bump version and CHANGELOG

# Conflicts:
#	osf/apps.py
#	osf/migrations/__init__.py

CHANGELOG

@@ -2,6 +2,12 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+22.06.0 (2022-06-23)
+====================
+- Fix support for Dataverse files
+- Match Legacy behavior with new `show_as_unviewed` File field
+- Other assorted fixes for new Files page
+
 22.05.0 (2022-06-09)
 ====================
 - Add institutional affiliations via ROR to minted DOIs

api/base/settings/defaults.py

@@ -351,3 +351,5 @@
 DEFAULT_ES_NULL_VALUE = 'N/A'
 
 TRAVIS_ENV = False
+
+CITATION_STYLES_REPO_URL = 'https://github.com/CenterForOpenScience/styles/archive/88e6ed31a91e9f5a480b486029cda97b535935d4.zip'

framework/auth/cas.py

@@ -10,6 +10,7 @@
 from framework.auth import authenticate, external_first_login_authenticate
 from framework.auth.core import get_user, generate_verification_key
 from framework.auth.utils import print_cas_log, LogLevel
+from framework.celery_tasks.handlers import enqueue_task
 from framework.flask import redirect
 from framework.exceptions import HTTPError
 from website import settings
@@ -376,6 +377,9 @@ def get_user_from_cas_resp(cas_resp):
                             external_id=external_credential['id'])
             # existing user found
             if user:
+                # Send to celery the following async task to affiliate the user with eligible institutions if verified
+                from framework.auth.tasks import update_affiliation_for_orcid_sso_users
+                enqueue_task(update_affiliation_for_orcid_sso_users.s(user._id, external_credential['id']))
                 return user, external_credential, 'authenticate'
             # user first time login through external identity provider
             else:

framework/auth/tasks.py

@@ -1,11 +1,23 @@
 from datetime import datetime
+import itertools
+import logging
+
+from lxml import etree
 import pytz
+import requests
+
+from framework import sentry
+from framework.celery_tasks import app as celery_app
+from website.settings import (DATE_LAST_LOGIN_THROTTLE_DELTA, EXTERNAL_IDENTITY_PROFILE,
+                              ORCID_PUBLIC_API_V3_URL, ORCID_PUBLIC_API_ACCESS_TOKEN,
+                              ORCID_PUBLIC_API_REQUEST_TIMEOUT, ORCID_RECORD_ACCEPT_TYPE,
+                              ORCID_RECORD_EDUCATION_PATH, ORCID_RECORD_EMPLOYMENT_PATH)
+
 
-from framework.celery_tasks import app
-from website.settings import DATE_LAST_LOGIN_THROTTLE_DELTA
+logger = logging.getLogger(__name__)
 
 
-@app.task
+@celery_app.task()
 def update_user_from_activity(user_id, login_time, cas_login=False, updates=None):
     from osf.models import OSFUser
     if not updates:
@@ -27,3 +39,122 @@ def update_user_from_activity(user_id, login_time, cas_login=False, updates=None
         should_save = True
     if should_save:
         user.save()
+
+
+@celery_app.task()
+def update_affiliation_for_orcid_sso_users(user_id, orcid_id):
+    """This is an asynchronous task that runs during CONFIRMED ORCiD SSO logins and makes eligible
+    institution affiliations.
+    """
+    from osf.models import OSFUser
+    user = OSFUser.load(user_id)
+    if not user or not verify_user_orcid_id(user, orcid_id):
+        # This should not happen as long as this task is called at the right place at the right time.
+        error_message = f'Invalid ORCiD ID [{orcid_id}] for [{user_id}]' if user else f'User [{user_id}] Not Found'
+        logger.error(error_message)
+        sentry.log_message(error_message)
+        return
+    institution = check_institution_affiliation(orcid_id)
+    if institution:
+        logger.info(f'Eligible institution affiliation has been found for ORCiD SSO user: '
+                    f'institution=[{institution._id}], user=[{user_id}], orcid_id=[{orcid_id}]')
+        if not user.is_affiliated_with_institution(institution):
+            user.affiliated_institutions.add(institution)
+
+
+def verify_user_orcid_id(user, orcid_id):
+    """Verify that the given ORCiD ID is verified for the given user.
+    """
+    provider = EXTERNAL_IDENTITY_PROFILE.get('OrcidProfile')
+    status = user.external_identity.get(provider, {}).get(orcid_id, None)
+    return status == 'VERIFIED'
+
+
+def check_institution_affiliation(orcid_id):
+    """Check user's public ORCiD record and return eligible institution affiliations.
+
+    Note: Current implementation only support one affiliation (i.e. loop returns once eligible
+          affiliation is found, which improves performance). In the future, if we have multiple
+          institutions using this feature, we can update the loop easily.
+    """
+    from osf.models import Institution
+    from osf.models.institution import IntegrationType
+    employment_source_list = get_orcid_employment_sources(orcid_id)
+    education_source_list = get_orcid_education_sources(orcid_id)
+    via_orcid_institutions = Institution.objects.filter(
+        delegation_protocol=IntegrationType.AFFILIATION_VIA_ORCID.value,
+        is_deleted=False
+    )
+    # Check both employment and education records
+    for source in itertools.chain(employment_source_list, education_source_list):
+        # Check source against all "affiliation-via-orcid" institutions
+        for institution in via_orcid_institutions:
+            if source == institution.orcid_record_verified_source:
+                logger.debug(f'Institution has been found with matching source: '
+                             f'institution=[{institution._id}], source=[{source}], orcid_id=[{orcid_id}]')
+                return institution
+    logger.debug(f'No institution with matching source has been found: orcid_id=[{orcid_id}]')
+    return None
+
+
+def get_orcid_employment_sources(orcid_id):
+    """Retrieve employment records for the given ORCiD ID.
+    """
+    employment_data = orcid_public_api_make_request(ORCID_RECORD_EMPLOYMENT_PATH, orcid_id)
+    source_list = []
+    if employment_data is not None:
+        affiliation_groups = employment_data.findall('{http://www.orcid.org/ns/activities}affiliation-group')
+        for affiliation_group in affiliation_groups:
+            employment_summary = affiliation_group.find('{http://www.orcid.org/ns/employment}employment-summary')
+            source = employment_summary.find('{http://www.orcid.org/ns/common}source')
+            source_name = source.find('{http://www.orcid.org/ns/common}source-name')
+            source_list.append(source_name.text)
+    return source_list
+
+
+def get_orcid_education_sources(orcid_id):
+    """Retrieve education records for the given ORCiD ID.
+    """
+    education_data = orcid_public_api_make_request(ORCID_RECORD_EDUCATION_PATH, orcid_id)
+    source_list = []
+    if education_data is not None:
+        affiliation_groups = education_data.findall('{http://www.orcid.org/ns/activities}affiliation-group')
+        for affiliation_group in affiliation_groups:
+            education_summary = affiliation_group.find('{http://www.orcid.org/ns/education}education-summary')
+            source = education_summary.find('{http://www.orcid.org/ns/common}source')
+            source_name = source.find('{http://www.orcid.org/ns/common}source-name')
+            source_list.append(source_name.text)
+    return source_list
+
+
+def orcid_public_api_make_request(path, orcid_id):
+    """Make the ORCiD public API request and returned a deserialized response.
+    """
+    request_url = ORCID_PUBLIC_API_V3_URL + orcid_id + path
+    headers = {
+        'Accept': ORCID_RECORD_ACCEPT_TYPE,
+        'Authorization': f'Bearer {ORCID_PUBLIC_API_ACCESS_TOKEN}',
+    }
+    try:
+        response = requests.get(request_url, headers=headers, timeout=ORCID_PUBLIC_API_REQUEST_TIMEOUT)
+    except Exception:
+        error_message = f'ORCiD public API request has encountered an exception: url=[{request_url}]'
+        logger.error(error_message)
+        sentry.log_message(error_message)
+        sentry.log_exception()
+        return None
+    if response.status_code != 200:
+        error_message = f'ORCiD public API request has failed: url=[{request_url}], ' \
+                        f'status=[{response.status_code}], response = [{response.content}]'
+        logger.error(error_message)
+        sentry.log_message(error_message)
+        return None
+    try:
+        xml_data = etree.XML(response.content)
+    except Exception:
+        error_message = 'Fail to read and parse ORCiD record response as XML'
+        logger.error(error_message)
+        sentry.log_message(error_message)
+        sentry.log_exception()
+        return None
+    return xml_data

framework/auth/views.py

@@ -20,6 +20,7 @@
 from framework.auth.decorators import block_bing_preview, collect_auth, must_be_logged_in
 from framework.auth.forms import ResendConfirmationForm, ForgotPasswordForm, ResetPasswordForm
 from framework.auth.utils import ensure_external_identity_uniqueness, validate_recaptcha
+from framework.celery_tasks.handlers import enqueue_task
 from framework.exceptions import HTTPError
 from framework.flask import redirect  # VOL-aware redirect
 from framework.sessions.utils import remove_sessions_for_user, remove_session
@@ -672,6 +673,10 @@ def external_login_confirm_email_get(auth, uid, token):
             can_change_preferences=False,
         )
 
+    # Send to celery the following async task to affiliate the user with eligible institutions if verified
+    from framework.auth.tasks import update_affiliation_for_orcid_sso_users
+    enqueue_task(update_affiliation_for_orcid_sso_users.s(user._id, provider_id))
+
     # redirect to CAS and authenticate the user with the verification key
     return redirect(cas.get_login_url(
         service_url,

osf/apps.py

@@ -1,10 +1,14 @@
 import logging
+
 from django.apps import AppConfig as BaseAppConfig
 from django.db.models.signals import post_migrate
+
 from osf.migrations import (
+    add_registration_schemas,
+    create_cache_table,
+    update_blocked_email_domains,
     update_permission_groups,
     update_waffle_flags,
-    create_cache_table,
     update_storage_regions,
     update_license
 )
@@ -13,16 +17,24 @@
 
 
 class AppConfig(BaseAppConfig):
+
     name = 'osf'
     app_label = 'osf'
     managed = True
 
     def ready(self):
         super(AppConfig, self).ready()
+
+        post_migrate.connect(
+            add_registration_schemas,
+            dispatch_uid='osf.apps.add_registration_schemas'
+        )
+
         post_migrate.connect(
             update_permission_groups,
             dispatch_uid='osf.apps.update_permissions_groups'
         )
+
         post_migrate.connect(
             update_storage_regions,
             dispatch_uid='osf.apps.update_storage_regions'
@@ -31,11 +43,18 @@ def ready(self):
             update_license,
             dispatch_uid='osf.apps.ensure_licenses',
         )
+
         post_migrate.connect(
             update_waffle_flags,
             dispatch_uid='osf.apps.update_waffle_flags'
         )
+
         post_migrate.connect(
             create_cache_table,
             dispatch_uid='osf.apps.create_cache_table'
         )
+
+        post_migrate.connect(
+            update_blocked_email_domains,
+            dispatch_uid='osf.apps.update_blocked_email_domains'
+        )