Merge branch 'develop' of https://github.com/CenterForOpenScience/osf.io

 into update-README

* 'develop' of https://github.com/CenterForOpenScience/osf.io:
  Bump version and update CHANGELOG
  [ENG-3854] Add management command to sync all DOI metadata (CenterForOpenScience#9972)
  Re-add non-anonymized fields removed in CenterForOpenScience#10009 (CenterForOpenScience#10022)
  Update shield logo for colorado (UC Boulder)
  Update description for maglab
  [ENG-3249] Improve registration anonymization (CenterForOpenScience#10009)
  "backport" artifact changes and swap schema_response.justification (CenterForOpenScience#10003)
  Add new instn purdue
  Ensure BitBucket token is string, not bytes

CHANGELOG

@@ -2,6 +2,13 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+22.07.0 (2022-08-18)
+====================
+- Migrations in support of Registration Resources
+- Fix for Admin App colorpicker
+- Management command to sync all DOIs for OSF resources
+- Change "revision_justification" to TextField to alleviate max length concerns
+
 22.06.0 (2022-06-23)
 ====================
 - Fix support for Dataverse files

addons/bitbucket/api.py

@@ -3,14 +3,14 @@
 from addons.bitbucket import settings
 
 from framework.exceptions import HTTPError
-
+from osf.utils.fields import ensure_str
 from website.util.client import BaseClient
 
 
 class BitbucketClient(BaseClient):
 
     def __init__(self, access_token=None):
-        self.access_token = access_token
+        self.access_token = ensure_str(access_token)
 
     @property
     def _default_headers(self):

api_tests/identifiers/managment_commands/test_sync_dois.py

@@ -0,0 +1,97 @@
+import pytest
+import datetime
+from django.utils import timezone
+
+from django.core.management import call_command
+
+from osf_tests.factories import (
+    PreprintFactory,
+    RegistrationFactory,
+)
+
+from website import settings
+from website.identifiers.clients import CrossRefClient
+
+
+@pytest.mark.django_db
+class TestSyncDOIs:
+
+    @pytest.fixture()
+    def preprint(self):
+        preprint = PreprintFactory()
+        doi = CrossRefClient(settings.CROSSREF_URL).build_doi(preprint)
+        preprint.set_identifier_value('doi', doi)
+        return preprint
+
+    @pytest.fixture()
+    def registration(self):
+        registration = RegistrationFactory()
+        doi = registration.request_identifier_update('doi')
+        registration.set_identifier_value('doi', doi)
+        registration.is_public = True
+        registration.save()
+        return registration
+
+    @pytest.fixture()
+    def registration_private(self, registration):
+        registration.is_public = False
+        registration.save()
+        return registration
+
+    @pytest.fixture()
+    def registration_identifier(self, registration):
+        identifier = registration.identifiers.first()
+        identifier.modified = timezone.now() - datetime.timedelta(days=1)
+        identifier.save(update_modified=False)
+        return identifier
+
+    @pytest.fixture()
+    def preprint_identifier(self, preprint):
+        identifier = preprint.identifiers.first()
+        identifier.modified = timezone.now() - datetime.timedelta(days=1)
+        identifier.save(update_modified=False)
+        return identifier
+
+    @pytest.mark.enable_enqueue_task
+    def test_doi_synced_datacite(self, app, registration, registration_identifier, mock_datacite):
+        assert registration_identifier.modified.date() < datetime.datetime.now().date()
+
+        call_command('sync_doi_metadata', f'-m={datetime.datetime.now()}')
+        assert len(mock_datacite.calls) == 2
+        update_metadata, update_doi = mock_datacite.calls
+        assert update_metadata.request.url == f'{settings.DATACITE_URL}/metadata'
+        assert update_doi.request.url == f'{settings.DATACITE_URL}/doi'
+
+        registration_identifier.reload()
+        assert registration_identifier.modified.date() == datetime.datetime.now().date()
+
+    @pytest.mark.enable_enqueue_task
+    def test_doi_synced_crossref(self, app, preprint_identifier, mock_crossref):
+        assert preprint_identifier.modified.date() < datetime.datetime.now().date()
+
+        call_command('sync_doi_metadata', f'-m={datetime.datetime.now()}')
+
+        assert len(mock_crossref.calls) == 1
+
+        preprint_identifier.reload()
+        assert preprint_identifier.modified.date() == datetime.datetime.now().date()
+
+    @pytest.mark.enable_enqueue_task
+    def test_doi_sync_private(self, app, registration_private, registration_identifier, mock_datacite):
+
+        assert registration_identifier.modified.date() < datetime.datetime.now().date()
+
+        call_command('sync_doi_metadata', f'-m={datetime.datetime.now()}', '--sync_private')
+
+        datacite_request, = mock_datacite.calls
+        assert datacite_request.request.method == 'DELETE'  # removes metadata for private
+        assert datacite_request.request.url == f'{settings.DATACITE_URL}/metadata/10.70102/FK2osf.io/{registration_private._id}'
+
+        assert registration_identifier.modified.date() < datetime.datetime.now().date()
+        assert registration_identifier.modified.date() < datetime.datetime.now().date()
+
+    @pytest.mark.enable_enqueue_task
+    def test_doi_sync_public_only(self, app, registration_private, registration_identifier, mock_datacite):
+        call_command('sync_doi_metadata', f'-m={datetime.datetime.now()}')
+
+        assert len(mock_datacite.calls) == 0

conftest.py

@@ -201,6 +201,20 @@ def mock_datacite(registration):
                     yield rsps
 
 
+@pytest.fixture
+def mock_crossref():
+    """
+    This should be used to mock our our crossref integration.
+    Relevant endpoints:
+    """
+    with mock.patch.object(website_settings, 'CROSSREF_URL', 'https://test.crossref.org/servlet/deposit'):
+        with mock.patch.object(website_settings, 'CROSSREF_USERNAME', 'TestCrossrefUsername'):
+            with mock.patch.object(website_settings, 'CROSSREF_PASSWORD', 'TestCrossrefPassword'):
+                with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
+                    rsps.add(responses.POST, website_settings.CROSSREF_URL, status=200)
+                    yield rsps
+
+
 @pytest.fixture
 def mock_oopspam():
     """

osf/management/commands/sync_doi_metadata.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import datetime
+import logging
+
+from django.core.management.base import BaseCommand
+from osf.models import Identifier
+
+from framework.celery_tasks import app
+from framework.celery_tasks.handlers import enqueue_task
+
+logger = logging.getLogger(__name__)
+
+
+@app.task(name='osf.management.commands.sync_doi_metadata', max_retries=5, default_retry_delay=60)
+def sync_identifier_doi(identifier):
+    identifier.referent.request_identifier_update('doi')
+    identifier.save()
+    logger.info(f' doi update for {identifier.value} complete')
+
+
+def sync_doi_metadata(modified_date, batch_size=100, dry_run=True, sync_private=False):
+
+    identifiers = Identifier.objects.filter(
+        category='doi',
+        deleted__isnull=True,
+        modified__lte=modified_date,
+        object_id__isnull=False,
+    )[:batch_size]
+    logger.info(f'{"[DRY RUN]: " if dry_run else ""}'
+                f'{identifiers.count()} identifiers to mint')
+
+    for identifier in identifiers:
+        if not dry_run:
+            if (identifier.referent.is_public and not identifier.referent.deleted and not identifier.referent.is_retracted) or sync_private:
+                enqueue_task(sync_identifier_doi.s(identifier))
+
+        logger.info(f'{"[DRY RUN]: " if dry_run else ""}'
+                    f' doi minting for {identifier.value} started')
+
+
+class Command(BaseCommand):
+    """ Adds updates all DOIs, will remove metadata for DOI bearing resources that have been withdrawn. """
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--dry_run',
+            action='store_true',
+            dest='dry_run',
+        )
+        parser.add_argument(
+            '--sync_private',
+            action='store_true',
+            dest='sync_private',
+        )
+        parser.add_argument(
+            '--batch_size',
+            '-b',
+            type=int,
+            default=100,
+            help='number of dois to update in this batch.',
+        )
+        parser.add_argument(
+            '--modified_date',
+            '-m',
+            type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S.%f'),
+            help='include all dois updated before this date.',
+            required=True
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options.get('dry_run')
+        sync_private = options.get('sync_private')
+        batch_size = options.get('batch_size')
+        modified_date = options.get('modified_date')
+        sync_doi_metadata(modified_date, batch_size, dry_run=dry_run, sync_private=sync_private)

osf/migrations/0248_artifact_tweaks.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2022-08-01 19:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osf', '0247_artifact_finalized_and_deleted'),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name='outcomeartifact',
+            name='osf_outcome_outcome_a62f5c_idx',
+        ),
+        migrations.AlterField(
+            model_name='outcomeartifact',
+            name='identifier',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artifact_metadata', to='osf.Identifier'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='outcomeartifact',
+            unique_together=set([]),
+        ),
+        migrations.AddIndex(
+            model_name='outcomeartifact',
+            index=models.Index(fields=['artifact_type', 'outcome'], name='osf_outcome_artifac_5eb92d_idx'),
+        ),
+    ]

osf/migrations/0249_schema_response_justification_to_text_field.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2022-08-03 13:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osf', '0248_artifact_tweaks'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='schemaresponse',
+            name='revision_justification',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

osf/models/outcome_artifacts.py

@@ -79,7 +79,8 @@ class OutcomeArtifact(ObjectIDMixin, BaseModel):
     identifier = models.ForeignKey(
         'osf.identifier',
         null=True,
-        on_delete=models.CASCADE,
+        blank=True,
+        on_delete=models.SET_NULL,
         related_name='artifact_metadata'
     )
 
@@ -97,9 +98,8 @@ class OutcomeArtifact(ObjectIDMixin, BaseModel):
     objects = ArtifactManager()
 
     class Meta:
-        unique_together = ('outcome', 'identifier', 'artifact_type')
         indexes = [
-            models.Index(fields=['outcome', 'artifact_type'])
+            models.Index(fields=['artifact_type', 'outcome'])
         ]
         ordering = ['artifact_type', 'title']
 

osf/models/schema_response.py

@@ -54,7 +54,7 @@ class SchemaResponse(ObjectIDMixin, BaseModel):
         blank=True
     )
 
-    revision_justification = models.CharField(max_length=2048, null=True, blank=True)
+    revision_justification = models.TextField(null=True, blank=True)
     submitted_timestamp = NonNaiveDateTimeField(null=True, blank=True)
 
     pending_approvers = models.ManyToManyField('osf.osfuser', related_name='pending_submissions')

package.json

@@ -1,6 +1,6 @@
 {
   "name": "OSF",
-  "version": "22.06.0",
+  "version": "22.07.0",
   "description": "Facilitating Open Science",
   "repository": "https://github.com/CenterForOpenScience/osf.io",
   "author": "Center for Open Science",

scripts/populate_institutions.py

@@ -592,7 +592,7 @@ def main(default_args=False):
             {
                 '_id': 'nationalmaglab',
                 'name': 'National High Magnetic Field Laboratory',
-                'description': 'This research project management platform is provided by the National MagLab to enable collaboration, sharing, and dissemination of the products of user research according to the principles of <a href="https://www.go-fair.org/fair-principles/">FAIR</a> and open science. All public projects must adhere to <a href="https://nationalmaglab.org/about/policies-procedures">MagLab policies & procedures</a> related to confidentiality and proper data management.',
+                'description': 'This platform is provided to enable collaboration, sharing, and dissemination of research products from the National High Magnetic Field Laboratory according to the principles of <a href="https://www.go-fair.org/fair-principles/">FAIR</a> and open science. All public projects must adhere to <a href="https://nationalmaglab.org/about/policies-procedures">National MagLab policies & procedures</a> related to confidentiality and proper data management.',
                 'banner_name': 'nationalmaglab-banner.png',
                 'logo_name': 'nationalmaglab-shield.png',
                 'login_url': SHIBBOLETH_SP_LOGIN.format(encode_uri_component('https://idp.fsu.edu')),
@@ -686,6 +686,18 @@ def main(default_args=False):
                 'email_domains': [],
                 'delegation_protocol': 'saml-shib',
             },
+            {
+                '_id': 'purdue',
+                'name': 'Purdue University',
+                'description': 'This open scholarship platform is provided by <a href="https://www.lib.purdue.edu/">Purdue University Libraries</a> in partnership with the University\'s <a href="https://www.purdue.edu/gradschool/">Graduate School</a>, <a href="https://www.purdue.edu/research/oevprp/regulatory-affairs/">Regulatory Affairs</a>, and <a href="https://www.purdue.edu/provost/researchIntegrity/">Research Integrity Office</a>.<br><br>All projects must adhere to Purdue\'s <a href="https://www.purdue.edu/policies/information-technology/viib8.html#statement">Information security</a>, <a href="https://www.purdue.edu/policies/academic-research-affairs/ic1.html">Human subjects research</a> policies, and related <a href="https://www.purdue.edu/securepurdue/data-handling/index.php">data classification and handling procedures</a>. Associated guidance on regulations is available via the <a href="https://www.purdue.edu/research/oevprp/regulatory-affairs/responsible-conduct.php">Responsible Conductof Research website</a> and the <a href="https://www.purdue.edu/provost/researchIntegrity/">Research Integrity Office</a>. For questions and support please reach out to <a href="mailto:[email protected]">Purdue\'s OSF contact</a>.',
+                'banner_name': 'purdue-banner.png',
+                'logo_name': 'purdue-shield.png',
+                'login_url': SHIBBOLETH_SP_LOGIN.format(encode_uri_component('https://idp.purdue.edu/idp/shibboleth')),
+                'logout_url': SHIBBOLETH_SP_LOGOUT.format(encode_uri_component('https://osf.io/goodbye')),
+                'domains': [],
+                'email_domains': [],
+                'delegation_protocol': 'saml-shib',
+            },
             {
                 '_id': 'sc',
                 'name': 'University of South Carolina Libraries',
@@ -1689,7 +1701,7 @@ def main(default_args=False):
             {
                 '_id': 'nationalmaglab',
                 'name': 'National High Magnetic Field Laboratory [Test]',
-                'description': 'This research project management platform is provided by the National MagLab to enable collaboration, sharing, and dissemination of the products of user research according to the principles of <a href="https://www.go-fair.org/fair-principles/">FAIR</a> and open science. All public projects must adhere to <a href="https://nationalmaglab.org/about/policies-procedures">MagLab policies & procedures</a> related to confidentiality and proper data management.',
+                'description': 'This platform is provided to enable collaboration, sharing, and dissemination of research products from the National High Magnetic Field Laboratory according to the principles of <a href="https://www.go-fair.org/fair-principles/">FAIR</a> and open science. All public projects must adhere to <a href="https://nationalmaglab.org/about/policies-procedures">National MagLab policies & procedures</a> related to confidentiality and proper data management.',
                 'banner_name': 'nationalmaglab-banner.png',
                 'logo_name': 'nationalmaglab-shield.png',
                 'login_url': SHIBBOLETH_SP_LOGIN.format(encode_uri_component('https://idp.fsu.edu')),
@@ -1783,6 +1795,18 @@ def main(default_args=False):
                 'email_domains': [],
                 'delegation_protocol': 'saml-shib',
             },
+            {
+                '_id': 'purdue',
+                'name': 'Purdue University [Test]',
+                'description': 'This open scholarship platform is provided by <a href="https://www.lib.purdue.edu/">Purdue University Libraries</a> in partnership with the University\'s <a href="https://www.purdue.edu/gradschool/">Graduate School</a>, <a href="https://www.purdue.edu/research/oevprp/regulatory-affairs/">Regulatory Affairs</a>, and <a href="https://www.purdue.edu/provost/researchIntegrity/">Research Integrity Office</a>.<br><br>All projects must adhere to Purdue\'s <a href="https://www.purdue.edu/policies/information-technology/viib8.html#statement">Information security</a>, <a href="https://www.purdue.edu/policies/academic-research-affairs/ic1.html">Human subjects research</a> policies, and related <a href="https://www.purdue.edu/securepurdue/data-handling/index.php">data classification and handling procedures</a>. Associated guidance on regulations is available via the <a href="https://www.purdue.edu/research/oevprp/regulatory-affairs/responsible-conduct.php">Responsible Conductof Research website</a> and the <a href="https://www.purdue.edu/provost/researchIntegrity/">Research Integrity Office</a>. For questions and support please reach out to <a href="mailto:[email protected]">Purdue\'s OSF contact</a>.',
+                'banner_name': 'purdue-banner.png',
+                'logo_name': 'purdue-shield.png',
+                'login_url': SHIBBOLETH_SP_LOGIN.format(encode_uri_component('https://idp.purdue.edu/idp/shibboleth')),
+                'logout_url': SHIBBOLETH_SP_LOGOUT.format(encode_uri_component('https://test.osf.io/goodbye')),
+                'domains': [],
+                'email_domains': [],
+                'delegation_protocol': 'saml-shib',
+            },
             {
                 '_id': 'sc',
                 'name': 'University of South Carolina Libraries [Test]',