diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index a8483bc9b6e..8a092a6173a 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -1084,10 +1084,20 @@ def can_rebuild(self): def external_version_name(self): return external_version_name(self) - def using_latest_config(self): - if self.config: - return int(self.config.get('version', '1')) == LATEST_CONFIGURATION_VERSION - return False + def deprecated_config_used(self): + """ + Check whether this particular build is using a deprecated config file. + + When using v1 or not having a config file at all, it returns ``True``. + Returns ``False`` only when it has a config file and it is using v2. + + Note we are using this to communicate deprecation of v1 file and not using a config file. + See https://github.com/readthedocs/readthedocs.org/issues/10342 + """ + if not self.config: + return True + + return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION def reset(self): """ diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index da5a16c667f..d802883cfbb 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -3,9 +3,13 @@ import structlog from celery.worker.request import Request -from django.db.models import Q +from django.conf import settings +from django.contrib.auth.models import User +from django.db.models import Q, Sum from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from djstripe.enums import SubscriptionStatus +from messages_extends.constants import WARNING_PERSISTENT from readthedocs.builds.constants import ( BUILD_FINAL_STATES, @@ -14,7 +18,12 @@ ) from readthedocs.builds.models import Build from readthedocs.builds.tasks import send_build_status +from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.filesystem import safe_rmtree +from readthedocs.notifications import Notification, SiteNotification +from readthedocs.notifications.backends import EmailBackend +from readthedocs.notifications.constants import REQUIREMENT +from readthedocs.projects.models import Project from readthedocs.storage import build_media_storage from readthedocs.worker import app @@ -154,6 +163,158 @@ def send_external_build_status(version_type, build_pk, commit, status): send_build_status.delay(build_pk, commit, status) +class DeprecatedConfigFileSiteNotification(SiteNotification): + + # TODO: mention all the project slugs here + # Maybe trim them to up to 5 projects to avoid sending a huge blob of text + failure_message = _( + 'Your project(s) "{{ project_slugs }}" don\'t have a configuration file. ' + "Configuration files will soon be required by projects, " + "and will no longer be optional. " + 'Read our blog post to create one ' # noqa + "and ensure your project continues building successfully." + ) + failure_level = WARNING_PERSISTENT + + +class DeprecatedConfigFileEmailNotification(Notification): + + app_templates = "projects" + name = "deprecated_config_file_used" + context_object_name = "project" + subject = "[Action required] Add a configuration file to your project to prevent build failure" + level = REQUIREMENT + + def send(self): + """Method overwritten to remove on-site backend.""" + backend = EmailBackend(self.request) + backend.send(self) + + +@app.task(queue="web") +def deprecated_config_file_used_notification(): + """ + Create a notification about not using a config file for all the maintainers of the project. + + This is a scheduled task to be executed on the webs. + Note the code uses `.iterator` and `.only` to avoid killing the db with this query. + Besdies, it excludes projects with enough spam score to be skipped. + """ + # Skip projects with a spam score bigger than this value. + # Currently, this gives us ~250k in total (from ~550k we have in our database) + spam_score = 300 + + projects = set() + start_datetime = datetime.datetime.now() + queryset = Project.objects.exclude(users__profile__banned=True) + if settings.ALLOW_PRIVATE_REPOS: + # Only send emails to active customers + queryset = queryset.filter( + organizations__stripe_subscription__status=SubscriptionStatus.active + ) + else: + # Take into account spam score on community + queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( + Q(spam_score__lt=spam_score) | Q(is_spam=False) + ) + queryset = queryset.only("slug", "default_version").order_by("id") + n_projects = queryset.count() + + for i, project in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Finding projects without a configuration file.", + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + projects_found=len(projects), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + # Only check for the default version because if the project is using tags + # they won't be able to update those and we will send them emails forever. + # We can update this query if we consider later. + version = ( + project.versions.filter(slug=project.default_version).only("id").first() + ) + if version: + build = ( + version.builds.filter(success=True) + .only("_config") + .order_by("-date") + .first() + ) + if build and build.deprecated_config_used(): + projects.add(project.slug) + + # Store all the users we want to contact + users = set() + + n_projects = len(projects) + queryset = Project.objects.filter(slug__in=projects).order_by("id") + for i, project in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Querying all the users we want to contact.", + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + users_found=len(users), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + users.update(AdminPermission.owners(project).values_list("username", flat=True)) + + # Only send 1 email per user, + # even if that user has multiple projects without a configuration file. + # The notification will mention all the projects. + queryset = User.objects.filter(username__in=users, profile__banned=False).order_by( + "id" + ) + n_users = queryset.count() + for i, user in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Sending deprecated config file notification to users.", + progress=f"{i}/{n_users}", + current_user_pk=user.pk, + current_user_username=user.username, + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + # All the projects for this user that don't have a configuration file + user_projects = ( + AdminPermission.projects(user, admin=True) + .filter(slug__in=projects) + .only("slug") + ) + + user_project_slugs = ", ".join([p.slug for p in user_projects[:5]]) + if user_projects.count() > 5: + user_project_slugs += " and others..." + + n_site = DeprecatedConfigFileSiteNotification( + user=user, + context_object=user_projects, + extra_context={"project_slugs": user_project_slugs}, + success=False, + ) + n_site.send() + + # TODO: uncomment this code when we are ready to send email notifications + # n_email = DeprecatedConfigFileEmailNotification( + # user=user, + # context_object=user_projects, + # extra_context={"project_slugs": user_project_slugs}, + # ) + # n_email.send() + + log.info( + "Finish sending deprecated config file notifications.", + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + class BuildRequest(Request): def on_timeout(self, soft, timeout): diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 2a4da4e1092..a58c47385f4 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -249,7 +249,7 @@ def test_build_is_stale(self): self.assertTrue(build_two.is_stale) self.assertFalse(build_three.is_stale) - def test_using_latest_config(self): + def test_deprecated_config_used(self): now = timezone.now() build = get( @@ -260,12 +260,12 @@ def test_using_latest_config(self): state='finished', ) - self.assertFalse(build.using_latest_config()) + self.assertTrue(build.deprecated_config_used()) build.config = {'version': 2} build.save() - self.assertTrue(build.using_latest_config()) + self.assertFalse(build.deprecated_config_used()) def test_build_is_external(self): # Turn the build version to EXTERNAL type. diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 139c2279fdb..b3e736d55e2 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -532,6 +532,11 @@ def TEMPLATES(self): 'schedule': crontab(minute='*/15'), 'options': {'queue': 'web'}, }, + 'weekly-config-file-notification': { + 'task': 'readthedocs.projects.tasks.utils.deprecated_config_file_used_notification', + 'schedule': crontab(day_of_week='wednesday', hour=11, minute=15), + 'options': {'queue': 'web'}, + }, } MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE] diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index e47d129ae9a..17bf9424074 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -161,18 +161,21 @@

{% endif %} - {% if build.finished and not build.using_latest_config %} + + {# This message is not dynamic and only appears when loading the page after the build has finished #} + {% if build.finished and build.deprecated_config_used %}

- {% blocktrans trimmed with config_file_link="https://docs.readthedocs.io/page/config-file/v2.html" %} - Configure your documentation builds! - Adding a .readthedocs.yaml file to your project - is the recommended way to configure your documentation builds. - You can declare dependencies, set up submodules, and many other great features. + {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %} + Your builds will stop working soon!
+ Configuration files will soon be required by projects, and will no longer be optional. + Read our blog post to create one + and ensure your project continues building successfully. {% endblocktrans %}

{% endif %} + {% endif %} {% if build.finished and build.config.build.commands %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html new file mode 100644 index 00000000000..81778d139cb --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -0,0 +1,29 @@ +{% extends "core/email/common.html" %} +{% block content %} +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023. +We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. +Keep these dates in mind to avoid unexpected behaviours: + + + +We have identified the following projects where you are admin are impacted by this deprecation: + + + +You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications. + +For more information on how to create a required configuration file, +read our blog post + +Get in touch with us via our support +and let us know if you are unable to use a configuration file for any reason. +{% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt new file mode 100644 index 00000000000..31d7fb8734c --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -0,0 +1,25 @@ +{% extends "core/email/common.txt" %} +{% block content %} +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023. +We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. +Keep these dates in mind to avoid unexpected behaviours: + +* Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon) +* Monday, August 14, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, September 4, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, September 25, 2023: Fully remove support for building documentation without configuration file v2. + +We have identified the following projects where you are admin are impacted by this deprecation: + +{% for project in object %} +* {{ project.slug }} +{% endfor %} + +You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications. + +For more information on how to create a required configuration file, see: +https://blog.readthedocs.com/migrate-configuration-v2/ + +Get in touch with us at {{ production_uri }}{% url 'support' %} +and let us know if you are unable to use a configuration file for any reason. +{% endblock %}