From e3b548418161aef25a77d41004bff6296590a91e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 00:33:05 -0700 Subject: [PATCH] Config: deprecated notification for projects without config file (#10354) * Config: deprecated notification for builds without config file When we detect a build is built without a Read the Docs configuration file (`.readthedocs.yaml`) we show multiple notifications: - a static warning message in the build detail's page - a persistent on-site notification to all maintainers/admin of the project - send a weekly email (at most) This is the initial step to attempt making users to migrate to our config file v2, giving them a enough window to do this and avoid breaking their builds in the future. Closes #10348 * Test: invert logic * Notification's copy: feedback from review * It's a function * Notifications: use a scheduled Celery task to send them Instead of sending an onsite notification on each build, we use a scheduled Celery task that runs once a week to send them. It filter projects that are not using the v2 config file. * Feedback from the review * Darker failed on CircleCI because of this * Links pointing to blog post * Add more logging for this task * Space typo * Ignore projects that are potentially spam * Order queryset by PK so we can track it Also, add log for current project in case we need to recover from that task. * Improve query a little more * Make the query to work on .com as well * Query only active subscriptions on .com * Consistency on naming * Only check for `Project.default_version` * Log progress while iterating to know it's moving * Simplify versions query * More logging to the progress * Send only one notification per user The notification will include all the projects the user is admin that are affected by this deprecation. Users will receive at most one notification per week. * Modify email template to include all the projects and dates * Typo * Improve logging * Keep adding logging :) * Db query for active subscriptions on .com * Email subject * Update onsite notification message * Do not set emails just yet * Minor updates * Update emails with new dates --- readthedocs/builds/models.py | 18 +- readthedocs/projects/tasks/utils.py | 163 +++++++++++++++++- readthedocs/rtd_tests/tests/test_builds.py | 6 +- readthedocs/settings/base.py | 5 + .../templates/builds/build_detail.html | 15 +- .../deprecated_config_file_used_email.html | 29 ++++ .../deprecated_config_file_used_email.txt | 25 +++ 7 files changed, 247 insertions(+), 14 deletions(-) create mode 100644 readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html create mode 100644 readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index a8483bc9b6e..8a092a6173a 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -1084,10 +1084,20 @@ def can_rebuild(self): def external_version_name(self): return external_version_name(self) - def using_latest_config(self): - if self.config: - return int(self.config.get('version', '1')) == LATEST_CONFIGURATION_VERSION - return False + def deprecated_config_used(self): + """ + Check whether this particular build is using a deprecated config file. + + When using v1 or not having a config file at all, it returns ``True``. + Returns ``False`` only when it has a config file and it is using v2. + + Note we are using this to communicate deprecation of v1 file and not using a config file. + See https://github.com/readthedocs/readthedocs.org/issues/10342 + """ + if not self.config: + return True + + return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION def reset(self): """ diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index da5a16c667f..d802883cfbb 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -3,9 +3,13 @@ import structlog from celery.worker.request import Request -from django.db.models import Q +from django.conf import settings +from django.contrib.auth.models import User +from django.db.models import Q, Sum from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from djstripe.enums import SubscriptionStatus +from messages_extends.constants import WARNING_PERSISTENT from readthedocs.builds.constants import ( BUILD_FINAL_STATES, @@ -14,7 +18,12 @@ ) from readthedocs.builds.models import Build from readthedocs.builds.tasks import send_build_status +from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.filesystem import safe_rmtree +from readthedocs.notifications import Notification, SiteNotification +from readthedocs.notifications.backends import EmailBackend +from readthedocs.notifications.constants import REQUIREMENT +from readthedocs.projects.models import Project from readthedocs.storage import build_media_storage from readthedocs.worker import app @@ -154,6 +163,158 @@ def send_external_build_status(version_type, build_pk, commit, status): send_build_status.delay(build_pk, commit, status) +class DeprecatedConfigFileSiteNotification(SiteNotification): + + # TODO: mention all the project slugs here + # Maybe trim them to up to 5 projects to avoid sending a huge blob of text + failure_message = _( + 'Your project(s) "{{ project_slugs }}" don\'t have a configuration file. ' + "Configuration files will soon be required by projects, " + "and will no longer be optional. " + 'Read our blog post to create one ' # noqa + "and ensure your project continues building successfully." + ) + failure_level = WARNING_PERSISTENT + + +class DeprecatedConfigFileEmailNotification(Notification): + + app_templates = "projects" + name = "deprecated_config_file_used" + context_object_name = "project" + subject = "[Action required] Add a configuration file to your project to prevent build failure" + level = REQUIREMENT + + def send(self): + """Method overwritten to remove on-site backend.""" + backend = EmailBackend(self.request) + backend.send(self) + + +@app.task(queue="web") +def deprecated_config_file_used_notification(): + """ + Create a notification about not using a config file for all the maintainers of the project. + + This is a scheduled task to be executed on the webs. + Note the code uses `.iterator` and `.only` to avoid killing the db with this query. + Besdies, it excludes projects with enough spam score to be skipped. + """ + # Skip projects with a spam score bigger than this value. + # Currently, this gives us ~250k in total (from ~550k we have in our database) + spam_score = 300 + + projects = set() + start_datetime = datetime.datetime.now() + queryset = Project.objects.exclude(users__profile__banned=True) + if settings.ALLOW_PRIVATE_REPOS: + # Only send emails to active customers + queryset = queryset.filter( + organizations__stripe_subscription__status=SubscriptionStatus.active + ) + else: + # Take into account spam score on community + queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( + Q(spam_score__lt=spam_score) | Q(is_spam=False) + ) + queryset = queryset.only("slug", "default_version").order_by("id") + n_projects = queryset.count() + + for i, project in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Finding projects without a configuration file.", + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + projects_found=len(projects), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + # Only check for the default version because if the project is using tags + # they won't be able to update those and we will send them emails forever. + # We can update this query if we consider later. + version = ( + project.versions.filter(slug=project.default_version).only("id").first() + ) + if version: + build = ( + version.builds.filter(success=True) + .only("_config") + .order_by("-date") + .first() + ) + if build and build.deprecated_config_used(): + projects.add(project.slug) + + # Store all the users we want to contact + users = set() + + n_projects = len(projects) + queryset = Project.objects.filter(slug__in=projects).order_by("id") + for i, project in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Querying all the users we want to contact.", + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + users_found=len(users), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + users.update(AdminPermission.owners(project).values_list("username", flat=True)) + + # Only send 1 email per user, + # even if that user has multiple projects without a configuration file. + # The notification will mention all the projects. + queryset = User.objects.filter(username__in=users, profile__banned=False).order_by( + "id" + ) + n_users = queryset.count() + for i, user in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Sending deprecated config file notification to users.", + progress=f"{i}/{n_users}", + current_user_pk=user.pk, + current_user_username=user.username, + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + # All the projects for this user that don't have a configuration file + user_projects = ( + AdminPermission.projects(user, admin=True) + .filter(slug__in=projects) + .only("slug") + ) + + user_project_slugs = ", ".join([p.slug for p in user_projects[:5]]) + if user_projects.count() > 5: + user_project_slugs += " and others..." + + n_site = DeprecatedConfigFileSiteNotification( + user=user, + context_object=user_projects, + extra_context={"project_slugs": user_project_slugs}, + success=False, + ) + n_site.send() + + # TODO: uncomment this code when we are ready to send email notifications + # n_email = DeprecatedConfigFileEmailNotification( + # user=user, + # context_object=user_projects, + # extra_context={"project_slugs": user_project_slugs}, + # ) + # n_email.send() + + log.info( + "Finish sending deprecated config file notifications.", + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + class BuildRequest(Request): def on_timeout(self, soft, timeout): diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 2a4da4e1092..a58c47385f4 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -249,7 +249,7 @@ def test_build_is_stale(self): self.assertTrue(build_two.is_stale) self.assertFalse(build_three.is_stale) - def test_using_latest_config(self): + def test_deprecated_config_used(self): now = timezone.now() build = get( @@ -260,12 +260,12 @@ def test_using_latest_config(self): state='finished', ) - self.assertFalse(build.using_latest_config()) + self.assertTrue(build.deprecated_config_used()) build.config = {'version': 2} build.save() - self.assertTrue(build.using_latest_config()) + self.assertFalse(build.deprecated_config_used()) def test_build_is_external(self): # Turn the build version to EXTERNAL type. diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 139c2279fdb..b3e736d55e2 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -532,6 +532,11 @@ def TEMPLATES(self): 'schedule': crontab(minute='*/15'), 'options': {'queue': 'web'}, }, + 'weekly-config-file-notification': { + 'task': 'readthedocs.projects.tasks.utils.deprecated_config_file_used_notification', + 'schedule': crontab(day_of_week='wednesday', hour=11, minute=15), + 'options': {'queue': 'web'}, + }, } MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE] diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index e47d129ae9a..17bf9424074 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -161,18 +161,21 @@

{% endif %} - {% if build.finished and not build.using_latest_config %} + + {# This message is not dynamic and only appears when loading the page after the build has finished #} + {% if build.finished and build.deprecated_config_used %}

- {% blocktrans trimmed with config_file_link="https://docs.readthedocs.io/page/config-file/v2.html" %} - Configure your documentation builds! - Adding a .readthedocs.yaml file to your project - is the recommended way to configure your documentation builds. - You can declare dependencies, set up submodules, and many other great features. + {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %} + Your builds will stop working soon!
+ Configuration files will soon be required by projects, and will no longer be optional. + Read our blog post to create one + and ensure your project continues building successfully. {% endblocktrans %}

{% endif %} + {% endif %} {% if build.finished and build.config.build.commands %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html new file mode 100644 index 00000000000..81778d139cb --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -0,0 +1,29 @@ +{% extends "core/email/common.html" %} +{% block content %} +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023. +We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. +Keep these dates in mind to avoid unexpected behaviours: + + + +We have identified the following projects where you are admin are impacted by this deprecation: + + + +You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications. + +For more information on how to create a required configuration file, +read our blog post + +Get in touch with us via our support +and let us know if you are unable to use a configuration file for any reason. +{% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt new file mode 100644 index 00000000000..31d7fb8734c --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -0,0 +1,25 @@ +{% extends "core/email/common.txt" %} +{% block content %} +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023. +We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. +Keep these dates in mind to avoid unexpected behaviours: + +* Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon) +* Monday, August 14, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, September 4, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, September 25, 2023: Fully remove support for building documentation without configuration file v2. + +We have identified the following projects where you are admin are impacted by this deprecation: + +{% for project in object %} +* {{ project.slug }} +{% endfor %} + +You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications. + +For more information on how to create a required configuration file, see: +https://blog.readthedocs.com/migrate-configuration-v2/ + +Get in touch with us at {{ production_uri }}{% url 'support' %} +and let us know if you are unable to use a configuration file for any reason. +{% endblock %}