diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 57166330510..03d91ca9142 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -1104,6 +1104,21 @@ def deprecated_config_used(self): return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION + def deprecated_build_image_used(self): + """ + Check whether this particular build is using the deprecated "build.image" config. + + Note we are using this to communicate deprecation of "build.image". + See https://github.com/readthedocs/meta/discussions/48 + """ + if not self.config: + # Don't notify users without a config file. + # We hope they will migrate to `build.os` in the process of adding a `.readthedocs.yaml` + return False + + build_config_key = self.config.get("build", {}) + return "image" in build_config_key + def reset(self): """ Reset the build so it can be re-used when re-trying. diff --git a/readthedocs/core/forms.py b/readthedocs/core/forms.py index 32eaef7a471..9ea8dba507b 100644 --- a/readthedocs/core/forms.py +++ b/readthedocs/core/forms.py @@ -23,7 +23,10 @@ class Meta: model = UserProfile # Don't allow users edit someone else's user page profile_fields = ["first_name", "last_name", "homepage"] - optout_email_fields = ["optout_email_config_file_deprecation"] + optout_email_fields = [ + "optout_email_config_file_deprecation", + "optout_email_build_image_deprecation", + ] fields = ( *profile_fields, *optout_email_fields, diff --git a/readthedocs/core/migrations/0014_optout_email_build_image_deprecation.py b/readthedocs/core/migrations/0014_optout_email_build_image_deprecation.py new file mode 100644 index 00000000000..5d2d94dcb9a --- /dev/null +++ b/readthedocs/core/migrations/0014_optout_email_build_image_deprecation.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.20 on 2023-08-01 13:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0013_add_optout_email_config_file_deprecation"), + ] + + operations = [ + migrations.AddField( + model_name="historicaluserprofile", + name="optout_email_build_image_deprecation", + field=models.BooleanField( + default=False, + null=True, + verbose_name="Opt-out from email about '\"build.image\" config key deprecation'", + ), + ), + migrations.AddField( + model_name="userprofile", + name="optout_email_build_image_deprecation", + field=models.BooleanField( + default=False, + null=True, + verbose_name="Opt-out from email about '\"build.image\" config key deprecation'", + ), + ), + ] diff --git a/readthedocs/core/models.py b/readthedocs/core/models.py index e476bd4f2f4..11efef7f34f 100644 --- a/readthedocs/core/models.py +++ b/readthedocs/core/models.py @@ -51,6 +51,13 @@ class UserProfile(TimeStampedModel): default=False, null=True, ) + # NOTE: this is a temporary field that we can remove after October 16, 2023 + # See https://blog.readthedocs.com/use-build-os-config/ + optout_email_build_image_deprecation = models.BooleanField( + _("Opt-out from email about '\"build.image\" config key deprecation'"), + default=False, + null=True, + ) # Model history history = ExtraHistoricalRecords() diff --git a/readthedocs/doc_builder/director.py b/readthedocs/doc_builder/director.py index e94b391b957..0bc4aeac859 100644 --- a/readthedocs/doc_builder/director.py +++ b/readthedocs/doc_builder/director.py @@ -13,6 +13,7 @@ import structlog import yaml from django.conf import settings +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from readthedocs.builds.constants import EXTERNAL @@ -255,6 +256,29 @@ def checkout(self): ) and self.data.config.version not in ("2", 2): raise BuildUserError(BuildUserError.NO_CONFIG_FILE_DEPRECATED) + # Raise a build error if the project is using "build.image" on their config file + + now = timezone.now() + + # fmt: off + # These browndates matches https://blog.readthedocs.com/use-build-os-config/ + browndates = any([ + timezone.datetime(2023, 8, 28, 0, 0, 0, tzinfo=timezone.utc) < now < timezone.datetime(2023, 8, 28, 12, 0, 0, tzinfo=timezone.utc), # First, 12hs + timezone.datetime(2023, 9, 18, 0, 0, 0, tzinfo=timezone.utc) < now < timezone.datetime(2023, 9, 19, 0, 0, 0, tzinfo=timezone.utc), # Second, 24hs + timezone.datetime(2023, 10, 2, 0, 0, 0, tzinfo=timezone.utc) < now < timezone.datetime(2023, 10, 4, 0, 0, 0, tzinfo=timezone.utc), # Third, 48hs + timezone.datetime(2023, 10, 16, 0, 0, 0, tzinfo=timezone.utc) < now, # Fully removal + ]) + # fmt: on + + if browndates: + build_config_key = self.data.config.source_config.get("build", {}) + if "image" in build_config_key: + raise BuildUserError(BuildUserError.BUILD_IMAGE_CONFIG_KEY_DEPRECATED) + + # TODO: move this validation to the Config object once we are settled here + if "image" not in build_config_key and "os" not in build_config_key: + raise BuildUserError(BuildUserError.BUILD_OS_REQUIRED) + if self.vcs_repository.supports_submodules: self.vcs_repository.update_submodules(self.data.config) diff --git a/readthedocs/doc_builder/exceptions.py b/readthedocs/doc_builder/exceptions.py index 8af29ee7f28..89d16011658 100644 --- a/readthedocs/doc_builder/exceptions.py +++ b/readthedocs/doc_builder/exceptions.py @@ -66,6 +66,15 @@ class BuildUserError(BuildBaseException): "Add a configuration file to your project to make it build successfully. " "Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html" ) + BUILD_IMAGE_CONFIG_KEY_DEPRECATED = gettext_noop( + 'The configuration key "build.image" is deprecated. ' + 'Use "build.os" instead to continue building your project. ' + "Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html#build-os" + ) + BUILD_OS_REQUIRED = gettext_noop( + 'The configuration key "build.os" is required to build your documentation. ' + "Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html#build-os" + ) class BuildUserSkip(BuildUserError): diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 09572c5d00f..6864d499d99 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -323,6 +323,160 @@ def deprecated_config_file_used_notification(): ) +class DeprecatedBuildImageSiteNotification(SiteNotification): + failure_message = _( + 'Your project(s) "{{ project_slugs }}" are using the deprecated "build.image" ' + 'config on their ".readthedocs.yaml" file. ' + 'This config is deprecated in favor of "build.os" and will be removed on October 16, 2023. ' # noqa + 'Read our blog post to migrate to "build.os" ' # noqa + "and ensure your project continues building successfully." + ) + failure_level = WARNING_PERSISTENT + + +class DeprecatedBuildImageEmailNotification(Notification): + app_templates = "projects" + name = "deprecated_build_image_used" + subject = '[Action required] Update your ".readthedocs.yaml" file to use "build.os"' + level = REQUIREMENT + + def send(self): + """Method overwritten to remove on-site backend.""" + backend = EmailBackend(self.request) + backend.send(self) + + +@app.task(queue="web") +def deprecated_build_image_notification(): + """ + Send an email notification about using "build.image" to all maintainers of the project. + + This is a scheduled task to be executed on the webs. + Note the code uses `.iterator` and `.only` to avoid killing the db with this query. + Besdies, it excludes projects with enough spam score to be skipped. + """ + # Skip projects with a spam score bigger than this value. + # Currently, this gives us ~250k in total (from ~550k we have in our database) + spam_score = 300 + + projects = set() + start_datetime = datetime.datetime.now() + queryset = Project.objects.exclude(users__profile__banned=True) + if settings.ALLOW_PRIVATE_REPOS: + # Only send emails to active customers + queryset = queryset.filter( + organizations__stripe_subscription__status=SubscriptionStatus.active + ) + else: + # Take into account spam score on community + queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( + Q(spam_score__lt=spam_score) | Q(is_spam=False) + ) + queryset = queryset.only("slug", "default_version").order_by("id") + n_projects = queryset.count() + + for i, project in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + 'Finding projects using "build.image" config key.', + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + projects_found=len(projects), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + # Only check for the default version because if the project is using tags + # they won't be able to update those and we will send them emails forever. + # We can update this query if we consider later. + version = ( + project.versions.filter(slug=project.default_version).only("id").first() + ) + if version: + # Use a fixed date here to avoid changing the date on each run + years_ago = timezone.datetime(2020, 8, 1) + build = ( + version.builds.filter(success=True, date__gt=years_ago) + .only("_config") + .order_by("-date") + .first() + ) + if build and build.deprecated_build_image_used(): + projects.add(project.slug) + + # Store all the users we want to contact + users = set() + + n_projects = len(projects) + queryset = Project.objects.filter(slug__in=projects).order_by("id") + for i, project in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + 'Querying all the users we want to contact about "build.image" deprecation.', + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + users_found=len(users), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + users.update(AdminPermission.owners(project).values_list("username", flat=True)) + + # Only send 1 email per user, + # even if that user has multiple projects using "build.image". + # The notification will mention all the projects. + queryset = User.objects.filter( + username__in=users, + profile__banned=False, + profile__optout_email_build_image_deprecation=False, + ).order_by("id") + + n_users = queryset.count() + for i, user in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + 'Sending deprecated "build.image" config key notification to users.', + progress=f"{i}/{n_users}", + current_user_pk=user.pk, + current_user_username=user.username, + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + # All the projects for this user that are using "build.image". + # Use set() intersection in Python that's pretty quick since we only need the slugs. + # Otherwise we have to pass 82k slugs to the DB query, which makes it pretty slow. + user_projects = AdminPermission.projects(user, admin=True).values_list( + "slug", flat=True + ) + user_projects_slugs = list(set(user_projects) & projects) + user_projects = Project.objects.filter(slug__in=user_projects_slugs) + + # Create slug string for onsite notification + user_project_slugs = ", ".join(user_projects_slugs[:5]) + if len(user_projects) > 5: + user_project_slugs += " and others..." + + n_site = DeprecatedBuildImageSiteNotification( + user=user, + context_object=user, + extra_context={"project_slugs": user_project_slugs}, + success=False, + ) + n_site.send() + + n_email = DeprecatedBuildImageEmailNotification( + user=user, + context_object=user, + extra_context={"projects": user_projects}, + ) + n_email.send() + + log.info( + 'Finish sending deprecated "build.image" config key notifications.', + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, + ) + + @app.task(queue="web") def set_builder_scale_in_protection(instance, protected_from_scale_in): """ diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index d52923c7533..b148854f50e 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -539,6 +539,11 @@ def TEMPLATES(self): 'schedule': crontab(day_of_week='wednesday', hour=11, minute=15), 'options': {'queue': 'web'}, }, + 'weekly-build-image-notification': { + 'task': 'readthedocs.projects.tasks.utils.deprecated_build_image_notification', + 'schedule': crontab(day_of_week='wednesday', hour=9, minute=15), + 'options': {'queue': 'web'}, + }, } # Sentry diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index 17bf9424074..5bc6428f140 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -163,17 +163,29 @@ {% endif %} {# This message is not dynamic and only appears when loading the page after the build has finished #} - {% if build.finished and build.deprecated_config_used %} -
+ {% if build.finished and build.deprecated_build_image_used %} +

- {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %} + {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/build-image-config-deprecated/" %} Your builds will stop working soon!
- Configuration files will soon be required by projects, and will no longer be optional. - Read our blog post to create one + "build.image" config key is deprecated and it will be removed soon. + Read our blog post to know how to migrate to new key "build.os" and ensure your project continues building successfully. - {% endblocktrans %} + {% endblocktrans %}

-
+
+ {% endif %} + {% if build.finished and build.deprecated_config_used %} +
+

+ {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %} + Your builds will stop working soon!
+ Configuration files will soon be required by projects, and will no longer be optional. + Read our blog post to create one + and ensure your project continues building successfully. + {% endblocktrans %} +

+
{% endif %} {% endif %} diff --git a/readthedocs/templates/projects/notifications/deprecated_build_image_used_email.html b/readthedocs/templates/projects/notifications/deprecated_build_image_used_email.html new file mode 100644 index 00000000000..05c45695145 --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_build_image_used_email.html @@ -0,0 +1,35 @@ +{% extends "core/email/common.html" %} +{% block content %} +The build.image config key on .readthedocs.yaml has been deprecated, and will be removed on October 16, 2023. +We are sending weekly notifications about this issue to all impacted users, +as well as temporary build failures (brownouts) as the date approaches for those who haven't migrated their projects. + +The timeline for this deprecation is as follows: + + + +We have identified that the following projects which you maintain, and were built in the last year, are impacted by this deprecation: + + + +Please use build.os on your configuration file to ensure that they continue building successfully and to stop receiving these notifications. +If you want to opt-out from these emails, you can edit your preferences in your account settings. + +For more information on how to use build.os, +read our blog post + +Get in touch with us via our support +and let us know if you are unable to use a configuration file for any reason. +{% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_build_image_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_build_image_used_email.txt new file mode 100644 index 00000000000..f68aa6fe3b1 --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_build_image_used_email.txt @@ -0,0 +1,31 @@ +{% extends "core/email/common.txt" %} +{% block content %} +The "build.image" config key on ".readthedocs.yaml" has been deprecated, and will be removed on October 16, 2023. +We are sending weekly notifications about this issue to all impacted users, +as well as temporary build failures (brownouts) as the date approaches for those who haven't migrated their projects. + +The timeline for this deprecation is as follows: + +* Monday, August 28, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon) +* Monday, September 18, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, October 2, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to October 3, 2023 23:59 PST (midnight) +* Monday, October 16, 2023: Fully remove support for building documentation using "build.image" on the configuration file + +We have identified that the following projects which you maintain, and were built in the last year, are impacted by this deprecation: + +{% for project in projects|slice:":15" %} +* {{ project.slug }} ({{ production_uri }}{{ project.get_absolute_url }}) +{% endfor %} +{% if projects.count > 15 %} +* ... and {{ projects.count|add:"-15" }} more projects. +{% endif %} + +Please use "build.os" on your configuration file to ensure that they continue building successfully and to stop receiving these notifications. +If you want to opt-out from these emails, you can edit your preferences in your account settings, at https://readthedocs.org/accounts/edit/. + +For more information on how to use "build.os", +read our blog post at https://blog.readthedocs.com/use-build-os-config/ + +Get in touch with us via our support ({{ production_uri }}{% url 'support' %}) +and let us know if you are unable to use a configuration file for any reason. +{% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html index 100c6c0b9ae..404ba87d5ee 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -13,7 +13,7 @@
  • Monday, September 25, 2023: Fully remove support for building documentation without configuration file v2.
    • -We have identified that the following projects which you maintain, and were built in the last year, are impacted by this deprecation: +We have identified that the following projects which you maintain, and were built in the last 3 years, are impacted by this deprecation: