Hosting: manual integrations via build contract

dockerfiles/nginx/proxito.conf.template

@@ -88,6 +88,18 @@ server {
         proxy_hide_header Content-Security-Policy;
         set $content_security_policy $upstream_http_content_security_policy;
         add_header Content-Security-Policy $content_security_policy always;
+
+        # This header allows us to decide whether or not inject the script at CloudFlare level
+        # Now, I'm injecting it in all the NGINX responses because `sub_filter` is not allowed inside an `if` statement.
+        set $rtd_hosting_integrations $upstream_http_x_rtd_hosting_integrations;
+        add_header X-RTD-Hosting-Integrations $rtd_hosting_integrations always;
+
+        # Inject our own script dynamically
+        # TODO: decide where is the best place to do this
+        sub_filter '</head>' '<script language="javascript" src="http://devthedocs.org/static/core/js/integrations.js"></script>\n</head>';
+        # sub_filter_types text/html;
+        sub_filter_last_modified on;
+        sub_filter_once on;
     }
 
     # Serve 404 pages here

readthedocs/core/static/core/js/integrations.js

@@ -0,0 +1,44 @@
+// Unique entry point that our servers will inject
+
+// Load Read the Docs data first
+// This data is generated at build time by the doctool
+fetch("/en/manual-integration-docsify/readthedocs-data.html", {method: 'GET'})
+    .then(response => {
+        return response.text();
+    })
+    .then(text => {
+        document.head.insertAdjacentHTML("beforeend", text);
+    })
+    .then(() => {
+        // TODO: use `READTHEDOCS_DATA.features.hosting.version` to decide
+        // what version of these Javascript libraries we need to inject here.
+        // See https://github.com/readthedocs/readthedocs.org/issues/9063#issuecomment-1325483505
+        let link = document.createElement("link");
+        link.setAttribute("rel", "stylesheet");
+        link.setAttribute("type", "text/css");
+        link.setAttribute("href", "/_/static/css/readthedocs-doc-embed.css");
+        document.head.appendChild(link);
+
+        let embed = document.createElement("script");
+        embed.setAttribute("async", "async");
+        embed.setAttribute("src", "/_/static/javascript/readthedocs-doc-embed.js");
+        document.head.appendChild(embed);
+
+        let analytics = document.createElement("script");
+        analytics.setAttribute("async", "async");
+        analytics.setAttribute("src", "/_/static/javascript/readthedocs-analytics.js");
+        document.head.appendChild(analytics);
+
+        let hosting = document.createElement("script");
+        hosting.setAttribute("async", "async");
+        hosting.setAttribute("src", "/_/static/core/js/readthedocs-hosting.js");
+        document.head.appendChild(hosting);
+
+        // TODO: insert search-as-you-type once it becomes a JS library
+        // decoupled from Sphinx.
+        // See https://github.com/readthedocs/readthedocs-sphinx-search/issues/67
+
+        // TODO: find a way to remove the Sphinx default search, since it duplicates the results sometimes.
+        // This removal is currently happening at our Sphinx extension:
+        // https://github.com/readthedocs/readthedocs-sphinx-ext/blob/7cc1e60f7dcdeb7af35e3479509a621d5bac0976/readthedocs_ext/readthedocs.py#L239
+    });

readthedocs/core/static/core/js/readthedocs-hosting.js

@@ -0,0 +1,77 @@
+READTHEDOCS_DATA = JSON.parse(document.getElementById('READTHEDOCS_DATA').textContent);
+
+function injectExternalVersionWarning() {
+    // TODO: make all these banners (injected HTML) templates that users can override with their own.
+    // This way, we allow customization of the look&feel without compromising the logic.
+    let admonition = `
+<div class="admonition warning">
+  <p class="admonition-title">Warning</p>
+  <p>
+    This page
+    <a class="reference external" href="${ window.location.protocol }//${ window.location.hostname }/projects/${ READTHEDOCS_DATA.project }/builds/${ READTHEDOCS_DATA.build.id }/">was created </a>
+    from a pull request
+    (<a class="reference external" href="${ READTHEDOCS_DATA.repository_url }/pull/${ READTHEDOCS_DATA.version }">#${ READTHEDOCS_DATA.version }</a>).
+  </p>
+</div>
+`;
+
+    let main = document.querySelector('[role=main]') || document.querySelector('#main');
+    let node = document.createElement("div");
+    node.innerHTML = admonition;
+    main.insertBefore(node, main.firstChild);
+}
+
+function injectDocDiff() {
+    let docdiff = document.createElement("script");
+    docdiff.setAttribute("async", "async");
+    docdiff.setAttribute("src", "/_/static/core/js/readthedocs-doc-diff.js");
+    document.head.appendChild(docdiff);
+}
+
+function injectOldVersionWarning() {
+    // TODO: compute if the user is reading the latest version or not before showing the warning.
+    let admonition = `
+<div class="admonition warning">
+  <p class="admonition-title">Warning</p>
+  <p>
+    This page documents version
+    <a class="reference" href="${ window.location.protocol }//${ window.location.hostname }/${ READTHEDOCS_DATA.language }/${ READTHEDOCS_DATA.version }/">${ READTHEDOCS_DATA.version }</a>.
+    The latest version is
+    <a class="reference" href="${ window.location.protocol }//${ window.location.hostname }/${ READTHEDOCS_DATA.language }/${ READTHEDOCS_DATA.version }/">${ READTHEDOCS_DATA.version }</a>.
+  </p>
+</div>
+`;
+
+    // Borrowed and adapted from:
+    // https://github.com/readthedocs/readthedocs.org/blob/7ce98a4d4f34a4c1845dc6e3e0e5112af7c39b0c/readthedocs/core/static-src/core/js/doc-embed/version-compare.js#L1
+    let main = document.querySelector('[role=main]') || document.querySelector('#main');
+    let node = document.createElement("div");
+    node.innerHTML = admonition;
+    main.insertBefore(node, main.firstChild);
+}
+
+
+window.addEventListener("load", (event) => {
+    // TODO: use the proxied API here
+    // "repository_url" could be retrived using the API, but there are some CORS issues and design decisions
+    // that it's probably smart to avoid and just rely on a fixed value from the build output :)
+    if (READTHEDOCS_DATA.build.external_version === true) {
+        injectExternalVersionWarning();
+
+        if (READTHEDOCS_DATA.features.docdiff.enabled === true) {
+            if(!window.location.pathname.endsWith("search.html")) {
+                // Avoid injecting doc-diff on search page because it doesn't make sense
+                injectDocDiff();
+            }
+        }
+    }
+    else {
+        // TODO: there some data we need from `/api/v2/footer_html/`,
+        // like `is_highest` and `show_version_warning`.
+        // However, this call is happening inside the `readthedocs-doc-embed.js`.
+        // We could make the call again here for now as demo,
+        // but it would be good to refactor that code
+        // Inject old version warning only for non-external versions
+        injectOldVersionWarning();
+    }
+});

readthedocs/doc_builder/director.py

@@ -2,7 +2,10 @@
 import tarfile
 
 import structlog
+import yaml
 from django.conf import settings
+from django.template.loader import render_to_string
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 from readthedocs.builds.constants import EXTERNAL
@@ -187,6 +190,7 @@ def build(self):
         self.build_epub()
 
         self.run_build_job("post_build")
+        self.generate_readthedocs_data_javascript()
 
         after_build.send(
             sender=self.data.version,
@@ -392,6 +396,7 @@ def run_build_commands(self):
         # Update the `Version.documentation_type` to match the doctype defined
         # by the config file. When using `build.commands` it will be `GENERIC`
         self.data.version.documentation_type = self.data.config.doctype
+        self.generate_readthedocs_data_javascript()
 
     def install_build_tools(self):
         """
@@ -625,3 +630,89 @@ def get_build_env_vars(self):
     def is_type_sphinx(self):
         """Is documentation type Sphinx."""
         return "sphinx" in self.data.config.doctype
+
+    def generate_readthedocs_data_javascript(self):
+        # load YAML from user
+        yaml_path = os.path.join(
+            self.data.project.artifact_path(
+                version=self.data.version.slug, type_="html"
+            ),
+            "readthedocs-build.yaml",
+        )
+
+        try:
+            log.warning(yaml_path)
+            data = yaml.safe_load(open(yaml_path, "r"))
+        except Exception:
+            # TODO: Improve this message
+            raise BuildUserError(
+                "The required 'readthedocs-build.yaml' file was not provided."
+            )
+
+        log.info("readthedocs-build.yaml loaded.", path=yaml_path)
+
+        # TODO: validate the YAML generated by the user
+        # self._validate_readthedocs_data_yaml(data)
+
+        # Populate the data provided by the doctool with data we have from the build context
+        # TODO: this is the new structure that has to be defined
+        # context = {
+        #     "project": {
+        #         "slug": self.data.project.slug,
+        #     },
+        #     "version": {
+        #         "slug": self.data.version.slug,
+        #     },
+        #     "build": {
+        #         "id": self.data.build["pk"],
+        #     },
+        # }
+
+        # NOTE: this is the OLD structure.
+        # This will be changed for a new structure that we have to decide yet.
+        # I'm using the old one for now just to keep compatibility.
+        context = {
+            "ad_free": False,
+            "api_host": "http://devthedocs.org",
+            "build_date": timezone.now(),
+            "builder": "$DOCTOOL_NAME",
+            "canonical_url": self.data.project.canonical_url,
+            "commit": self.data.build["commit"],
+            "docroot": "$DOCTOOL_DOCROOT",
+            "language": self.data.project.language,
+            "page": "$DOCTOOL_PAGE",  # Can we define this dynamically via Javascript?
+            "programming_language": self.data.project.programming_language,
+            "project": self.data.project.slug,
+            "version": self.data.version.slug,
+            "source_suffix": "$DOCTOOL_SOURCE_SUFFIX",
+            "theme": "$DOCTOOL_THEME",
+            # These can be removed
+            "user_analytics_code": None,
+            "global_analytics_code": None,
+            "proxied_api_host": f"/{settings.DOC_PATH_PREFIX}",
+            "subprojects": None,
+            # TODO: remove the following ones, they are just my own tests
+            # NOTE: eventually, some of these settings should be enabled/disabled by the reader
+            "build": {
+                "id": self.data.build_pk,
+                "external_version": self.data.version.type == EXTERNAL,
+            },
+            "repository_url": self.data.project.repo,
+        }
+
+        # Update user's generated data with our own data.
+        data.update(context)
+
+        js_path = os.path.join(
+            self.data.project.artifact_path(
+                version=self.data.version.slug, type_="html"
+            ),
+            "readthedocs-data.html",
+        )
+        content = render_to_string(
+            template_name="doc_builder/readthedocs-data.html",
+            context=dict(data=context),
+        )
+        with open(js_path, "w") as f:
+            f.write(content)
+        log.info("readthedocs-data.html written.", path=js_path)

readthedocs/doc_builder/environments.py

@@ -14,7 +14,7 @@
 from docker.errors import APIError as DockerAPIError
 from docker.errors import DockerException
 from docker.errors import NotFound as DockerNotFoundError
-from requests.exceptions import ConnectionError, ReadTimeout
+from requests.exceptions import ConnectionError, ReadTimeout  # noqa
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from readthedocs.api.v2.client import api as api_v2
@@ -73,7 +73,7 @@ def __init__(
         bin_path=None,
         record_as_success=False,
         demux=False,
-        **kwargs,
+        **kwargs,  # pylint: disable=unused-argument
     ):
         self.command = command
         self.shell = shell
@@ -252,8 +252,8 @@ def save(self):
                 {key: str(value) for key, value in data.items()}
             )
             resource = api_v2.command
-            resp = resource._store['session'].post(
-                resource._store['base_url'] + '/',
+            resp = resource._store["session"].post(  # pylint: disable=protected-access
+                resource._store["base_url"] + "/",  # pylint: disable=protected-access
                 data=encoder,
                 headers={
                     'Content-Type': encoder.content_type,
@@ -301,11 +301,35 @@ def run(self):
 
         self.start_time = datetime.utcnow()
         client = self.build_env.get_client()
+
+        # Create a copy of the environment to update PATH variable
+        environment = self._environment.copy()
+        # Default PATH variable
+        environment[
+            "PATH"
+        ] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+        # Add asdf extra paths
+        environment["PATH"] += (
+            ":/home/{settings.RTD_DOCKER_USER}/.asdf/shims"
+            ":/home/{settings.RTD_DOCKER_USER}/.asdf/bin"
+        )
+
+        if settings.RTD_DOCKER_COMPOSE:
+            environment["PATH"] += ":/root/.asdf/shims:/root/.asdf/bin"
+
+        # Prepend the BIN_PATH if it's defined
+        if self.bin_path:
+            path = environment.get("PATH")
+            bin_path = self._escape_command(self.bin_path)
+            environment["PATH"] = bin_path
+            if path:
+                environment["PATH"] += f":{path}"
+
         try:
             exec_cmd = client.exec_create(
                 container=self.build_env.container_id,
                 cmd=self.get_wrapped_command(),
-                environment=self._environment,
+                environment=environment,
                 user=self.user,
                 workdir=self.cwd,
                 stdout=True,
@@ -357,31 +381,18 @@ def get_wrapped_command(self):
         """
         Wrap command in a shell and optionally escape special bash characters.
 
-        In order to set the current working path inside a docker container, we
-        need to wrap the command in a shell call manually.
-
         Some characters will be interpreted as shell characters without
         escaping, such as: ``pip install requests<0.8``. When passing
         ``escape_command=True`` in the init method this escapes a good majority
         of those characters.
         """
-        prefix = ''
-        if self.bin_path:
-            bin_path = self._escape_command(self.bin_path)
-            prefix += f'PATH={bin_path}:$PATH '
-
         command = (
             ' '.join(
                 self._escape_command(part) if self.escape_command else part
                 for part in self.command
             )
         )
-        return (
-            "/bin/sh -c '{prefix}{cmd}'".format(
-                prefix=prefix,
-                cmd=command,
-            )
-        )
+        return f"/bin/bash -c '{command}'"
 
     def _escape_command(self, cmd):
         r"""Escape the command by prefixing suspicious chars with `\`."""
@@ -524,14 +535,14 @@ class BuildEnvironment(BaseEnvironment):
     """
 
     def __init__(
-            self,
-            project=None,
-            version=None,
-            build=None,
-            config=None,
-            environment=None,
-            record=True,
-            **kwargs,
+        self,
+        project=None,
+        version=None,
+        build=None,
+        config=None,
+        environment=None,
+        record=True,
+        **kwargs,  # pylint: disable=unused-argument
     ):
         super().__init__(project, environment)
         self.version = version
@@ -557,7 +568,7 @@ def run(self, *cmd, **kwargs):
         })
         return super().run(*cmd, **kwargs)
 
-    def run_command_class(self, *cmd, **kwargs):  # pylint: disable=arguments-differ
+    def run_command_class(self, *cmd, **kwargs):  # pylint: disable=signature-differs
         kwargs.update({
             'build_env': self,
         })

readthedocs/doc_builder/templates/doc_builder/readthedocs-data.html

@@ -0,0 +1,6 @@
+{{ data|json_script:"READTHEDOCS_DATA" }}
+
+{% comment %}
+The resulting file, will be available to use as:
+const value = JSON.parse(document.getElementById('readthedocs-data').textContent);
+{% endcomment %}