Vendor towncrier_draft_ext Sphinx extension

This change implements injecting the Towncrier draft change notes
RST output into the Sphinx-managed docs site.

It is heavily inspired by the work of @gaborbernat in tox and
reinvented as a native Sphinx extension.
Refs:
* tox-dev#859
* ansible/pylibssh@f5f9ef1

In internally generates an unreleased changelog for the next
unpublished project version (by calling `towncrier` in a subprocess)
and provides it for injection as an RST directive
called "towncrier-draft-entries".

To start using it, first, add it to extensions in `conf.py`:

    extensions.append('towncrier_draft_ext')

Then, optionally, set the global extensions options:

    towncrier_draft_autoversion_mode = 'scm-draft'  # or: 'scm', 'draft', 'sphinx-version', 'sphinx-release'
    towncrier_draft_include_empty = True
    towncrier_draft_working_directory = PROJECT_ROOT_DIR
    towncrier_draft_config_path = 'pyproject.toml'  # relative to cwd

Their meaning is as follows:

    * towncrier_draft_autoversion_mode -- mechanism for the fallback
      version detection. It kicks in if, when using the directive, you
      don't specify the version argument and then will be passed to the
      `towncrier` invocation.

      Possible values are:

          * 'scm-draft' -- default, use setuptools-scm followed by a
            string "[UNRELEASED DRAFT]"
          * 'scm' -- use setuptools-scm
          * 'draft' -- use just a string "[UNRELEASED DRAFT]"
          * 'sphinx-version' -- use value of the "version" var in
            as set in `conf.py`
          * 'sphinx-release' -- use value of the "release" var in
            as set in `conf.py`

    * towncrier_draft_include_empty -- control whether the directive
      injects anything if there's no fragments in the repo. Boolean,
      defaults to `True`.

    * towncrier_draft_working_directory -- if set, this will be the
      current working directory of the `towncrier` invocation.

    * towncrier_draft_config_path -- path of the custom config to use
      for the invocation. Should be relative to the working directory.
      Not yet supported: the corresponding Towncrier CLI option is in
      their master but is not yet released. Don't use it unless you
      install a bleading-edge towncrier copy (or they make a release).

Finally, use the directive in your RST source as follows:

    .. towncrier-draft-entries:: |release| [UNRELEASED DRAFT]

The inline argument of the directive is what is passed to towncrier as
a target version. It is optional and if not set, the fallback from the
global config will be used.

Pro tip: you can use RST substitutions like `|release|` or `|version|`
in order to match the version with what's set in Sphinx and other
release-related configs.

src:
https://github.com/ansible/pylibssh/blob/8b21ad7/docs/_ext/towncrier_draft_ext.py

Resolves tox-dev#1639

.gitignore

@@ -21,9 +21,6 @@ __pycache__
 # tools
 /.*_cache
 
-# documentation
-/docs/_draft.rst
-
 # release
 credentials.json
 

docs/_ext/towncrier_draft_ext.py

@@ -0,0 +1,180 @@
+# fmt: no
+# Requires Python 3.6+
+"""Sphinx extension for making titles with dates from Git tags."""
+
+
+import subprocess  # noqa: S404
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from sphinx.application import Sphinx
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import nodes
+
+
+# isort: split
+
+from docutils import statemachine
+from setuptools_scm import get_version
+
+
+PROJECT_ROOT_DIR = Path(__file__).parents[2].resolve()
+TOWNCRIER_DRAFT_CMD = (
+    sys.executable, '-m',  # invoke via runpy under the same interpreter
+    'towncrier',
+    '--draft',  # write to stdout, don't change anything on disk
+)
+
+
+@lru_cache(typed=True)
+def _get_changelog_draft_entries(
+        target_version: str,
+        allow_empty: bool = False,
+        working_dir: str = None,
+        config_path: str = None,
+) -> str:
+    """Retrieve the unreleased changelog entries from Towncrier."""
+    extra_cli_args = (
+        '--version',
+        rf'\ {target_version}',  # version value to be used in the RST title
+        # NOTE: The escaped space sequence (`\ `) is necessary to address
+        # NOTE: a corner case when the towncrier config has something like
+        # NOTE: `v{version}` in the title format **and** the directive target
+        # NOTE: argument starts with a substitution like `|release|`. And so
+        # NOTE: when combined, they'd produce `v|release|` causing RST to not
+        # NOTE: substitute the `|release|` part. But adding an escaped space
+        # NOTE: solves this: that escaped space renders as an empty string and
+        # NOTE: the substitution gets processed properly so the result would
+        # NOTE: be something like `v1.0` as expected.
+    )
+    if config_path is not None:
+        # This isn't actually supported by a released version of Towncrier yet:
+        # https://github.com/twisted/towncrier/pull/157#issuecomment-666549246
+        # https://github.com/twisted/towncrier/issues/269
+        extra_cli_args += '--config', str(config_path)
+    towncrier_output = subprocess.check_output(  # noqa: S603
+        TOWNCRIER_DRAFT_CMD + extra_cli_args,
+        cwd=str(working_dir) if working_dir else None,
+        text=True,
+    ).strip()
+
+    if not allow_empty and 'No significant changes' in towncrier_output:
+        raise LookupError('There are no unreleased changelog entries so far')
+
+    return towncrier_output
+
+
+@lru_cache(maxsize=1, typed=True)
+def _autodetect_scm_version():
+    """Retrieve an SCM-based project version."""
+    for scm_checkout_path in Path(__file__).parents:  # noqa: WPS500
+        is_scm_checkout = (
+            (scm_checkout_path / '.git').exists() or
+            (scm_checkout_path / '.hg').exists()
+        )
+        if is_scm_checkout:
+            return get_version(root=scm_checkout_path)
+    else:
+        raise LookupError("Failed to locate the project's SCM repo")
+
+
+@lru_cache(maxsize=1, typed=True)
+def _get_draft_version_fallback(strategy: str, sphinx_config: Dict[str, Any]):
+    """Generate a fallback version string for towncrier draft."""
+    known_strategies = {'scm-draft', 'scm', 'draft', 'sphinx-version', 'sphinx-release'}
+    if strategy not in known_strategies:
+        raise ValueError(
+            'Expected "stragegy" to be '
+            f'one of {known_strategies!r} but got {strategy!r}',
+        )
+
+    if 'sphinx' in strategy:
+        return (
+            sphinx_config.release
+            if 'release' in strategy
+            else sphinx_config.version
+        )
+
+    draft_msg = '[UNRELEASED DRAFT]'
+    msg_chunks = ()
+    if 'scm' in strategy:
+        msg_chunks += (_autodetect_scm_version(), )
+    if 'draft' in strategy:
+        msg_chunks += (draft_msg, )
+
+    return ' '.join(msg_chunks)
+
+
+class TowncrierDraftEntriesDirective(SphinxDirective):
+    """Definition of the ``towncrier-draft-entries`` directive."""
+
+    has_content = True  # default: False
+
+    def run(self) -> List[nodes.Node]:
+        """Generate a node tree in place of the directive."""
+        target_version = self.content[:1][0] if self.content[:1] else None
+        if self.content[1:]:  # inner content present
+            raise self.error(
+                f'Error in "{self.name!s}" directive: '
+                'only one argument permitted.',
+            )
+
+        config = self.state.document.settings.env.config  # noqa: WPS219
+        autoversion_mode = config.towncrier_draft_autoversion_mode
+        include_empty = config.towncrier_draft_include_empty
+
+        try:
+            draft_changes = _get_changelog_draft_entries(
+                target_version or
+                _get_draft_version_fallback(autoversion_mode, config),
+                allow_empty=include_empty,
+                working_dir=config.towncrier_draft_working_directory,
+                config_path=config.towncrier_draft_config_path,
+            )
+        except subprocess.CalledProcessError as proc_exc:
+            raise self.error(proc_exc)
+        except LookupError:
+            return []
+
+        self.state_machine.insert_input(
+            statemachine.string2lines(draft_changes),
+            '[towncrier draft]',
+        )
+        return []
+
+
+def setup(app: Sphinx) -> Dict[str, Union[bool, str]]:
+    """Initialize the extension."""
+    rebuild_trigger = 'html'  # rebuild full html on settings change
+    app.add_config_value(
+        'towncrier_draft_config_path',
+        default=None,
+        rebuild=rebuild_trigger,
+    )
+    app.add_config_value(
+        'towncrier_draft_autoversion_mode',
+        default='scm-draft',
+        rebuild=rebuild_trigger,
+    )
+    app.add_config_value(
+        'towncrier_draft_include_empty',
+        default=True,
+        rebuild=rebuild_trigger,
+    )
+    app.add_config_value(
+        'towncrier_draft_working_directory',
+        default=None,
+        rebuild=rebuild_trigger,
+    )
+    app.add_directive(
+        'towncrier-draft-entries',
+        TowncrierDraftEntriesDirective,
+    )
+
+    return {
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+        'version': get_version(root=PROJECT_ROOT_DIR),
+    }

docs/changelog.rst

@@ -7,7 +7,7 @@ Versions follow `Semantic Versioning <https://semver.org/>`_ (``<major>.<minor>.
 Backward incompatible (breaking) changes will only be introduced in major versions
 with advance notice in the **Deprecations** section of releases.
 
-.. include:: _draft.rst
+.. towncrier-draft-entries:: DRAFT
 
 .. towncrier release notes start
 

docs/conf.py

@@ -1,6 +1,4 @@
 import os
-import re
-import subprocess
 import sys
 from datetime import date
 from pathlib import Path
@@ -16,36 +14,15 @@
     "sphinx.ext.intersphinx",
     "sphinx.ext.viewcode",
     "sphinxcontrib.autoprogram",
+    "towncrier_draft_ext",  # in-tree
 ]
-ROOT_SRC_TREE_DIR = Path(__file__).parents[1]
-
-
-def generate_draft_news():
-    home = "https://github.com"
-    issue = "{}/issue".format(home)
-    fragments_path = ROOT_SRC_TREE_DIR / "docs" / "changelog"
-    for pattern, replacement in (
-        (r"[^`]@([^,\s]+)", r"`@\1 <{}/\1>`_".format(home)),
-        (r"[^`]#([\d]+)", r"`#pr\1 <{}/\1>`_".format(issue)),
-    ):
-        for path in fragments_path.glob("*.rst"):
-            path.write_text(re.sub(pattern, replacement, path.read_text()))
-    env = os.environ.copy()
-    env["PATH"] += os.pathsep.join(
-        [os.path.dirname(sys.executable)] + env["PATH"].split(os.pathsep),
-    )
-    changelog = subprocess.check_output(
-        ["towncrier", "--draft", "--version", "DRAFT"], cwd=str(ROOT_SRC_TREE_DIR), env=env,
-    ).decode("utf-8")
-    if "No significant changes" in changelog:
-        content = ""
-    else:
-        note = "*Changes in master, but not released yet are under the draft section*."
-        content = "{}\n\n{}".format(note, changelog)
-    (ROOT_SRC_TREE_DIR / "docs" / "_draft.rst").write_text(content)
-
-
-generate_draft_news()
+ROOT_SRC_TREE_DIR = Path(__file__).parents[1].resolve()
+SPHINX_EXTENSIONS_DIR = (Path(__file__).parent / "_ext").resolve()
+# Make in-tree extension importable in non-tox setups/envs, like RTD.
+# Refs:
+# https://github.com/readthedocs/readthedocs.org/issues/6311
+# https://github.com/readthedocs/readthedocs.org/issues/7182
+sys.path.insert(0, str(SPHINX_EXTENSIONS_DIR))
 
 project = u"tox"
 _full_version = tox.__version__
@@ -127,3 +104,12 @@ def parse_node(env, text, node):
     "pull": ("https://github.com/tox-dev/tox/pull/%s", "p"),
     "user": ("https://github.com/%s", "@"),
 }
+
+# -- Options for towncrier_draft extension -----------------------------------
+
+towncrier_draft_autoversion_mode = (
+    "draft"  # or: 'scm-draft' (default, 'scm', 'sphinx-version', 'sphinx-release'
+)
+towncrier_draft_include_empty = False
+towncrier_draft_working_directory = ROOT_SRC_TREE_DIR
+# Not yet supported: towncrier_draft_config_path = 'pyproject.toml'  # relative to cwd

setup.cfg

@@ -55,6 +55,7 @@ console_scripts =
 [options.extras_require]
 docs =
     pygments-github-lexers>=0.0.5
+    setuptools-scm
     sphinx>=2.0.0
     sphinxcontrib-autoprogram>=0.1.5
     towncrier>=18.5.0

tox.ini

@@ -155,7 +155,7 @@ include_trailing_comma = True
 force_grid_wrap = 0
 line_length = 99
 known_first_party = tox,tests
-known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pluggy,py,pytest,setuptools,six,sphinx,toml
+known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pluggy,py,pytest,setuptools,setuptools_scm,six,sphinx,toml
 
 [testenv:release]
 description = do a release, required posarg of the version number