📝✨ Embed towncrier draft output via Sphinx ext

This change implements injecting the Towncrier draft change notes
RST output into the Sphinx-managed docs site.

It is heavily inspired by the work of @gaborbernat in tox and
reinvented as a native Sphinx extension.
Ref: tox-dev/tox#859

In internally generates an unreleased changelog for the next
unpublished project version (by calling `towncrier` in a subprocess)
and provides it for injection as an RST directive
called "towncrier-draft-entries".

To start using it, first, add it to extensions in `conf.py`:

    extensions.append(towncrier_draft_ext)

Then, optionally, set the global extensions options:

    towncrier_draft_autoversion_mode = 'scm-draft'  # or: 'scm', 'draft', 'sphinx-version', 'sphinx-release'
    towncrier_draft_include_empty = True
    towncrier_draft_working_directory = PROJECT_ROOT_DIR
    towncrier_draft_config_path = 'pyproject.toml'  # relative to cwd

Their meaning is as follows:

    * towncrier_draft_autoversion_mode -- mechanism for the fallback
      version detection. It kicks in if, when using the directive, you
      don't specify the version argument and then will be passed to the
      `towncrier` invocation.

      Possible values are:

          * 'scm-draft' -- default, use setuptools-scm followed by a
            string "[UNRELEASED DRAFT]"
          * 'scm' -- use setuptools-scm
          * 'draft' -- use just a string "[UNRELEASED DRAFT]"
          * 'sphinx-version' -- use value of the "version" var in
            as set in `conf.py`
          * 'sphinx-release' -- use value of the "release" var in
            as set in `conf.py`

    * towncrier_draft_include_empty -- control whether the directive
      injects anything if there's no fragments in the repo. Boolean,
      defaults to `True`.

    * towncrier_draft_working_directory -- if set, this will be the
      current working directory of the `towncrier` invocation.

    * towncrier_draft_config_path -- path of the custom config to use
      for the invocation. Should be relative to the working directory.
      Not yet supported: the corresponding Towncrier CLI option is in
      their master but is not yet released. Don't use it unless you
      install a bleading-edge towncrier copy (or they make a release).

Finally, use the directive in your RST source as follows:

    .. towncrier-draft-entries:: |release| [UNRELEASED DRAFT]

The inline argument of the directive is what is passed to towncrier as
a target version. It is optional and if not set, the fallback from the
global config will be used.

Pro tip: you can use RST substitutions like `|release|` or `|version|`
in order to match the version with what's set in Sphinx and other
release-related configs.

.flake8

@@ -67,6 +67,14 @@ per-file-ignores =
   # Sphinx builds aren't supposed to Python 2:
   docs/conf.py: WPS305
 
+  # WPS305 is unnecessary because this extension is
+  # Python 3 only and so f-strings are allowed,
+  # WPS317/WPS318 enforces weird indents,
+  # WPS326 doesn't allow implicit string concat,
+  # E800 reports a lot of false-positives for legit
+  # tool-related comments:
+  docs/_ext/towncrier_draft_ext.py: E800, WPS305, WPS317, WPS318, WPS326
+
   # The package has imports exposing private things to the public:
   src/pylibsshext/__init__.py: WPS412
 

docs/_ext/towncrier_draft_ext.py

@@ -0,0 +1,178 @@
+# Requires Python 3.6+
+"""Sphinx extension for making titles with dates from Git tags."""
+
+
+import subprocess  # noqa: S404
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from sphinx.application import Sphinx
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import nodes
+
+
+# isort: split
+
+from docutils import statemachine
+from setuptools_scm import get_version
+
+
+PROJECT_ROOT_DIR = Path(__file__).parents[2].resolve()
+TOWNCRIER_DRAFT_CMD = (
+    sys.executable, '-m',  # invoke via runpy under the same interpreter
+    'towncrier',
+    '--draft',  # write to stdout, don't change anything on disk
+)
+
+
+@lru_cache(typed=True)
+def _get_changelog_draft_entries(
+        target_version: str,
+        allow_empty: bool = False,
+        working_dir: str = None,
+        config_path: str = None,
+) -> str:
+    """Retrieve the unreleased changelog entries from Towncrier."""
+    extra_cli_args = (
+        '--version',
+        rf'\ {target_version}',  # version value to be used in the RST title
+        # NOTE: The escaped space sequence (`\ `) is necessary to address
+        # NOTE: a corner case when the towncrier config has something like
+        # NOTE: `v{version}` in the title format **and** the directive target
+        # NOTE: argument starts with a substitution like `|release|`. And so
+        # NOTE: when combined, they'd produce `v|release|` causing RST to not
+        # NOTE: substitute the `|release|` part. But adding an escaped space
+        # NOTE: solves this: that escaped space renders as an empty string and
+        # NOTE: the substitution gets processed properly so the result would
+        # NOTE: be something like `v1.0` as expected.
+    )
+    if config_path is not None:
+        # This isn't actually supported by a released version of Towncrier yet:
+        # https://github.com/twisted/towncrier/pull/157#issuecomment-666549246
+        # https://github.com/twisted/towncrier/issues/269
+        extra_cli_args += '--config', str(config_path)
+    towncrier_output = subprocess.check_output(  # noqa: S603
+        TOWNCRIER_DRAFT_CMD + extra_cli_args,
+        cwd=str(working_dir) if working_dir else None,
+        stderr=subprocess.DEVNULL,
+        text=True,
+    ).strip()
+
+    if not allow_empty and 'No significant changes' in towncrier_output:
+        raise LookupError('There are no unreleased changelog entries so far')
+
+    return towncrier_output
+
+
+@lru_cache(maxsize=1, typed=True)
+def _autodetect_scm_version():
+    """Retrieve an SCM-based project version."""
+    for scm_checkout_path in Path(__file__).parents:  # noqa: WPS500
+        is_scm_checkout = (
+            (scm_checkout_path / '.git').exists() or
+            (scm_checkout_path / '.hg').exists()
+        )
+        if is_scm_checkout:
+            return get_version(root=scm_checkout_path)
+    else:
+        raise LookupError("Failed to locate the project's SCM repo")
+
+
+@lru_cache(maxsize=1, typed=True)
+def _get_draft_version_fallback(strategy: str, sphinx_config: Dict[str, Any]):
+    """Generate a fallback version string for towncrier draft."""
+    known_strategies = {'scm-draft', 'scm', 'draft', 'sphinx-version', 'sphinx-release'}
+    if strategy not in known_strategies:
+        raise ValueError(
+            'Expected "stragegy" to be '
+            f'one of {known_strategies!r} but got {strategy!r}',
+        )
+
+    if 'sphinx' in strategy:
+        return (
+            sphinx_config.release
+            if 'release' in strategy
+            else sphinx_config.version
+        )
+
+    draft_msg = '[UNRELEASED DRAFT]'
+    msg_chunks = ()
+    if 'scm' in strategy:
+        msg_chunks += (_autodetect_scm_version(), )
+    if 'draft' in strategy:
+        msg_chunks += (draft_msg, )
+
+    return ' '.join(msg_chunks)
+
+
+class TowncrierDraftEntriesDirective(SphinxDirective):
+    """Definition of the ``towncrier-draft-entries`` directive."""
+
+    has_content = True  # default: False
+
+    def run(self) -> List[nodes.Node]:
+        """Generate a node tree in place of the directive."""
+        target_version = self.content[:1][0] if self.content[:1] else None
+        if self.content[1:]:  # inner content present
+            raise self.error(
+                f'Error in "{self.name!s}" directive: '
+                'only one argument permitted.',
+            )
+
+        config = self.state.document.settings.env.config  # noqa: WPS219
+        autoversion_mode = config.towncrier_draft_autoversion_mode
+        include_empty = config.towncrier_draft_include_empty
+
+        try:
+            draft_changes = _get_changelog_draft_entries(
+                target_version or
+                _get_draft_version_fallback(autoversion_mode, config),
+                allow_empty=include_empty,
+                working_dir=config.towncrier_draft_working_directory,
+                config_path=config.towncrier_draft_config_path,
+            )
+        except LookupError:
+            return []
+
+        self.state_machine.insert_input(
+            statemachine.string2lines(draft_changes),
+            '[towncrier draft]',
+        )
+        return []
+
+
+def setup(app: Sphinx) -> Dict[str, Union[bool, str]]:
+    """Initialize the extension."""
+    rebuild_trigger = 'html'  # rebuild full html on settings change
+    app.add_config_value(
+        'towncrier_draft_config_path',
+        default=None,
+        rebuild=rebuild_trigger,
+    )
+    app.add_config_value(
+        'towncrier_draft_autoversion_mode',
+        default='scm-draft',
+        rebuild=rebuild_trigger,
+    )
+    app.add_config_value(
+        'towncrier_draft_include_empty',
+        default=True,
+        rebuild=rebuild_trigger,
+    )
+    app.add_config_value(
+        'towncrier_draft_working_directory',
+        default=None,
+        rebuild=rebuild_trigger,
+    )
+    app.add_directive(
+        'towncrier-draft-entries',
+        TowncrierDraftEntriesDirective,
+    )
+
+    return {
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+        'version': get_version(root=PROJECT_ROOT_DIR),
+    }

docs/changelog.rst

@@ -10,6 +10,8 @@ versions with advance notice in the **Deprecations** section of releases.
 
 .. _Semantic Versioning: https://semver.org/
 
+.. towncrier-draft-entries:: |release| [UNRELEASED DRAFT]
+
 .. towncrier release notes start
 
 0.0.1 Unreleased

docs/conf.py

@@ -3,6 +3,7 @@
 # Ref: https://www.sphinx-doc.org/en/master/usage/configuration.html
 """Configuration for the Sphinx documentation generator."""
 
+import sys
 from pathlib import Path
 
 from setuptools_scm import get_version
@@ -13,6 +14,13 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute.
+PROJECT_ROOT_DIR = Path(__file__).parents[1].resolve()
+SPHINX_EXTENSIONS_DIR = (Path(__file__).parent / '_ext').resolve()
+# Make in-tree extension importable in non-tox setups/envs, like RTD.
+# Refs:
+# https://github.com/readthedocs/readthedocs.org/issues/6311
+# https://github.com/readthedocs/readthedocs.org/issues/7182
+sys.path.insert(0, str(SPHINX_EXTENSIONS_DIR))
 
 
 # -- Project information -----------------------------------------------------
@@ -32,7 +40,7 @@
 version = '.'.join(
     get_version(
         local_scheme='no-local-version',
-        root=(Path(__file__).parents[1]).resolve(),
+        root=PROJECT_ROOT_DIR,
     ).split('.')[:3],
 )
 
@@ -63,6 +71,7 @@
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
     # 'sphinxcontrib.apidoc',
+    'towncrier_draft_ext',  # in-tree
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -170,3 +179,10 @@
     r'https://github\.com/ansible/pylibssh/workflows/[^/]+/badge\.svg',
 ]
 linkcheck_workers = 25
+
+# -- Options for towncrier_draft extension -----------------------------------
+
+towncrier_draft_autoversion_mode = 'scm-draft'  # or: 'scm', 'draft', 'sphinx-version', 'sphinx-release'
+towncrier_draft_include_empty = True
+towncrier_draft_working_directory = PROJECT_ROOT_DIR
+# Not yet supported: towncrier_draft_config_path = 'pyproject.toml'  # relative to cwd

docs/requirements.in

@@ -2,3 +2,4 @@ setuptools_scm >= 3.5.0
 Sphinx >= 3
 sphinx-ansible-theme >= 0.3.1
 sphinxcontrib-apidoc >= 0.3.0
+towncrier >= 19.2.0

docs/requirements.txt

@@ -20,6 +20,10 @@ chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
     # via requests
+click==7.1.2 \
+    --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
+    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc \
+    # via towncrier
 docutils==0.16 \
     --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
     --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
@@ -32,10 +36,14 @@ imagesize==1.2.0 \
     --hash=sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1 \
     --hash=sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1 \
     # via sphinx
+incremental==17.5.0 \
+    --hash=sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f \
+    --hash=sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3 \
+    # via towncrier
 jinja2==2.11.2 \
     --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
     --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \
-    # via sphinx
+    # via sphinx, towncrier
 markupsafe==1.1.1 \
     --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
     --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
@@ -151,6 +159,14 @@ sphinxcontrib-serializinghtml==1.1.4 \
     --hash=sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc \
     --hash=sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a \
     # via sphinx
+toml==0.10.1 \
+    --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f \
+    --hash=sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88 \
+    # via towncrier
+towncrier==19.2.0 \
+    --hash=sha256:48251a1ae66d2cf7e6fa5552016386831b3e12bb3b2d08eb70374508c17a8196 \
+    --hash=sha256:de19da8b8cb44f18ea7ed3a3823087d2af8fcf497151bb9fd1e1b092ff56ed8d \
+    # via -r docs/requirements.in
 urllib3==1.25.9 \
     --hash=sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527 \
     --hash=sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115 \

tox.ini

@@ -248,6 +248,8 @@ skip_install = true
 
 [testenv:build-docs]
 basepython = python3
+depends =
+  make-changelog
 deps =
   -r{toxinidir}/docs/requirements.in
   -c{toxinidir}/docs/requirements.txt
@@ -316,6 +318,7 @@ commands =
     {posargs:'[UNRELEASED DRAFT]' --draft}
 deps =
   towncrier
+  -c{toxinidir}/docs/requirements.txt
 isolated_build = true
 skip_install = true