adding basic tests w/ pytest regressions

.github/workflows/tests.yml

@@ -62,6 +62,10 @@ jobs:
         export PATH="$HOME/miniconda/bin:$PATH"
         sphinx-build -b html docs/ docs/_build/html -W --keep-going
 
+    # Run tests
+    - name: Run the tests
+      run: pytest
+
   # Run local Lighthouse audit against built site
   audit:
 

docs/contributing.rst

@@ -114,6 +114,7 @@ Steps to develop the theme
 2. Install theme dependencies
 3. Run development server
 4. Build production assets
+5. Install the testing infrastructure
 
 .. Attention::
 
@@ -189,6 +190,28 @@ run:
     yarn build:production
 
 
+Install the test infrastructure
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This theme uses ``pytest`` for its testing, with a lightweight fixture defined
+in the ``test_build.py`` script that makes it easy to run a Sphinx build using
+this theme and inspect the results.
+
+In addition, we use `pytest-regressions <https://pytest-regressions.readthedocs.io/en/latest/>`_
+to ensure that the HTML generated by the theme is what we'd expect. This module
+provides a ``file_regression`` fixture that will check the contents of an object
+against a reference file on disk. If the structure of the two differs, then the
+test will fail. If we *expect* the structure to differ, then delete the file on
+disk and run the test. A new file will be created, and subsequent tests will pass.
+
+Install the testing dependencies with:
+
+.. code-block:: bash
+
+   pip install pytest pytest-regressions
+
+Then run the tests by calling ``pytest`` from the repository root.
+
 Changing fonts
 --------------
 

docs/requirements.txt

@@ -5,3 +5,6 @@ pandas
 jupyter_sphinx
 plotly
 numpy
+pytest
+pytest-regressions
+beautifulsoup4

tests/sites/base/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

tests/sites/base/conf.py

@@ -0,0 +1,20 @@
+# -- Project information -----------------------------------------------------
+
+project = "PyData Tests"
+copyright = "2020, Pydata community"
+author = "Pydata community"
+
+master_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+html_theme = "pydata_sphinx_theme"
+html_copy_source = True
+html_sourcelink_suffix = ""
+
+# Base options, we can add other key/vals later
+html_theme_options = {}

tests/sites/base/index.rst

@@ -0,0 +1,10 @@
+Index ``with code`` in title
+============================
+
+.. toctree::
+   :caption: My caption
+   :numbered:
+
+   page1
+   page2
+   section1/index

tests/sites/base/page1.rst

@@ -0,0 +1,2 @@
+Page 1
+======

tests/sites/base/page2.rst

@@ -0,0 +1,2 @@
+Page 2
+======

tests/sites/base/section1/index.rst

@@ -0,0 +1,6 @@
+Section 1 index
+===============
+.. toctree::
+
+   page1
+   https://google.com

tests/sites/base/section1/page1.rst

@@ -0,0 +1,2 @@
+Section 1 page1
+===============

tests/test_build.py

@@ -0,0 +1,70 @@
+from bs4 import BeautifulSoup
+from pathlib import Path
+from subprocess import run
+from shutil import copytree, rmtree
+import pytest
+
+
+path_tests = Path(__file__).parent.resolve()
+path_base = path_tests.joinpath("sites", "base")
+
+
+@pytest.fixture(scope="session")
+def sphinx_build(tmpdir_factory):
+    class SphinxBuild:
+        path_tmp = Path(tmpdir_factory.mktemp("build"))
+        path_docs = path_tmp.joinpath("testdocs")
+        path_build = path_docs.joinpath("_build")
+        path_html = path_build.joinpath("html")
+        path_pg_index = path_html.joinpath("index.html")
+        cmd_base = ["sphinx-build", ".", "_build/html", "-a", "-W"]
+
+        def copy(self, path=None):
+            """Copy the specified book to our tests folder for building."""
+            if path is None:
+                path = path_base
+            if not self.path_docs.exists():
+                copytree(path, self.path_docs)
+
+        def build(self, cmd=None):
+            """Build the test book"""
+            cmd = [] if cmd is None else cmd
+            run(self.cmd_base + cmd, cwd=self.path_docs, check=True)
+
+        def get(self, pagename):
+            path_page = self.path_html.joinpath(pagename)
+            if not path_page.exists():
+                raise ValueError(f"{path_page} does not exist")
+            return BeautifulSoup(path_page.read_text(), "html.parser")
+
+        def clean(self):
+            """Clean the _build folder so files don't clash with new tests."""
+            rmtree(self.path_build)
+
+    return SphinxBuild()
+
+
+def test_build_book(file_regression, sphinx_build):
+    """Test building the base book template and config."""
+    sphinx_build.copy()
+
+    # Basic build with defaults
+    sphinx_build.build()
+    index_html = sphinx_build.get("index.html")
+    subpage_html = sphinx_build.get("section1/index.html")
+
+    # Navbar structure
+    navbar = index_html.select("div#navbar-menu")[0]
+    file_regression.check(navbar.prettify(), basename="navbar_ix", extension=".html")
+
+    # Sidebar structure
+    sidebar = index_html.select(".bd-sidebar")[0]
+    file_regression.check(sidebar.prettify(), basename="sidebar_ix", extension=".html")
+
+    # Sidebar subpage
+    sidebar = subpage_html.select(".bd-sidebar")[0]
+    file_regression.check(
+        sidebar.prettify(), basename="sidebar_subpage", extension=".html"
+    )
+
+    sphinx_build.clean()

tests/test_build/navbar_ix.html

@@ -0,0 +1,21 @@
+<div class="col-lg-9 collapse navbar-collapse" id="navbar-menu">
+ <ul class="navbar-nav mr-auto" id="navbar-main-elements">
+  <li class="nav-item">
+   <a class="nav-link" href="page1.html">
+    Page 1
+   </a>
+  </li>
+  <li class="nav-item">
+   <a class="nav-link" href="page2.html">
+    Page 2
+   </a>
+  </li>
+  <li class="nav-item">
+   <a class="nav-link" href="section1/index.html">
+    Section 1 index
+   </a>
+  </li>
+ </ul>
+ <ul class="navbar-nav">
+ </ul>
+</div>

tests/test_build/sidebar_ix.html

@@ -0,0 +1,13 @@
+<div class="col-12 col-md-3 bd-sidebar">
+ <form action="search.html" class="bd-search d-flex align-items-center" method="get">
+  <i class="icon fas fa-search">
+  </i>
+  <input aria-label="Search the docs ..." autocomplete="off" class="form-control" id="search-input" name="q" placeholder="Search the docs ..." type="search"/>
+ </form>
+ <nav aria-label="Main navigation" class="bd-links" id="bd-docs-nav">
+  <div class="bd-toc-item active">
+   <ul class="nav bd-sidenav">
+   </ul>
+  </div>
+ </nav>
+</div>

tests/test_build/sidebar_subpage.html

@@ -0,0 +1,23 @@
+<div class="col-12 col-md-3 bd-sidebar">
+ <form action="../search.html" class="bd-search d-flex align-items-center" method="get">
+  <i class="icon fas fa-search">
+  </i>
+  <input aria-label="Search the docs ..." autocomplete="off" class="form-control" id="search-input" name="q" placeholder="Search the docs ..." type="search"/>
+ </form>
+ <nav aria-label="Main navigation" class="bd-links" id="bd-docs-nav">
+  <div class="bd-toc-item active">
+   <ul class="nav bd-sidenav">
+    <li class="">
+     <a href="page1.html">
+      Section 1 page1
+     </a>
+    </li>
+    <li class="">
+     <a href="https://google.com">
+      https://google.com
+     </a>
+    </li>
+   </ul>
+  </div>
+ </nav>
+</div>