Fix tutorial deployment to GitHub Pages (espressomd#4656)

Fixes espressomd#4651

Description of changes:
- use cloud providers to fetch JavaScript dependencies
- cleanup test cases involved in the tutorial toolchain

maintainer/CI/jupyter_warnings.py

@@ -33,7 +33,7 @@
 sphinx_docs = {}
 
 
-def detect_invalid_urls(nb, sphinx_root='.'):
+def detect_invalid_urls(nb, build_root='.', html_exporter=None):
     '''
     Find all links. Check that links to the Sphinx documentation are valid
     (the target HTML files exist and contain the anchors). These links are
@@ -44,23 +44,29 @@ def detect_invalid_urls(nb, sphinx_root='.'):
     Parameters
     ----------
     nb: :obj:`nbformat.notebooknode.NotebookNode`
-        Jupyter notebook to process
+        Jupyter notebook to process.
+    build_root: :obj:`str`
+        Path to the ESPResSo build directory. The Sphinx files will be
+        searched in :file:`doc/sphinx/html`.
+    html_exporter: :obj:`nbformat.HTMLExporter`
+        Custom NB convert HTML exporter.
 
     Returns
     -------
     :obj:`list`
         List of warnings formatted as strings.
     '''
     # convert notebooks to HTML
-    html_exporter = nbconvert.HTMLExporter()
+    if html_exporter is None:
+        html_exporter = nbconvert.HTMLExporter()
     html_exporter.template_name = 'classic'
     html_string = html_exporter.from_notebook_node(nb)[0]
     # parse HTML
     html_parser = lxml.etree.HTMLParser()
     root = lxml.etree.fromstring(html_string, parser=html_parser)
     # process all links
     espressomd_website_root = 'https://espressomd.github.io/doc/'
-    sphinx_html_root = pathlib.Path(sphinx_root) / 'doc' / 'sphinx' / 'html'
+    sphinx_html_root = pathlib.Path(build_root) / 'doc' / 'sphinx' / 'html'
     broken_links = []
     for link in root.xpath('//a'):
         url = link.attrib.get('href', '')
@@ -76,7 +82,7 @@ def detect_invalid_urls(nb, sphinx_root='.'):
             basename = url.split(espressomd_website_root, 1)[1]
             filepath = sphinx_html_root / basename
             if not filepath.is_file():
-                broken_links.append(f'{url} does not exist')
+                broken_links.append(f'"{url}" does not exist')
                 continue
             # check anchor exists
             if anchor is not None:
@@ -86,19 +92,27 @@ def detect_invalid_urls(nb, sphinx_root='.'):
                 doc = sphinx_docs[filepath]
                 nodes = doc.xpath(f'//*[@id="{anchor}"]')
                 if not nodes:
-                    broken_links.append(f'{url} has no anchor "{anchor}"')
+                    broken_links.append(f'"{url}" has no anchor "{anchor}"')
         elif url.startswith('#'):
             # check anchor exists
             anchor = url[1:]
             nodes = root.xpath(f'//*[@id="{anchor}"]')
             if not nodes:
                 broken_links.append(f'notebook has no anchor "{anchor}"')
+        elif url.startswith('file:///'):
+            broken_links.append(f'"{url}" is an absolute path to a local file')
+    for link in root.xpath('//script'):
+        url = link.attrib.get('src', '')
+        if url.startswith('file:///'):
+            broken_links.append(f'"{url}" is an absolute path to a local file')
     return broken_links
 
 
 if __name__ == '__main__':
     error_code = 0
-    for nb_filepath in sorted(pathlib.Path().glob('doc/tutorials/*/*.ipynb')):
+    nb_filepaths = sorted(pathlib.Path().glob('doc/tutorials/*/*.ipynb'))
+    assert len(nb_filepaths) != 0, 'no Jupyter notebooks could be found!'
+    for nb_filepath in nb_filepaths:
         with open(nb_filepath, encoding='utf-8') as f:
             nb = nbformat.read(f, as_version=4)
         issues = detect_invalid_urls(nb)

requirements.txt

@@ -18,6 +18,7 @@ sphinx>=2.3.0,!=3.0.0
 sphinx-toggleprompt==0.0.5
 sphinxcontrib-bibtex>=2.4.1
 # jupyter dependencies
+nbconvert==6.4.5
 jupyter_contrib_nbextensions==0.5.1
 tqdm>=4.30.0
 # pep8 and its dependencies

testsuite/scripts/CMakeLists.txt

@@ -20,9 +20,9 @@
 set(TEST_FILE_CONFIGURED_IMPORTLIB_WRAPPER
     ${CMAKE_CURRENT_BINARY_DIR}/test_importlib_wrapper.py)
 configure_file(importlib_wrapper.py
-               ${CMAKE_CURRENT_BINARY_DIR}/importlib_wrapper.py)
+               ${CMAKE_CURRENT_BINARY_DIR}/importlib_wrapper.py COPYONLY)
 configure_file(test_importlib_wrapper.py
-               ${TEST_FILE_CONFIGURED_IMPORTLIB_WRAPPER})
+               ${TEST_FILE_CONFIGURED_IMPORTLIB_WRAPPER} COPYONLY)
 
 macro(PYTHON_SCRIPTS_TEST)
   cmake_parse_arguments(TEST "" "FILE;SUFFIX;TYPE" "DEPENDENCIES;LABELS"

testsuite/scripts/importlib_wrapper.py

@@ -237,138 +237,6 @@ def substitute_variable_values(code, strings_as_is=False, keep_original=True,
     return "\n".join(lines)
 
 
-class GetPrngSeedEspressomdSystem(ast.NodeVisitor):
-    """
-    Find all assignments of :class:`espressomd.system.System` in the global
-    namespace. Assignments made in classes or function raise an error. Detect
-    random seed setup of the numpy PRNG.
-    """
-
-    def __init__(self):
-        self.numpy_random_aliases = set()
-        self.es_system_aliases = set()
-        self.variable_system_aliases = set()
-        self.numpy_seeds = []
-        self.abort_message = None
-        self.error_msg_multi_assign = "Cannot parse {} in a multiple assignment (line {})"
-
-    def visit_Import(self, node):
-        # get system aliases
-        for child in node.names:
-            if child.name == "espressomd.system.System":
-                name = (child.asname or child.name)
-                self.es_system_aliases.add(name)
-            elif child.name == "espressomd.system":
-                name = (child.asname or child.name)
-                self.es_system_aliases.add(name + ".System")
-            elif child.name == "espressomd.System":
-                name = (child.asname or child.name)
-                self.es_system_aliases.add(name)
-            elif child.name == "espressomd":
-                name = (child.asname or "espressomd")
-                self.es_system_aliases.add(name + ".system.System")
-                self.es_system_aliases.add(name + ".System")
-            elif child.name == "numpy.random":
-                name = (child.asname or child.name)
-                self.numpy_random_aliases.add(name)
-            elif child.name == "numpy":
-                name = (child.asname or "numpy")
-                self.numpy_random_aliases.add(name + ".random")
-
-    def visit_ImportFrom(self, node):
-        # get system aliases
-        for child in node.names:
-            if node.module == "espressomd" and child.name == "system":
-                name = (child.asname or child.name)
-                self.es_system_aliases.add(name + ".System")
-            elif node.module == "espressomd" and child.name == "System":
-                self.es_system_aliases.add(child.asname or child.name)
-            elif node.module == "espressomd.system" and child.name == "System":
-                self.es_system_aliases.add(child.asname or child.name)
-            elif node.module == "numpy" and child.name == "random":
-                self.numpy_random_aliases.add(child.asname or child.name)
-            elif node.module == "numpy.random":
-                self.numpy_random_aliases.add(child.asname or child.name)
-
-    def is_es_system(self, child):
-        if hasattr(child, "value"):
-            if hasattr(child.value, "value") and hasattr(
-                    child.value.value, "id"):
-                if (child.value.value.id + "." + child.value.attr +
-                        "." + child.attr) in self.es_system_aliases:
-                    return True
-            else:
-                if hasattr(child.value, "id") and (child.value.id + "." +
-                                                   child.attr) in self.es_system_aliases:
-                    return True
-        elif isinstance(child, ast.Call):
-            if hasattr(child, "id") and child.func.id in self.es_system_aliases:
-                return True
-            elif hasattr(child, "func") and hasattr(child.func, "value") and (
-                    hasattr(child.func.value, "value") and
-                    (child.func.value.value.id + "." +
-                     child.func.value.attr + "." + child.func.attr)
-                    or (child.func.value.id + "." + child.func.attr)) in self.es_system_aliases:
-                return True
-            elif hasattr(child, "func") and hasattr(child.func, "id") and child.func.id in self.es_system_aliases:
-                return True
-        elif hasattr(child, "id") and child.id in self.es_system_aliases:
-            return True
-        return False
-
-    def detect_es_system_instances(self, node):
-        varname = None
-        for target in node.targets:
-            if isinstance(target, ast.Name):
-                if hasattr(target, "id") and hasattr(node.value, "func") and \
-                        self.is_es_system(node.value.func):
-                    varname = target.id
-            elif isinstance(target, ast.Tuple):
-                value = node.value
-                if (isinstance(value, ast.Tuple) or isinstance(value, ast.List)) \
-                        and any(map(self.is_es_system, node.value.elts)):
-                    raise AssertionError(self.error_msg_multi_assign.format(
-                        "espressomd.System", node.lineno))
-        if varname is not None:
-            assert len(node.targets) == 1, self.error_msg_multi_assign.format(
-                "espressomd.System", node.lineno)
-            assert self.abort_message is None, \
-                "Cannot process espressomd.System assignments in " + self.abort_message
-            self.variable_system_aliases.add(varname)
-
-    def detect_np_random_expr_seed(self, node):
-        if hasattr(node.value, "func") and hasattr(node.value.func, "value") \
-                and (hasattr(node.value.func.value, "id") and node.value.func.value.id in self.numpy_random_aliases
-                     or hasattr(node.value.func.value, "value")
-                     and hasattr(node.value.func.value.value, "id")
-                     and hasattr(node.value.func.value, "attr")
-                     and (node.value.func.value.value.id + "." +
-                          node.value.func.value.attr) in self.numpy_random_aliases
-                     ) and node.value.func.attr == "seed":
-            self.numpy_seeds.append(node.lineno)
-
-    def visit_Assign(self, node):
-        self.detect_es_system_instances(node)
-
-    def visit_Expr(self, node):
-        self.detect_np_random_expr_seed(node)
-
-    def visit_ClassDef(self, node):
-        self.abort_message = "class definitions"
-        ast.NodeVisitor.generic_visit(self, node)
-        self.abort_message = None
-
-    def visit_FunctionDef(self, node):
-        self.abort_message = "function definitions"
-        ast.NodeVisitor.generic_visit(self, node)
-        self.abort_message = None
-
-    def visit_AsyncFunctionDef(self, node):
-        self.abort_message = "function definitions"
-        ast.NodeVisitor.generic_visit(self, node)
-        self.abort_message = None
-
-
 def delimit_statements(code):
     """
     For every Python statement, map the line number where it starts to the

testsuite/scripts/test_importlib_wrapper.py

@@ -237,68 +237,6 @@ def test_matplotlib_pyplot_visitor(self):
         self.assertEqual(v.matplotlib_backend_linenos, [17, 18])
         self.assertEqual(v.ipython_magic_linenos, [19])
 
-    def test_prng_seed_espressomd_system_visitor(self):
-        import_stmt = [
-            'sys0 = espressomd.System()   # nothing: espressomd not imported',
-            'import espressomd as es1',
-            'import espressomd.system as es2',
-            'import espressomd.System as s1, espressomd.system.System as s2',
-            'from espressomd import System as s3, electrostatics',
-            'from espressomd.system import System as s4',
-            'from espressomd import system as es5',
-            'sys1 = es1.System()',
-            'sys2 = es1.system.System()',
-            'sys3 = es2.System()',
-            'sys4 = s1()',
-            'sys5 = s2()',
-            'sys6 = s3()',
-            'sys7 = s4()',
-            'sys8 = es5.System()',
-            'import numpy as np',
-            'import numpy.random as npr1',
-            'from numpy import random as npr2',
-            'np.random.seed(1)',
-            'npr1.seed(1)',
-            'npr2.seed(1)',
-        ]
-        tree = ast.parse('\n'.join(import_stmt))
-        v = iw.GetPrngSeedEspressomdSystem()
-        v.visit(tree)
-        # find all aliases for espressomd.system.System
-        expected_es_sys_aliases = {'es1.System', 'es1.system.System',
-                                   'es2.System', 's1', 's2', 's3', 's4',
-                                   'es5.System'}
-        self.assertEqual(v.es_system_aliases, expected_es_sys_aliases)
-        # find all variables of type espressomd.system.System
-        expected_es_sys_objs = set(f'sys{i}' for i in range(1, 9))
-        self.assertEqual(v.variable_system_aliases, expected_es_sys_objs)
-        # find all seeds setup
-        self.assertEqual(v.numpy_seeds, [19, 20, 21])
-        # test exceptions
-        str_es_sys_list = [
-            'import espressomd.System',
-            'import espressomd.system.System',
-            'from espressomd import System',
-            'from espressomd.system import System',
-        ]
-        exception_stmt = [
-            's, var = System(), 5',
-            'class A:\n\ts = System()',
-            'def A():\n\ts = System()',
-        ]
-        for str_es_sys in str_es_sys_list:
-            for str_stmt in exception_stmt:
-                for alias in ['', ' as EsSystem']:
-                    str_import = str_es_sys + alias + '\n'
-                    alias = str_import.split()[-1]
-                    code = str_import + str_stmt.replace('System', alias)
-                    v = iw.GetPrngSeedEspressomdSystem()
-                    tree = ast.parse(code)
-                    err_msg = v.__class__.__name__ + \
-                        ' should fail on ' + repr(code)
-                    with self.assertRaises(AssertionError, msg=err_msg):
-                        v.visit(tree)
-
     def test_delimit_statements(self):
         lines = [
             'a = 1 # NEWLINE becomes NL after a comment',

testsuite/scripts/utils/test_maintainer_CI_jupyter_warnings.py

@@ -1,3 +1,4 @@
+#
 # Copyright (C) 2020-2022 The ESPResSo project
 #
 # This file is part of ESPResSo.
@@ -14,9 +15,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
 
 import sys
 import nbformat
+import nbconvert
 import importlib
 import unittest as ut
 
@@ -36,18 +39,22 @@ class Test(ut.TestCase):
 invalid: https://espressomd.github.io/doc/index.html#unknown_anchor
 invalid: https://espressomd.github.io/doc/unknown_file.html
 invalid: [footnote 1](#unknown-footnote-1)
+invalid: [resource](file:///home/espresso/image.png)
 '''
 
     def test_detect_invalid_urls(self):
+        html_exporter = nbconvert.HTMLExporter()
         nb = nbformat.v4.new_notebook()
         cell_md = nbformat.v4.new_markdown_cell(source=self.cell_md_src)
         nb['cells'].append(cell_md)
         ref_issues = [
-            'https://espressomd.github.io/doc/index.html has no anchor "unknown_anchor"',
-            'https://espressomd.github.io/doc/unknown_file.html does not exist',
-            'notebook has no anchor "unknown-footnote-1"'
+            '"https://espressomd.github.io/doc/index.html" has no anchor "unknown_anchor"',
+            '"https://espressomd.github.io/doc/unknown_file.html" does not exist',
+            'notebook has no anchor "unknown-footnote-1"',
+            '"file:///home/espresso/image.png" is an absolute path to a local file',
         ]
-        issues = module.detect_invalid_urls(nb, '@CMAKE_BINARY_DIR@')
+        issues = module.detect_invalid_urls(
+            nb, build_root='@CMAKE_BINARY_DIR@', html_exporter=html_exporter)
         self.assertEqual(issues, ref_issues)