Merge branch 'master' into v/isolatedenv

Project.toml

@@ -2,7 +2,7 @@ name = "DocumentationGenerator"
 uuid = "8f0d3306-d70b-5309-b898-24bb6ab47850"
 authors = ["Sebastian Pfitzner <[email protected]>", "Simon Danisch <[email protected]>", "Venkatesh Dayananda <[email protected]>"]
 repo = "https://github.com/JuliaDocs/DocumentationGenerator.jl.git"
-version = "0.7.5"
+version = "0.7.6"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
@@ -22,7 +22,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
-AbstractTrees = "0.2, 0.3"
+AbstractTrees = "0.4"
 Documenter = "0.24, 0.25, 0.26, 0.27"
 GitHub = "5.1, 5.2, 5.3, 5.4"
 GitForge = "0.4"

src/DocumentationGenerator.jl

@@ -93,7 +93,7 @@ function build_documentation(
         false
     end
 
-    regpath = get_registry(basepath, sync = sync_registry)
+    regpath = get_registry(basepath; sync = sync_registry)
     process_queue = []
 
     # make sure registry is updated *before* we start multiple processes that might try that at the same time

src/builders.jl

@@ -475,43 +475,69 @@ function postprocess_html_readme(html; src_prefix="", href_prefix="")
         href_prefix = string(href_prefix, '/')
     end
 
-    header_refs = Set([])
-    for el in PreOrderDFS(doc)
+    # We'll be transforming some of the elements in the tree. For headings and
+    # code highlighting, we actually have to mutate the tree. So in that case
+    # we must perform the mutation after iterating with PreOrderDFS(), since the
+    # iteration is stateful and will get confused if the tree's structure gets
+    # changed underneath it.
+    heading_elements, highlight_elements = HTMLElement[], HTMLElement[]
+    for el in AbstractTrees.PreOrderDFS(doc)
         if el isa HTMLElement
             if Gumbo.tag(el) in [:h1, :h2, :h3, :h4, :h5]
-                heading = replace(text(el), r"\s" => "-")
-                heading = replace(heading, r"[^\w-]" => "")
-
-                while heading in header_refs
-                    new_heading = replace(heading, r"_\d+$" => s -> begin
-                        string('_', parse(Int, s[2:end]) + 1)
-                    end)
-                    if new_heading == heading
-                        heading = string(heading, "_1")
-                    else
-                        heading = new_heading
-                    end
+                push!(heading_elements, el)
+            elseif Gumbo.tag(el) == :code
+                if Gumbo.tag(el.parent) == :pre && length(el.children) == 1 && typeof(el.children[1]) == HTMLText
+                    push!(highlight_elements, el)
                 end
+            end
+        end
+    end
 
-                orig_content = deepcopy(el.children)
-                a_el = HTMLElement{:a}(orig_content, el, Dict(
-                    "href" => string('#', heading),
-                    "class" => "docs-heading-anchor"
-                ))
+    # Mutate <h*> tags
+    header_refs = Set([]) # this is to avoid duplicate IDs
+    for el in heading_elements
+        # For heading elements we essentially just push a new <a>
+        # tag between the <h*> tag and the child nodes.
+        heading = replace(text(el), r"\s" => "-")
+        heading = replace(heading, r"[^\w-]" => "")
+
+        while heading in header_refs
+            new_heading = replace(heading, r"_\d+$" => s -> begin
+                string('_', parse(Int, s[2:end]) + 1)
+            end)
+            if new_heading == heading
+                heading = string(heading, "_1")
+            else
+                heading = new_heading
+            end
+        end
 
-                for c in orig_content
-                    c.parent = a_el
-                end
+        orig_content = deepcopy(el.children)
+        a_el = HTMLElement{:a}(orig_content, el, Dict(
+            "href" => string('#', heading),
+            "class" => "docs-heading-anchor"
+        ))
 
-                el.children = [a_el]
+        for c in orig_content
+            c.parent = a_el
+        end
 
-                setattr!(el, "id", heading)
-                push!(header_refs, heading)
-            elseif Gumbo.tag(el) == :code
-                if Gumbo.tag(el.parent) == :pre && length(el.children) == 1 && typeof(el.children[1]) == HTMLText
-                    highlight_syntax_html(el)
-                end
-            elseif hasattr(el, "src")
+        el.children = [a_el]
+
+        setattr!(el, "id", heading)
+        push!(header_refs, heading)
+    end
+
+    # Mutate the highlighting-related elements
+    for el in highlight_elements
+        highlight_syntax_html(el)
+    end
+
+    # Update the 'src' and 'href' attributes of all the tags that have them,
+    # applying a URL prefix.
+    for el in AbstractTrees.PreOrderDFS(doc)
+        if el isa HTMLElement
+            if hasattr(el, "src")
                 replace_url(el, "src", src_prefix, fix_github = true)
             elseif hasattr(el, "href")
                 replace_url(el, "href", href_prefix)

src/registry.jl

@@ -1,39 +1,63 @@
-const DOCS_REGISTRY = "https://github.com/JuliaDocs/DocumentationGeneratorRegistry.git"
+# The registry cloning URLs are tried in order. Currently (on the 0.7 branch),
+# we still try to clone the registry first from JuliaDocs, remaining consistent
+# with the earlier releases. But we will fall back to the JuliaComputing URL
+# if it stops working.
+const DOCS_REGISTRIES = [
+    "https://github.com/JuliaDocs/DocumentationGeneratorRegistry.git",
+    "https://github.com/JuliaComputing/DocumentationGeneratorRegistry.git",
+]
 
 """
-    get_registry(basepath; registry=DOCS_REGISTRY, sync = true)
+    get_registry(basepath; sync = true)
 
 Clone the DocumentationGenerator registry into `basepath`. No download will occur if `sync == false`
 and the registry already exists.
 
 Returns the path to the `Registry.toml` (or `nothing` if an error occured).
 """
-function get_registry(basepath; registry=DOCS_REGISTRY, sync = true)
-    tomlpath = joinpath(basepath, "DocumentationGeneratorRegistry", "Registry.toml")
+function get_registry(basepath::AbstractString; sync::Bool = true)
+    destdir = joinpath(basepath, "DocumentationGeneratorRegistry")
     if sync
-        try
-
-            destdir = joinpath(basepath, "DocumentationGeneratorRegistry")
-            mktempdir() do temp
-                tempclone = joinpath(temp, "registry")
-                run(`git clone --depth=1 $(registry) $(tempclone)`)
-                @assert isfile(joinpath(tempclone, "Registry.toml"))
-                mv(tempclone, destdir, force = true)
+        for registry in DOCS_REGISTRIES
+            try
+                return _clone_registry(registry; destdir)
+            catch e
+                @warn "Unable to clone DocumentationGeneratorRegistry" registry exception = e
             end
-            return tomlpath
-        catch err
-            @warn("Couldn't download docs registry.", exception = err)
         end
+        # If we weren't able to download the registry from any of the URLs, we fall back
+        # to checking the local filesystem.
+        @warn "Couldn't download docs registry, see the warnings above."
     end
+    tomlpath = joinpath(destdir, "Registry.toml")
     if isfile(tomlpath)
         return tomlpath
     elseif !sync
         @warn("No registry found at `$(tomlpath)`. Cloning again.")
-        return get_registry(basepath; registry = registry, sync = true)
+        return get_registry(basepath; sync = true)
     end
     return nothing
 end
 
+function _clone_registry(registry::AbstractString; destdir::AbstractString)
+    mktempdir() do temp
+        tempclone = joinpath(temp, "registry")
+        git = Sys.which("git")
+        isnothing(git) && error("Sys.which was unable to find the 'git' binary")
+        git = addenv(`$git`, "GIT_TERMINAL_PROMPT" => "0")
+        run(`$git clone --depth=1 $(registry) $(tempclone)`)
+        if !isfile(joinpath(tempclone, "Registry.toml"))
+            error("Failed to clone registry: Registry.toml missing")
+        end
+        mv(tempclone, destdir, force = true)
+    end
+    tomlpath = joinpath(destdir, "Registry.toml")
+    if !isfile(tomlpath)
+        error("Registry clone succesful, but Registry.toml missing\n at $(tomlpath)")
+    end
+    return tomlpath
+end
+
 """
     doctype(packagespec, registry)