From a58fc5b9ae1169a173d77955df9359d605057e2c Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 30 Nov 2023 10:32:00 +1300 Subject: [PATCH 1/3] Use AbstractTrees 0.4 (#212) --- Project.toml | 4 +-- src/builders.jl | 86 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/Project.toml b/Project.toml index 2616c819..1bb88ecb 100644 --- a/Project.toml +++ b/Project.toml @@ -2,7 +2,7 @@ name = "DocumentationGenerator" uuid = "8f0d3306-d70b-5309-b898-24bb6ab47850" authors = ["Sebastian Pfitzner ", "Simon Danisch ", "Venkatesh Dayananda "] repo = "https://github.com/JuliaDocs/DocumentationGenerator.jl.git" -version = "0.7.5" +version = "0.7.6" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" @@ -22,7 +22,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -AbstractTrees = "0.2, 0.3" +AbstractTrees = "0.4" Documenter = "0.24, 0.25, 0.26, 0.27" GitHub = "5.1, 5.2, 5.3, 5.4" GitForge = "0.4" diff --git a/src/builders.jl b/src/builders.jl index 37670643..ed6791c5 100644 --- a/src/builders.jl +++ b/src/builders.jl @@ -475,43 +475,69 @@ function postprocess_html_readme(html; src_prefix="", href_prefix="") href_prefix = string(href_prefix, '/') end - header_refs = Set([]) - for el in PreOrderDFS(doc) + # We'll be transforming some of the elements in the tree. For headings and + # code highlighting, we actually have to mutate the tree. So in that case + # we must perform the mutation after iterating with PreOrderDFS(), since the + # iteration is stateful and will get confused if the tree's structure gets + # changed underneath it. + heading_elements, highlight_elements = HTMLElement[], HTMLElement[] + for el in AbstractTrees.PreOrderDFS(doc) if el isa HTMLElement if Gumbo.tag(el) in [:h1, :h2, :h3, :h4, :h5] - heading = replace(text(el), r"\s" => "-") - heading = replace(heading, r"[^\w-]" => "") - - while heading in header_refs - new_heading = replace(heading, r"_\d+$" => s -> begin - string('_', parse(Int, s[2:end]) + 1) - end) - if new_heading == heading - heading = string(heading, "_1") - else - heading = new_heading - end + push!(heading_elements, el) + elseif Gumbo.tag(el) == :code + if Gumbo.tag(el.parent) == :pre && length(el.children) == 1 && typeof(el.children[1]) == HTMLText + push!(highlight_elements, el) end + end + end + end - orig_content = deepcopy(el.children) - a_el = HTMLElement{:a}(orig_content, el, Dict( - "href" => string('#', heading), - "class" => "docs-heading-anchor" - )) + # Mutate tags + header_refs = Set([]) # this is to avoid duplicate IDs + for el in heading_elements + # For heading elements we essentially just push a new + # tag between the tag and the child nodes. + heading = replace(text(el), r"\s" => "-") + heading = replace(heading, r"[^\w-]" => "") + + while heading in header_refs + new_heading = replace(heading, r"_\d+$" => s -> begin + string('_', parse(Int, s[2:end]) + 1) + end) + if new_heading == heading + heading = string(heading, "_1") + else + heading = new_heading + end + end - for c in orig_content - c.parent = a_el - end + orig_content = deepcopy(el.children) + a_el = HTMLElement{:a}(orig_content, el, Dict( + "href" => string('#', heading), + "class" => "docs-heading-anchor" + )) - el.children = [a_el] + for c in orig_content + c.parent = a_el + end - setattr!(el, "id", heading) - push!(header_refs, heading) - elseif Gumbo.tag(el) == :code - if Gumbo.tag(el.parent) == :pre && length(el.children) == 1 && typeof(el.children[1]) == HTMLText - highlight_syntax_html(el) - end - elseif hasattr(el, "src") + el.children = [a_el] + + setattr!(el, "id", heading) + push!(header_refs, heading) + end + + # Mutate the highlighting-related elements + for el in highlight_elements + highlight_syntax_html(el) + end + + # Update the 'src' and 'href' attributes of all the tags that have them, + # applying a URL prefix. + for el in AbstractTrees.PreOrderDFS(doc) + if el isa HTMLElement + if hasattr(el, "src") replace_url(el, "src", src_prefix, fix_github = true) elseif hasattr(el, "href") replace_url(el, "href", href_prefix) From 88e21e839f739ba54359d80eeeddc0b6a6ffef10 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 30 Nov 2023 15:56:51 +1300 Subject: [PATCH 2/3] Support registry hosted under JuliaComputing (#213) --- src/DocumentationGenerator.jl | 2 +- src/registry.jl | 52 ++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/DocumentationGenerator.jl b/src/DocumentationGenerator.jl index 15c0eac0..ce273aa1 100644 --- a/src/DocumentationGenerator.jl +++ b/src/DocumentationGenerator.jl @@ -93,7 +93,7 @@ function build_documentation( false end - regpath = get_registry(basepath, sync = sync_registry) + regpath = get_registry(basepath; sync = sync_registry) process_queue = [] # make sure registry is updated *before* we start multiple processes that might try that at the same time diff --git a/src/registry.jl b/src/registry.jl index 8bd4a9ce..2037bc80 100644 --- a/src/registry.jl +++ b/src/registry.jl @@ -1,39 +1,59 @@ -const DOCS_REGISTRY = "https://github.com/JuliaDocs/DocumentationGeneratorRegistry.git" +const DOCS_REGISTRIES = [ + "https://github.com/JuliaComputing/DocumentationGeneratorRegistry.git", + "https://github.com/JuliaDocs/DocumentationGeneratorRegistry.git", +] """ - get_registry(basepath; registry=DOCS_REGISTRY, sync = true) + get_registry(basepath; sync = true) Clone the DocumentationGenerator registry into `basepath`. No download will occur if `sync == false` and the registry already exists. Returns the path to the `Registry.toml` (or `nothing` if an error occured). """ -function get_registry(basepath; registry=DOCS_REGISTRY, sync = true) - tomlpath = joinpath(basepath, "DocumentationGeneratorRegistry", "Registry.toml") +function get_registry(basepath::AbstractString; sync::Bool = true) + destdir = joinpath(basepath, "DocumentationGeneratorRegistry") if sync - try - - destdir = joinpath(basepath, "DocumentationGeneratorRegistry") - mktempdir() do temp - tempclone = joinpath(temp, "registry") - run(`git clone --depth=1 $(registry) $(tempclone)`) - @assert isfile(joinpath(tempclone, "Registry.toml")) - mv(tempclone, destdir, force = true) + for registry in DOCS_REGISTRIES + try + return _clone_registry(registry; destdir) + catch e + @warn "Unable to clone DocumentationGeneratorRegistry" registry exception = e end - return tomlpath - catch err - @warn("Couldn't download docs registry.", exception = err) end + # If we weren't able to download the registry from any of the URLs, we fall back + # to checking the local filesystem. + @warn "Couldn't download docs registry, see the warnings above." end + tomlpath = joinpath(destdir, "Registry.toml") if isfile(tomlpath) return tomlpath elseif !sync @warn("No registry found at `$(tomlpath)`. Cloning again.") - return get_registry(basepath; registry = registry, sync = true) + return get_registry(basepath; sync = true) end return nothing end +function _clone_registry(registry::AbstractString; destdir::AbstractString) + mktempdir() do temp + tempclone = joinpath(temp, "registry") + git = Sys.which("git") + isnothing(git) && error("Sys.which was unable to find the 'git' binary") + git = addenv(`$git`, "GIT_TERMINAL_PROMPT" => "0") + run(`$git clone --depth=1 $(registry) $(tempclone)`) + if !isfile(joinpath(tempclone, "Registry.toml")) + error("Failed to clone registry: Registry.toml missing") + end + mv(tempclone, destdir, force = true) + end + tomlpath = joinpath(destdir, "Registry.toml") + if !isfile(tomlpath) + error("Registry clone succesful, but Registry.toml missing\n at $(tomlpath)") + end + return tomlpath +end + """ doctype(packagespec, registry) From 429e6b420ccbdbeb98bca39880e4a66c16c27469 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 30 Nov 2023 22:30:43 +1300 Subject: [PATCH 3/3] Prefer JuliaDocs registry in 0.7, but fall back to JuliaComputing (#214) * Prefer JuliaDocs registry in 0.7, but fall back to JuliaComputing --- src/registry.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/registry.jl b/src/registry.jl index 2037bc80..1e627705 100644 --- a/src/registry.jl +++ b/src/registry.jl @@ -1,6 +1,10 @@ +# The registry cloning URLs are tried in order. Currently (on the 0.7 branch), +# we still try to clone the registry first from JuliaDocs, remaining consistent +# with the earlier releases. But we will fall back to the JuliaComputing URL +# if it stops working. const DOCS_REGISTRIES = [ - "https://github.com/JuliaComputing/DocumentationGeneratorRegistry.git", "https://github.com/JuliaDocs/DocumentationGeneratorRegistry.git", + "https://github.com/JuliaComputing/DocumentationGeneratorRegistry.git", ] """