Skip to content

Commit

Permalink
add code formatter job (QEDjl-project#15)
Browse files Browse the repository at this point in the history 
- format manually the source code
  • Loading branch information
@SimeonEhrig
SimeonEhrig authored Sep 28, 2023
1 parent 7164cef commit 6798754
Show file tree
Hide file tree
Showing 10 changed files with 222 additions and 106 deletions.
2 changes: 2 additions & 0 deletions .JuliaFormatter.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
style = "blue"

26 changes: 17 additions & 9 deletions .ci/SetupDevEnv/src/SetupDevEnv.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ using Pkg
Parse the commit message, if set via variable (usual `CI_COMMIT_MESSAGE`) and set custom URLs.
"""
function extract_env_vars_from_git_message!(var_name = "CI_COMMIT_MESSAGE")
function extract_env_vars_from_git_message!(var_name="CI_COMMIT_MESSAGE")
if haskey(ENV, var_name)
@info "Found env variable $var_name"
for line in split(ENV[var_name], "\n")
Expand All @@ -28,12 +28,16 @@ end
Parses a Project.toml located at `project_toml_path` and returns all dependencies, matching the regex `package_prefix`.
By default, the regex allows all dependencies.
"""
function get_dependencies(project_toml_path::AbstractString, package_prefix::Union{AbstractString,Regex}=r".*")::Set{AbstractString}
function get_dependencies(
project_toml_path::AbstractString, package_prefix::Union{AbstractString,Regex}=r".*"
)::Set{AbstractString}
project_toml = TOML.parsefile(project_toml_path)
if ! haskey(project_toml, "deps")
if !haskey(project_toml, "deps")
return Set()
end
filtered_deps = filter((dep_name)-> startswith(dep_name, package_prefix), keys(project_toml["deps"]))
filtered_deps = filter(
(dep_name) -> startswith(dep_name, package_prefix), keys(project_toml["deps"])
)
return filtered_deps
end

Expand All @@ -47,10 +51,10 @@ develop version, instead the the default develop branch (see `Pkg.develop(url=)`
function add_develop_dep(dependencies::Set{AbstractString})
# check if specific url was set for a dependency
env_prefix = "CI_UNIT_PKG_URL_"
modified_urls = Dict{String, String}()
modified_urls = Dict{String,String}()
for (env_key, env_var) in ENV
if startswith(env_key, env_prefix)
modified_urls[env_key[length(env_prefix)+1:end]] = env_var
modified_urls[env_key[(length(env_prefix) + 1):end]] = env_var
end
end

Expand All @@ -72,10 +76,14 @@ function add_develop_dep(dependencies::Set{AbstractString})

if length(split_url) == 1
@info "Pkg.develop(url=\"" * split_url[1] * "\")"
Pkg.add(url=split_url[1])
Pkg.add(; url=split_url[1])
else
@info "Pkg.develop(url=\"" * split_url[1] * ";\" rev=\"" * split_url[2] * "\")"
Pkg.add(url=split_url[1], rev=split_url[2])
@info "Pkg.develop(url=\"" *
split_url[1] *
";\" rev=\"" *
split_url[2] *
"\")"
Pkg.add(; url=split_url[1], rev=split_url[2])
end
else
@info "Pkg.develop(\"" * dep * "\")"
Expand Down
72 changes: 40 additions & 32 deletions .ci/SetupDevEnv/test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,17 @@ using Test
@test !haskey(ENV, message_var_name)
# should not throw an error if the environment CI_COMMIT_MESSAGE does not exist
SetupDevEnv.extract_env_vars_from_git_message!(message_var_name)

ENV[message_var_name] = """This is a normal feature.
The feature can do someting useful.
Be carful, when you use the feature.
"""
SetupDevEnv.extract_env_vars_from_git_message!(message_var_name)
@test isempty(filter((env_name) -> startswith(env_name, "CI_UNIT_PKG_URL_"), keys(ENV)))
@test isempty(
filter((env_name) -> startswith(env_name, "CI_UNIT_PKG_URL_"), keys(ENV))
)

ENV[message_var_name] = """This is a normal feature.
Expand Down Expand Up @@ -71,44 +73,50 @@ using Test
tmp_path = mktempdir()

@testset "no dependencies" begin
project_path=joinpath(tmp_path, "Project1.toml")
project_path = joinpath(tmp_path, "Project1.toml")
open(project_path, "w") do f
write(f, """
name = "QED"
uuid = "bb1fba1d-cf9b-41b3-874e-4b81465537b9"
authors = ["Uwe Hernandez Acosta <[email protected]>", "Simeon Ehrig", "Klaus Steiniger", "Tom Jungnickel", "Anton Reinhard"]
version = "0.1.0"
[compat]
julia = "1.9"
""")
write(
f,
"""
name = "QED"
uuid = "bb1fba1d-cf9b-41b3-874e-4b81465537b9"
authors = ["Uwe Hernandez Acosta <[email protected]>", "Simeon Ehrig", "Klaus Steiniger", "Tom Jungnickel", "Anton Reinhard"]
version = "0.1.0"
[compat]
julia = "1.9"
""",
)
end

@test isempty(SetupDevEnv.get_dependencies(project_path))
end

@testset "test filter" begin
project_path=joinpath(tmp_path, "Project2.toml")
project_path = joinpath(tmp_path, "Project2.toml")
open(project_path, "w") do f
write(f, """
name = "QED"
uuid = "bb1fba1d-cf9b-41b3-874e-4b81465537b9"
authors = ["Uwe Hernandez Acosta <[email protected]>", "Simeon Ehrig", "Klaus Steiniger", "Tom Jungnickel", "Anton Reinhard"]
version = "0.1.0"
[deps]
QEDbase = "10e22c08-3ccb-4172-bfcf-7d7aa3d04d93"
QEDevents = "fc3ce04a-5be5-4f3a-acff-eceaab723759"
QEDfields = "ac3a6c97-e859-4b9f-96bb-63d2a216042c"
QEDprocesses = "46de9c38-1bb3-4547-a1ec-da24d767fdad"
PhysicalConstants = "5ad8b20f-a522-5ce9-bfc9-ddf1d5bda6ab"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
SimpleTraits = "699a6c99-e7fa-54fc-8d76-47d257e15c1d"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
[compat]
julia = "1.9"
""")
write(
f,
"""
name = "QED"
uuid = "bb1fba1d-cf9b-41b3-874e-4b81465537b9"
authors = ["Uwe Hernandez Acosta <[email protected]>", "Simeon Ehrig", "Klaus Steiniger", "Tom Jungnickel", "Anton Reinhard"]
version = "0.1.0"
[deps]
QEDbase = "10e22c08-3ccb-4172-bfcf-7d7aa3d04d93"
QEDevents = "fc3ce04a-5be5-4f3a-acff-eceaab723759"
QEDfields = "ac3a6c97-e859-4b9f-96bb-63d2a216042c"
QEDprocesses = "46de9c38-1bb3-4547-a1ec-da24d767fdad"
PhysicalConstants = "5ad8b20f-a522-5ce9-bfc9-ddf1d5bda6ab"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
SimpleTraits = "699a6c99-e7fa-54fc-8d76-47d257e15c1d"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
[compat]
julia = "1.9"
""",
)
end
@test length(SetupDevEnv.get_dependencies(project_path)) == 8
@test length(SetupDevEnv.get_dependencies(project_path, r"^QED")) == 4
Expand Down
106 changes: 70 additions & 36 deletions .ci/integTestGen/src/integTestGen.jl
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
module integTestGen

import Pkg
import PkgDependency
import YAML
using Pkg: Pkg
using PkgDependency: PkgDependency
using YAML: YAML

"""
create_working_env(project_path::AbstractString)
Expand All @@ -24,7 +24,7 @@ function create_working_env(project_path::AbstractString)
Pkg.add("PkgDependency")
Pkg.add("YAML")
# add main project as dependency
Pkg.develop(path=project_path)
return Pkg.develop(; path=project_path)
end

"""
Expand All @@ -47,13 +47,17 @@ end
Parse the commit message, if set via variable (usual `CI_COMMIT_MESSAGE`), and set custom URLs.
"""
function extract_env_vars_from_git_message!(package_infos::AbstractDict{String, PackageInfo}, var_name = "CI_COMMIT_MESSAGE")
function extract_env_vars_from_git_message!(
package_infos::AbstractDict{String,PackageInfo}, var_name="CI_COMMIT_MESSAGE"
)
if haskey(ENV, var_name)
for line in split(ENV[var_name], "\n")
line = strip(line)
for pkg_info in values(package_infos)
if startswith(line, pkg_info.env_var * ": ")
ENV[pkg_info.env_var] = SubString(line, length(pkg_info.env_var * ": ") + 1)
ENV[pkg_info.env_var] = SubString(
line, length(pkg_info.env_var * ": ") + 1
)
end
end
end
Expand All @@ -66,7 +70,7 @@ end
Iterate over all entries of package_info. If an environment variable exists with the same name as,
the `env_var` entry, set the value of the environment variable to `modified_url`.
"""
function modify_package_url!(package_infos::AbstractDict{String, PackageInfo})
function modify_package_url!(package_infos::AbstractDict{String,PackageInfo})
for package_info in values(package_infos)
if haskey(ENV, package_info.env_var)
package_info.modified_url = ENV[package_info.env_var]
Expand All @@ -82,7 +86,7 @@ Read the name of the modified (project) package from the environment variable `C
# Returns
- The name of the modified (project) package
"""
function modified_package_name(package_infos::AbstractDict{String, PackageInfo})
function modified_package_name(package_infos::AbstractDict{String,PackageInfo})
for env_var in ["CI_DEPENDENCY_NAME", "CI_PROJECT_DIR"]
if !haskey(ENV, env_var)
error("Environment variable $env_var is not set.")
Expand Down Expand Up @@ -112,7 +116,11 @@ Return a list of packages, which have the package `package_name` as a dependency
- `::AbstractVector{String}`: all packages which have the search dependency
"""
function depending_projects(package_name::String, package_filter, project_tree=PkgDependency.builddict(Pkg.project().uuid, Pkg.project()))::AbstractVector{String}
function depending_projects(
package_name::String,
package_filter,
project_tree=PkgDependency.builddict(Pkg.project().uuid, Pkg.project()),
)::AbstractVector{String}
packages::AbstractVector{String} = []
visited_packages::AbstractVector{String} = []
traverse_tree!(package_name, package_filter, project_tree, packages, visited_packages)
Expand All @@ -126,7 +134,13 @@ Traverse a project tree and add any package to `packages`, that has the package
See [`depending_projects`](@ref)
"""
function traverse_tree!(package_name::String, package_filter, project_tree, packages::AbstractVector{String}, visited_packages::AbstractVector{String})
function traverse_tree!(
package_name::String,
package_filter,
project_tree,
packages::AbstractVector{String},
visited_packages::AbstractVector{String},
)
for project_name_version in keys(project_tree)
# remove project version from string -> usual shape: `packageName.jl version`
project_name = split(project_name_version)[1]
Expand All @@ -135,7 +149,10 @@ function traverse_tree!(package_name::String, package_filter, project_tree, pack
# - the dependency is not nothing (I think this representate, that the package was already set as dependency of a another package and therefore do not repead the dependencies)
# - has dependency
# - was not already checked
if project_name in package_filter && project_tree[project_name_version] !== nothing && !isempty(project_tree[project_name_version]) && !(project_name in visited_packages)
if project_name in package_filter &&
project_tree[project_name_version] !== nothing &&
!isempty(project_tree[project_name_version]) &&
!(project_name in visited_packages)
# only investigate each package one time
# assumption: package name with it's dependency is unique
push!(visited_packages, project_name)
Expand All @@ -147,7 +164,13 @@ function traverse_tree!(package_name::String, package_filter, project_tree, pack
end
end
# independent of a match, under investigate all dependencies too, because they can also have the package as dependency
traverse_tree!(package_name, package_filter, project_tree[project_name_version], packages, visited_packages)
traverse_tree!(
package_name,
package_filter,
project_tree[project_name_version],
packages,
visited_packages,
)
end
end
end
Expand All @@ -161,7 +184,9 @@ Generate GitLab CI job yaml for integration testing of a given package.
- `package_name::String`: Name of the package to test.
- `job_yaml::Dict`: Add generated job to this dict.
"""
function generate_job_yaml!(package_name::String, job_yaml::Dict, package_infos::AbstractDict{String, PackageInfo})
function generate_job_yaml!(
package_name::String, job_yaml::Dict, package_infos::AbstractDict{String,PackageInfo}
)
package_info = package_infos[package_name]
# if modified_url is empty, use original url
if package_info.modified_url == ""
Expand All @@ -170,12 +195,7 @@ function generate_job_yaml!(package_name::String, job_yaml::Dict, package_infos:
url = package_info.modified_url
end

script = [
"apt update",
"apt install -y git",
"cd /"
]

script = ["apt update", "apt install -y git", "cd /"]

split_url = split(url, "#")
if length(split_url) > 2
Expand All @@ -189,17 +209,26 @@ function generate_job_yaml!(package_name::String, job_yaml::Dict, package_infos:
push!(script, "git checkout $(split_url[2])")
end

push!(script, "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/QEDjl-project/registry.git\"));'")
push!(script, "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'")
push!(
script,
"julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/QEDjl-project/registry.git\"));'",
)
push!(
script,
"julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'",
)
ci_project_dir = ENV["CI_PROJECT_DIR"]
push!(script, "julia --project=. -e 'import Pkg; Pkg.develop(path=\"$ci_project_dir\");'")
push!(
script, "julia --project=. -e 'import Pkg; Pkg.develop(path=\"$ci_project_dir\");'"
)
push!(script, "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'")

job_yaml["IntegrationTest$package_name"] = Dict(
return job_yaml["IntegrationTest$package_name"] = Dict(
"image" => "julia:1.9",
"interruptible" => true,
"tags" => ["cpuonly"],
"script" => script)
"script" => script,
)
end

"""
Expand All @@ -211,28 +240,33 @@ Generates a GitLab CI dummy job, if required.
- `job_yaml::Dict`: Add generated job to this dict.
"""
function generate_dummy_job_yaml!(job_yaml::Dict)
job_yaml["DummyJob"] = Dict("image" => "alpine:latest",
return job_yaml["DummyJob"] = Dict(
"image" => "alpine:latest",
"interruptible" => true,
"script" => ["echo \"This is a dummy job so that the CI does not fail.\""])
"script" => ["echo \"This is a dummy job so that the CI does not fail.\""],
)
end

if abspath(PROGRAM_FILE) == @__FILE__
package_infos = Dict(
"QED" => PackageInfo(
"https://github.com/QEDjl-project/QED.jl.git",
"CI_INTG_PKG_URL_QED"),
"https://github.com/QEDjl-project/QED.jl.git", "CI_INTG_PKG_URL_QED"
),
"QEDfields" => PackageInfo(
"https://github.com/QEDjl-project/QEDfields.jl.git",
"CI_INTG_PKG_URL_QEDfields"),
"https://github.com/QEDjl-project/QEDfields.jl.git",
"CI_INTG_PKG_URL_QEDfields",
),
"QEDbase" => PackageInfo(
"https://github.com/QEDjl-project/QEDbase.jl.git",
"CI_INTG_PKG_URL_QEDbase"),
"https://github.com/QEDjl-project/QEDbase.jl.git", "CI_INTG_PKG_URL_QEDbase"
),
"QEDevents" => PackageInfo(
"https://github.com/QEDjl-project/QEDevents.jl.git",
"CI_INTG_PKG_URL_QEDevents"),
"https://github.com/QEDjl-project/QEDevents.jl.git",
"CI_INTG_PKG_URL_QEDevents",
),
"QEDprocesses" => PackageInfo(
"https://github.com/QEDjl-project/QEDprocesses.jl.git",
"CI_INTG_PKG_URL_QEDprocesses"),
"https://github.com/QEDjl-project/QEDprocesses.jl.git",
"CI_INTG_PKG_URL_QEDprocesses",
),
)

# custom commit message variable can be set as first argument
Expand Down
Loading

0 comments on commit 6798754

Please sign in to comment.