diff --git a/.ci/integTestGen/src/integTestGen.jl b/.ci/integTestGen/src/integTestGen.jl index 4f98bd3..fe00614 100644 --- a/.ci/integTestGen/src/integTestGen.jl +++ b/.ci/integTestGen/src/integTestGen.jl @@ -211,13 +211,18 @@ Generate GitLab CI job yaml for integration testing of a given package. # Args - `package_name::String`: Name of the package to test. - `target_branch::AbstractString`: Name of the target branch of the pull request. +- `ci_project_dir::AbstractString`: Path of QED project which should be used for the integration test. - `job_yaml::Dict`: Add generated job to this dict. +- `package_infos::AbstractDict{String,PackageInfo}`: Contains serveral information about QED packages +- `can_fail::Bool=false`: If true add `allow_failure=true` to the job yaml """ function generate_job_yaml!( package_name::String, target_branch::AbstractString, + ci_project_dir::AbstractString, job_yaml::Dict, package_infos::AbstractDict{String,PackageInfo}, + can_fail::Bool=false, ) package_info = package_infos[package_name] # if modified_url is empty, use original url @@ -257,7 +262,6 @@ function generate_job_yaml!( script, "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'", ) - ci_project_dir = ENV["CI_PROJECT_DIR"] push!( script, "julia --project=. -e 'import Pkg; Pkg.develop(path=\"$ci_project_dir\");'" ) @@ -268,12 +272,19 @@ function generate_job_yaml!( end push!(script, "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'") - return job_yaml["IntegrationTest$package_name"] = Dict( + current_job_yaml = Dict( "image" => "julia:1.9", "interruptible" => true, "tags" => ["cpuonly"], "script" => script, ) + + if can_fail + current_job_yaml["allow_failure"] = true + return job_yaml["IntegrationTest$(package_name)ReleaseTest"] = current_job_yaml + else + return job_yaml["IntegrationTest$package_name"] = current_job_yaml + end end """ @@ -292,15 +303,13 @@ function generate_dummy_job_yaml!(job_yaml::Dict) ) end -if abspath(PROGRAM_FILE) == @__FILE__ - if !haskey(ENV, "CI_COMMIT_REF_NAME") - @warn "Environemnt variable CI_COMMIT_REF_NAME not defined. Use default branch `dev`." - target_branch = "dev" - else - target_branch = get_target() - end +""" + get_package_info()::Dict{String,PackageInfo} - package_infos = Dict( +Returns a list with QED project package information. +""" +function get_package_info()::Dict{String,PackageInfo} + return Dict( "QED" => PackageInfo( "https://github.com/QEDjl-project/QED.jl.git", "CI_INTG_PKG_URL_QED" ), @@ -323,6 +332,17 @@ if abspath(PROGRAM_FILE) == @__FILE__ "https://github.com/QEDjl-project/QEDcore.jl.git", "CI_INTG_PKG_URL_QEDcore" ), ) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + if !haskey(ENV, "CI_COMMIT_REF_NAME") + @warn "Environemnt variable CI_COMMIT_REF_NAME not defined. Use default branch `dev`." + target_branch = "dev" + else + target_branch = get_target() + end + + package_infos = get_package_info() # custom commit message variable can be set as first argument if length(ARGS) < 1 @@ -343,7 +363,29 @@ if abspath(PROGRAM_FILE) == @__FILE__ if !isempty(depending_pkg) for p in depending_pkg - generate_job_yaml!(p, target_branch, job_yaml, package_infos) + # Handles the case of merging in the main branch. If we want to merge in the main branch, + # we do it because we want to publish the package. Therefore, we need to be sure that there + # is an existing version of the dependent QED packages that works with the new version of + # the package we want to release. The integration tests are tested against the development + # branch and the release version. + # - The dev branch version must pass, as this means that the latest version of the other + # QED packages is compatible with our release version. + # - The release version integration tests may or may not pass. + # 1. If all of these pass, we will not need to increase the minor version of this package. + # 2. If they do not all pass, the minor version must be increased and the failing packages + # must also be released later with an updated compat entry. + # In either case the release can proceed, as the released packages will continue to work + # because of their current compat entries. + if target_branch == "main" && is_pull_request() + generate_job_yaml!(p, "dev", ENV["CI_PROJECT_DIR"], job_yaml, package_infos) + generate_job_yaml!( + p, "main", ENV["CI_PROJECT_DIR"], job_yaml, package_infos, true + ) + else + generate_job_yaml!( + p, target_branch, ENV["CI_PROJECT_DIR"], job_yaml, package_infos + ) + end end else generate_dummy_job_yaml!(job_yaml) diff --git a/.ci/integTestGen/test/Project.toml b/.ci/integTestGen/test/Project.toml index 6fa13a9..b12d2e9 100644 --- a/.ci/integTestGen/test/Project.toml +++ b/.ci/integTestGen/test/Project.toml @@ -2,3 +2,4 @@ PkgDependency = "9eb5382b-762c-48ca-8139-e736883fe800" Term = "22787eb5-b846-44ae-b979-8e399b8463ab" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" diff --git a/.ci/integTestGen/test/generate_job_yaml.jl b/.ci/integTestGen/test/generate_job_yaml.jl new file mode 100644 index 0000000..ec92685 --- /dev/null +++ b/.ci/integTestGen/test/generate_job_yaml.jl @@ -0,0 +1,197 @@ +using YAML + +""" + yaml_diff(given, expected)::AbstractString + +Generates an error string that shows a given and an expected data structure in yaml +representation. + +# Returns +- Human readable error message for the comparison of two job yaml's. +""" +function yaml_diff(given, expected)::AbstractString + output = "\ngiven:\n" + output *= String(YAML.yaml(given)) + output *= "\nexpected:\n" + output *= String(YAML.yaml(expected)) + return output +end + +@testset "generate_job_yaml()" begin + package_infos = integTestGen.get_package_info() + + @testset "target main branch, no PR" begin + job_yaml = Dict() + integTestGen.generate_job_yaml!( + "QEDcore", "main", "/path/to/QEDcore.jl", job_yaml, package_infos + ) + @test length(job_yaml) == 1 + + expected_job_yaml = Dict() + expected_job_yaml["IntegrationTestQEDcore"] = Dict( + "image" => "julia:1.9", + "interruptible" => true, + "tags" => ["cpuonly"], + "script" => [ + "apt update", + "apt install -y git", + "cd /", + "git clone -b main $(package_infos["QEDcore"].url) integration_test", + "cd integration_test", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/QEDjl-project/registry.git\"));'", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'", + "julia --project=. -e 'import Pkg; Pkg.develop(path=\"/path/to/QEDcore.jl\");'", + "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'", + ], + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcore"]["script"] == + expected_job_yaml["IntegrationTestQEDcore"]["script"] yaml_diff( + job_yaml["IntegrationTestQEDcore"]["script"], + expected_job_yaml["IntegrationTestQEDcore"]["script"], + ); + true + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcore"] == + expected_job_yaml["IntegrationTestQEDcore"] yaml_diff( + job_yaml["IntegrationTestQEDcore"], + expected_job_yaml["IntegrationTestQEDcore"], + ); + true + ) + end + + @testset "target non main branch, if PR or not is the same" begin + job_yaml = Dict() + integTestGen.generate_job_yaml!( + "QEDcore", "feature3", "/path/to/QEDcore.jl", job_yaml, package_infos + ) + @test length(job_yaml) == 1 + + expected_job_yaml = Dict() + expected_job_yaml["IntegrationTestQEDcore"] = Dict( + "image" => "julia:1.9", + "interruptible" => true, + "tags" => ["cpuonly"], + "script" => [ + "apt update", + "apt install -y git", + "cd /", + "git clone -b feature3 $(package_infos["QEDcore"].url) integration_test", + "git clone -b dev https://github.com/QEDjl-project/QED.jl.git /integration_test_tools", + "cd integration_test", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/QEDjl-project/registry.git\"));'", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'", + "julia --project=. -e 'import Pkg; Pkg.develop(path=\"/path/to/QEDcore.jl\");'", + "julia --project=. /integration_test_tools/.ci/set_dev_dependencies.jl", + "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'", + ], + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcore"]["script"] == + expected_job_yaml["IntegrationTestQEDcore"]["script"] yaml_diff( + job_yaml["IntegrationTestQEDcore"]["script"], + expected_job_yaml["IntegrationTestQEDcore"]["script"], + ); + true + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcore"] == + expected_job_yaml["IntegrationTestQEDcore"] yaml_diff( + job_yaml["IntegrationTestQEDcore"], + expected_job_yaml["IntegrationTestQEDcore"], + ); + true + ) + end + + @testset "target main branch, PR" begin + job_yaml = Dict() + integTestGen.generate_job_yaml!( + "QEDcore", "dev", "/path/to/QEDcore.jl", job_yaml, package_infos + ) + integTestGen.generate_job_yaml!( + "QEDcore", "main", "/path/to/QEDcore.jl", job_yaml, package_infos, true + ) + @test length(job_yaml) == 2 + + expected_job_yaml = Dict() + expected_job_yaml["IntegrationTestQEDcore"] = Dict( + "image" => "julia:1.9", + "interruptible" => true, + "tags" => ["cpuonly"], + "script" => [ + "apt update", + "apt install -y git", + "cd /", + "git clone -b dev $(package_infos["QEDcore"].url) integration_test", + "git clone -b dev https://github.com/QEDjl-project/QED.jl.git /integration_test_tools", + "cd integration_test", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/QEDjl-project/registry.git\"));'", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'", + "julia --project=. -e 'import Pkg; Pkg.develop(path=\"/path/to/QEDcore.jl\");'", + "julia --project=. /integration_test_tools/.ci/set_dev_dependencies.jl", + "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'", + ], + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcore"]["script"] == + expected_job_yaml["IntegrationTestQEDcore"]["script"] yaml_diff( + job_yaml["IntegrationTestQEDcore"]["script"], + expected_job_yaml["IntegrationTestQEDcore"]["script"], + ); + true + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcore"] == + expected_job_yaml["IntegrationTestQEDcore"] yaml_diff( + job_yaml["IntegrationTestQEDcore"], + expected_job_yaml["IntegrationTestQEDcore"], + ); + true + ) + + expected_job_yaml["IntegrationTestQEDcoreReleaseTest"] = Dict( + "image" => "julia:1.9", + "interruptible" => true, + "tags" => ["cpuonly"], + "allow_failure" => true, + "script" => [ + "apt update", + "apt install -y git", + "cd /", + "git clone -b main $(package_infos["QEDcore"].url) integration_test", + "cd integration_test", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/QEDjl-project/registry.git\"));'", + "julia --project=. -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(url=\"https://github.com/JuliaRegistries/General\"));'", + "julia --project=. -e 'import Pkg; Pkg.develop(path=\"/path/to/QEDcore.jl\");'", + "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'", + ], + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcoreReleaseTest"]["script"] == + expected_job_yaml["IntegrationTestQEDcoreReleaseTest"]["script"] yaml_diff( + job_yaml["IntegrationTestQEDcoreReleaseTest"]["script"], + expected_job_yaml["IntegrationTestQEDcoreReleaseTest"]["script"], + ); + true + ) + + @test ( + @assert job_yaml["IntegrationTestQEDcoreReleaseTest"] == + expected_job_yaml["IntegrationTestQEDcoreReleaseTest"] yaml_diff( + job_yaml["IntegrationTestQEDcoreReleaseTest"], + expected_job_yaml["IntegrationTestQEDcoreReleaseTest"], + ); + true + ) + end +end diff --git a/.ci/integTestGen/test/runtests.jl b/.ci/integTestGen/test/runtests.jl index ac7ea7d..db14431 100644 --- a/.ci/integTestGen/test/runtests.jl +++ b/.ci/integTestGen/test/runtests.jl @@ -3,3 +3,4 @@ using Test include("./integTestGen.jl") include("./get_target_branch.jl") +include("./generate_job_yaml.jl")