add integration tests

- set CI_DEV_PKG_ environment in unit tests

.ci/CI/src/Bootloader.jl

@@ -1,8 +1,11 @@
 module Bootloader
 
 include("./get_target_branch.jl")
+include("./integTestGen.jl")
 
 using .TargetBranch
+using .integTestGen
+using IntegrationTests
 using TOML
 using Logging
 using YAML
@@ -32,7 +35,7 @@ function get_unit_test_julia_versions()::Vector{String}
     return ["1.10", "1.11", "rc", "nightly"]
 end
 
-function get_git_ci_tools_url_branch()::Tuple{String, String}
+function get_git_ci_tools_url_branch()::Tuple{String,String}
     url = "https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git"
     branch = "dev"
 
@@ -53,6 +56,9 @@ function add_unit_test_job_yaml!(
     job_dict::Dict,
     julia_versions::Vector{String},
     target_branch::String,
+    dev_package_name::AbstractString,
+    dev_package_version::AbstractString,
+    dev_package_path::AbstractString,
     git_ci_tools_url::String="https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git",
     git_ci_tools_branch::String="dev",
 )
@@ -65,15 +71,33 @@ function add_unit_test_job_yaml!(
     for version in julia_versions
         if version != "nightly"
             job_dict["unit_test_julia_$(replace(version, "." => "_"))"] = get_normal_unit_test(
-                version, target_branch, git_ci_tools_url, git_ci_tools_branch
+                version,
+                target_branch,
+                dev_package_name,
+                dev_package_version,
+                dev_package_path,
+                git_ci_tools_url,
+                git_ci_tools_branch,
             )
         else
             job_dict["unit_test_julia_nightly"] = get_nighlty_unit_test(
-                target_branch, git_ci_tools_url, git_ci_tools_branch
+                target_branch,
+                dev_package_name,
+                dev_package_version,
+                dev_package_path,
+                git_ci_tools_url,
+                git_ci_tools_branch,
             )
         end
     end
+end
 
+function add_unit_test_verify_job_yaml!(
+    job_dict::Dict,
+    target_branch::String,
+    git_ci_tools_url::String="https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git",
+    git_ci_tools_branch::String="dev",
+)
     # verification script that no custom URLs are used in unit tests
     if target_branch != "main"
         push!(job_dict["stages"], "verify-unit-test-deps")
@@ -94,20 +118,24 @@ end
 function get_normal_unit_test(
     version::String,
     target_branch::String,
+    dev_package_name::AbstractString,
+    dev_package_version::AbstractString,
+    dev_package_path::AbstractString,
     git_ci_tools_url::String,
     git_ci_tools_branch::String,
 )::Dict
     job_yaml = Dict()
     job_yaml["stage"] = "unit-test"
+    job_yaml["variables"] = Dict(
+        "CI_DEV_PKG_NAME" => dev_package_name,
+        "CI_DEV_PKG_VERSION" => dev_package_version,
+        "CI_DEV_PKG_PATH" => dev_package_path,
+    )
     job_yaml["image"] = "julia:$(version)"
 
     script = [
         "apt update && apt install -y git",
         "git clone --depth 1 -b $(git_ci_tools_branch) $(git_ci_tools_url) /tmp/integration_test_tools/",
-        "\$(julia --project=. /tmp/integration_test_tools/.ci/integTestGen/src/get_project_name_version_path.jl)",
-        "echo \"CI_DEV_PKG_NAME -> \$CI_DEV_PKG_NAME\"",
-        "echo \"CI_DEV_PKG_VERSION -> \$CI_DEV_PKG_VERSION\"",
-        "echo \"CI_DEV_PKG_PATH -> \$CI_DEV_PKG_PATH\"",
     ]
 
     if target_branch == "main"
@@ -138,14 +166,27 @@ function get_normal_unit_test(
 end
 
 function get_nighlty_unit_test(
-    target_branch::String, git_ci_tools_url::String, git_ci_tools_branch::String
+    target_branch::String,
+    dev_package_name::AbstractString,
+    dev_package_version::AbstractString,
+    dev_package_path::AbstractString,
+    git_ci_tools_url::String,
+    git_ci_tools_branch::String,
 )
     job_yaml = get_normal_unit_test(
-        "1", target_branch, git_ci_tools_url, git_ci_tools_branch
+        "1",
+        target_branch,
+        dev_package_name,
+        dev_package_version,
+        dev_package_path,
+        git_ci_tools_url,
+        git_ci_tools_branch,
     )
     job_yaml["image"] = "debian:bookworm-slim"
 
-    job_yaml["variables"] = Dict()
+    if !haskey(job_yaml, "variables")
+        job_yaml["variables"] = Dict()
+    end
     job_yaml["variables"]["JULIA_DONWLOAD"] = "/julia/download"
     job_yaml["variables"]["JULIA_EXTRACT"] = "/julia/extract"
 
@@ -176,6 +217,86 @@ fi",
     return job_yaml
 end
 
+function add_integration_test_job_yaml!(
+    job_dict::Dict,
+    package_name::AbstractString,
+    package_version::AbstractString,
+    package_path::AbstractString,
+    git_ci_tools_url::String,
+    git_ci_tools_branch::String,
+)
+    if !haskey(job_dict, "stages")
+        job_dict["stages"] = []
+    end
+
+    package_infos = integTestGen.get_package_info()
+    if target_branch != "main"
+        integTestGen.extract_env_vars_from_git_message!(package_infos)
+    end
+    integTestGen.modify_package_url!(package_infos)
+
+    custom_urls = Dict{String,String}()
+    for (name, info) in package_infos
+        if info.modified_url != ""
+            custom_urls[name] = info.modified_url
+        end
+    end
+    qed_path = mktempdir(; cleanup=false)
+    compat_changes = Dict{String,String}()
+
+    pkg_tree = integTestGen.build_qed_dependency_graph!(
+        qed_path, compat_changes, custom_urls
+    )
+    depending_pkg = IntegrationTests.depending_projects(
+        package_name, collect(keys(package_infos)), pkg_tree
+    )
+
+    if !isempty(depending_pkg)
+        push!(job_dict["stages"], "integ-test")
+        for p in depending_pkg
+            if target_branch == "main" && TargetBranch.is_pull_request()
+                integTestGen.generate_job_yaml!(
+                    p,
+                    "dev",
+                    package_name,
+                    package_version,
+                    package_path,
+                    job_dict,
+                    package_infos,
+                    git_ci_tools_url,
+                    git_ci_tools_branch,
+                )
+                integTestGen.generate_job_yaml!(
+                    p,
+                    "main",
+                    package_name,
+                    package_version,
+                    package_path,
+                    job_dict,
+                    package_infos,
+                    git_ci_tools_url,
+                    git_ci_tools_branch,
+                    "integ-test",
+                    true,
+                )
+            else
+                integTestGen.generate_job_yaml!(
+                    p,
+                    target_branch,
+                    package_name,
+                    package_version,
+                    package_path,
+                    job_dict,
+                    package_infos,
+                    git_ci_tools_url,
+                    git_ci_tools_branch,
+                    "integ-test",
+                )
+            end
+        end
+    end
+end
+
 function print_job_yaml(job_yaml::Dict, io::IO=stdout)
     job_yaml_copy = deepcopy(job_yaml)
 
@@ -195,6 +316,15 @@ function print_job_yaml(job_yaml::Dict, io::IO=stdout)
         end
     end
 
+    # print all integration tests with an empty line between
+    for (top_level_object, top_level_object_value) in job_yaml_copy
+        if startswith(top_level_object, "integration_test_")
+            YAML.write(io, top_level_object => top_level_object_value)
+            println()
+            delete!(job_yaml_copy, top_level_object)
+        end
+    end
+
     # print everything, which was not already printed
     if !isempty(job_yaml_copy)
         YAML.write(io, job_yaml_copy)
@@ -224,10 +354,26 @@ if abspath(PROGRAM_FILE) == @__FILE__
         job_yaml,
         unit_test_julia_versions,
         target_branch,
+        package_name,
+        package_version,
+        package_path,
+        git_ci_tools_url,
+        git_ci_tools_branch,
+    )
+
+    add_integration_test_job_yaml!(
+        job_yaml,
+        package_name,
+        package_version,
+        package_path,
         git_ci_tools_url,
         git_ci_tools_branch,
     )
 
+    add_unit_test_verify_job_yaml!(
+        job_yaml, target_branch, git_ci_tools_url, git_ci_tools_branch
+    )
+
     print_job_yaml(job_yaml)
 end