Support criu-ubi-portable test in other platforms (#4620)

- Use ubi image with criu binaries as the base image for criu-ubi-portable test - the base image will be switched to dockerhub semeru ubi criu images once available Signed-off-by: LongyuZhang <[email protected]>
adoptium · Jun 12, 2023 · 5acbb92 · 5acbb92
1 parent d9c9656
commit 5acbb92
Show file tree

Hide file tree

Showing 7 changed files with 71 additions and 28 deletions.
diff --git a/external/build_image.sh b/external/build_image.sh
@@ -19,7 +19,7 @@ source $(dirname "$0")/dockerfile_functions.sh
 buildArg=""
 container_build="docker build"
 
-if [ $# -ne 9 ] && [ $# -ne 8 ]; then
+if [ $# -ne 9 ] && [ $# -ne 10 ]; then
 	echo "The supported tests are ${supported_tests}"
 	echo
 	echo "usage: $0 test version vm os package build platform check_external_custom"
@@ -31,14 +31,15 @@ if [ $# -ne 9 ] && [ $# -ne 8 ]; then
 	echo "build   = ${supported_builds}"
 	# TO-DO: ${supported_platforms} will be added when portable tests support more platforms
 	echo "platform" = "mutiple platforms"
+	echo "base_docker_registry_dir" = "public or specified_link"
 	echo "buildArg" = "Optional: customized image"
 	exit -1
 fi
-if [ $# -eq 9 ]; then
-	buildArg="--build-arg IMAGE=$9"
+if [ $# -eq 10 ]; then
+	buildArg="--build-arg IMAGE=$10"
 fi
-check_external_custom=$8
-if [[ ${check_external_custom} -eq 0 ]]; then
+check_external_custom=$9
+if [[ "${check_external_custom}" == "0" ]]; then
 	set_test $1
 fi
 set_version $2
@@ -47,6 +48,7 @@ set_os $4
 set_package $5
 set_build $6
 set_platform $7
+set_base_docker_registry_dir "$8"
 
 # Build the Docker image with the given repo, build, build type and tags.
 function build_image() {
@@ -78,7 +80,7 @@ function build_image() {
 }
 
 # Handle making the directory for organizing the Dockerfiles
-if [[ ${check_external_custom} -eq 1 ]]; then
+if [[ "${check_external_custom}" == "1" ]]; then
 	dir="$(realpath $(dirname "$0"))/external_custom/dockerfile/${version}/${package}/${os}"
 else
 	dir="$(realpath $(dirname "$0"))/${test}/dockerfile/${version}/${package}/${os}"
@@ -89,7 +91,7 @@ mkdir -p ${dir}
 file="${dir}/Dockerfile.${vm}.${build}"
 
 # Generate Dockerfile
-generate_dockerfile ${file} ${test} ${version} ${vm} ${os} ${package} ${build} ${platform} ${check_external_custom}
+generate_dockerfile ${file} ${test} ${version} ${vm} ${os} ${package} ${build} ${platform} "${base_docker_registry_dir}" ${check_external_custom}
 
 # Check if Dockerfile exists
 if [ ! -f ${file} ]; then

diff --git a/external/common_functions.sh b/external/common_functions.sh
@@ -177,6 +177,11 @@ function set_platform() {
     platform=$1
 }
 
+# Set base_docker_registry_dir
+function set_base_docker_registry_dir() {
+    base_docker_registry_dir="$1"
+}
+
 # Reading properties of test.properties file
 function getProperty() {
     PROP_KEY=$1

diff --git a/external/criu-ubi-portable-checkpoint/build.xml b/external/criu-ubi-portable-checkpoint/build.xml
@@ -15,7 +15,29 @@
 		<mkdir dir="${DEST}"/>
 	</target>
 
-	<target name="dist" depends="move_scripts,clean_image,build_image" description="generate the distribution">
+	<target name="build_image" depends="move_scripts,clean_image" description="build the image">
+		<echo message="Executing external.sh --build --dir ${TEST} --tag ${dockerImageTag} --version ${JDK_VERSION} --impl ${JDK_IMPL} --base_docker_registry_dir 'ubi8-with-criu/${env.SPEC}-ubi8-criu' --docker_registry_url ${env.DOCKER_REGISTRY_URL} --docker_args ${extra_docker_args} " />
+		<exec executable="bash" failonerror="true">
+			<arg value="${DEST_EXTERNAL}/external.sh"/>
+			<arg value="--build"/>
+			<arg value="--dir"/>
+			<arg value="${TEST}"/>
+			<arg value="--tag"/>
+			<arg value="${dockerImageTag}"/>
+			<arg value="--version"/>
+			<arg value="${JDK_VERSION}"/>
+			<arg value="--impl"/>
+			<arg value="${JDK_IMPL}"/>
+			<arg value="--base_docker_registry_dir"/>
+			<arg value="ubi8-with-criu/${env.SPEC}-ubi8-criu"/>
+			<arg value="--docker_registry_url"/>
+			<arg value="${env.DOCKER_REGISTRY_URL}"/>
+			<arg value="--docker_args"/>
+			<arg value="${extra_docker_args}"/>
+		</exec>
+	</target>
+
+	<target name="dist" depends="build_image" description="generate the distribution">
 		<copy todir="${DEST}">
 			<fileset dir="${src}" includes="*.xml, *.mk"/>
 		</copy>

diff --git a/external/criu-ubi-portable-checkpoint/test.properties b/external/criu-ubi-portable-checkpoint/test.properties
@@ -2,8 +2,6 @@ github_url="https://github.com/adoptium/aqa-tests.git"
 test_results="testResults"
 gradle_version="5.1"
 environment_variable="MODE=java"
-ubi_packages="git wget xz perl unzip autoconf automake libtool make unzip tzdata openssl curl ca-certificates fontconfig glibc-langpack-en gzip tar iptables-libs jansson libibverbs libmnl libnet libnftnl libpcap nftables protobuf-c"
+ubi_packages="git wget xz perl unzip autoconf automake libtool make unzip"
 ant_version="1.10.5"
 ant_contrib_version="1.0b3"
-criu_version="3.16"
-jdk_install="true"
diff --git a/external/criu-ubi-portable-restore/playlist.xml b/external/criu-ubi-portable-restore/playlist.xml
@@ -20,12 +20,6 @@
 		$(TEST_STATUS); \
 		$(TEST_ROOT)$(D)external$(D)external.sh --clean --tag "${DOCKERIMAGE_TAG}" --dir criu-ubi-portable-restore  --platform "${PLATFORM}" --node_labels "${NODE_LABELS}" --docker_registry_url $(DOCKER_REGISTRY_URL) --docker_registry_dir "$(DOCKER_REGISTRY_DIR)" --criu_default_image_job_name "$(CRIU_DEFAULT_IMAGE_JOB_NAME)"
 		</command>
-		<disables>
-			<disable>
-				<comment>https://github.com/adoptium/aqa-tests/issues/4617</comment>
-				<platform>s390x_linux</platform>
-			</disable>
-		</disables>
 		<features>
 			<feature>CRIU:required</feature>
 		</features>

diff --git a/external/dockerfile_functions.sh b/external/dockerfile_functions.sh
@@ -80,6 +80,7 @@ print_image_args() {
     local vm=$4
     local package=$5
     local build=$6
+    local base_docker_registry_dir="$7"
 
     image_name="eclipse-temurin"
     tag=""
@@ -94,8 +95,9 @@ print_image_args() {
             tag=open-${tag}
         else
             # os is ubi
-            image_name="registry.access.redhat.com/ubi8/ubi"
-            tag="8.6"
+            # temporarily all ubi based testing use internal base image
+            image_name="$DOCKER_REGISTRY_URL/$base_docker_registry_dir"
+            tag="latest"
         fi
     fi
     image="${image_name}:${tag}"
@@ -455,7 +457,7 @@ print_test_files() {
     local test=$2
     local localPropertyFile=$3
 
-    if [[ ${check_external_custom_test} -eq 1 ]]; then 
+    if [[ "$check_external_custom_test" == "1" ]]; then 
         echo -e "# This is the main script to run ${test} tests" \
                 "\nCOPY external_custom/test.sh /test.sh" \
                 "\nCOPY test_base_functions.sh test_base_functions.sh\n" >> ${file}
@@ -561,14 +563,15 @@ generate_dockerfile() {
     package=$6
     build=$7
     platform=$8
-    check_external_custom_test=$9
+    base_docker_registry_dir="$9"
+    check_external_custom_test=$10
 
 
-    if [[ ${check_external_custom_test} -eq 1 ]]; then
+    if [[ "$check_external_custom_test" == "1" ]]; then
         tag_version=${EXTERNAL_REPO_BRANCH}
     fi
 
-    if [[ ${check_external_custom_test} -eq 1 ]]; then
+    if [[ "$check_external_custom_test" == "1" ]]; then
         set_external_custom_test_info ${test} ${check_external_custom_test}
     else
         set_test_info ${test} ${check_external_custom_test}
@@ -581,7 +584,7 @@ generate_dockerfile() {
     echo -n "Writing ${file} ... "
     print_legal ${file};
     print_adopt_test ${file} ${test};
-    print_image_args ${file} ${os} ${version} ${vm} ${package} ${build};
+    print_image_args ${file} ${os} ${version} ${vm} ${package} ${build} "${base_docker_registry_dir}";
     print_result_comment_arg ${file};
     print_test_tag_arg ${file} ${test} ${tag_version};
     print_${os}_pkg ${file} "${!packages}";
@@ -640,7 +643,7 @@ generate_dockerfile() {
     print_clone_project ${file} ${test} ${github_url};
     print_test_files ${file} ${test} ${localPropertyFile};
 
-    if [[ ${check_external_custom_test} -eq 1 ]]; then
+    if [[ "$check_external_custom_test" == "1" ]]; then
         print_external_custom_parameters ${file}
     fi
     print_workdir ${file};

diff --git a/external/external.sh b/external/external.sh
@@ -47,6 +47,7 @@ container_rmi="docker rmi"
 docker_registry_required="false"
 docker_registry_url=""
 docker_registry_dir=""
+base_docker_registry_dir="public"
 reportdst="false"
 reportsrc="false"
 docker_args=""
@@ -56,7 +57,7 @@ imageArg=""
 
 
 usage () {
-	echo 'Usage : external.sh  --dir TESTDIR --tag DOCKERIMAGE_TAG --version JDK_VERSION --impl JDK_IMPL [--docker_os docker_os][--platform PLATFORM] [--portable portable] [--node_name node_name] [--node_labels node_labels] [--docker_registry_required docker_registry_required] [--docker_registry_url DOCKER_REGISTRY_URL] [--docker_registry_dir DOCKER_REGISTRY_DIR] [--mount_jdk mount_jdk] [--test_root TEST_ROOT] [--reportsrc appReportDir] [--reportdst REPORTDIR] [--testtarget target] [--docker_args EXTRA_DOCKER_ARGS] [--build|--run|--load|--clean]'
+	echo 'Usage : external.sh  --dir TESTDIR --tag DOCKERIMAGE_TAG --version JDK_VERSION --impl JDK_IMPL [--docker_os docker_os][--platform PLATFORM] [--portable portable] [--node_name node_name] [--node_labels node_labels] [--docker_registry_required docker_registry_required] [--docker_registry_url DOCKER_REGISTRY_URL] [--docker_registry_dir DOCKER_REGISTRY_DIR] [--base_docker_registry_dir baseDockerRegistryDir] [--mount_jdk mount_jdk] [--test_root TEST_ROOT] [--reportsrc appReportDir] [--reportdst REPORTDIR] [--testtarget target] [--docker_args EXTRA_DOCKER_ARGS] [--build|--run|--load|--clean]'
 }
 
 supported_tests="external_custom aot camel criu-portable-checkpoint  criu-portable-restore criu-ubi-portable-checkpoint criu-ubi-portable-restore derby elasticsearch jacoco jenkins functional-test kafka lucene-solr openliberty-mp-tck payara-mp-tck quarkus quarkus_quickstarts scala system-test tomcat tomee wildfly wycheproof netty spring"
@@ -174,6 +175,14 @@ parseCommandLineArgs() {
 				docker_image_source_job_name=${dir_array[0]}
 				build_number=${dir_array[1]};;
 
+			"--base_docker_registry_dir" )
+				if [ -z "$1" ]; then 
+					base_docker_registry_dir="public";
+				else 
+  					base_docker_registry_dir="$1";
+				fi
+				shift;;
+
 			"--criu_default_image_job_name" )
 				criu_default_image_job_name="$1"; shift;;
 
@@ -262,8 +271,18 @@ parseCommandLineArgs "$@"
 # DOCKER_HOST=$(docker-ip $test-test)
 
 if [ $command_type == "build" ]; then
-	echo "build_image.sh $test $version $impl $docker_os $package $build_type $platform $check_external_custom $imageArg"
-	source $(dirname "$0")/build_image.sh $test $version $impl $docker_os $package $build_type $platform $check_external_custom $imageArg
+	# Temporarily ubi image with criu binary is only available internally
+	if [[ $base_docker_registry_dir != "public" ]]; then
+		echo "Private Docker Registry login starts to obtain base Docker Image:"
+		echo $DOCKER_REGISTRY_CREDENTIALS_PSW | $container_login --username=$DOCKER_REGISTRY_CREDENTIALS_USR --password-stdin $docker_registry_url
+
+		echo "$container_pull $docker_registry_url/$base_docker_registry_dir:latest"
+		$container_pull $docker_registry_url/$base_docker_registry_dir:latest
+
+		$container_logout $docker_registry_url
+	fi
+	echo "build_image.sh $test $version $impl $docker_os $package $build_type $platform $base_docker_registry_dir $check_external_custom $imageArg"
+	source $(dirname "$0")/build_image.sh $test $version $impl $docker_os $package $build_type $platform "$base_docker_registry_dir" $check_external_custom $imageArg
 fi
 
 if [ $command_type == "run" ]; then