Add HeCBench, GAMESS, libCEED to CI Flow

.github/workflows/check-unit-tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   unit-tests-llvm-15-debug:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -22,7 +22,7 @@ jobs:
         run: ./scripts/unit_tests.sh debug llvm-15
         shell: bash
   unit-tests-llvm-15-release:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -32,7 +32,7 @@ jobs:
         run: ./scripts/unit_tests.sh release llvm-15
         shell: bash
   unit-tests-llvm-16-debug:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -42,7 +42,7 @@ jobs:
         run: ./scripts/unit_tests.sh debug llvm-16
         shell: bash
   unit-tests-llvm-16-release:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -52,7 +52,7 @@ jobs:
         run: ./scripts/unit_tests.sh release llvm-16
         shell: bash
   unit-tests-llvm-17-debug:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -62,7 +62,7 @@ jobs:
         run: ./scripts/unit_tests.sh debug llvm-17
         shell: bash
   unit-tests-llvm-17-release:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -72,7 +72,7 @@ jobs:
         run: ./scripts/unit_tests.sh release llvm-17
         shell: bash
   unit-tests-llvm-18-debug:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
@@ -82,12 +82,36 @@ jobs:
         run: ./scripts/unit_tests.sh debug llvm-18
         shell: bash
   unit-tests-llvm-18-release:
-    runs-on: self-hosted
+    runs-on: X64
     steps:
       - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
       - name: Run unit test checking script
         run: ./scripts/unit_tests.sh release llvm-18
-        shell: bash
+        shell: bash
+  test-gamess:
+    runs-on: X64
+    needs: []
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - name: Run GAMESS test script
+        run: ./scripts/test_gamess.sh
+        shell: bash
+  test-libceed:
+    runs-on: X64
+    needs: []
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - name: Run LibCEED test script
+        run: ./scripts/test_libceed.sh
+        shell: bash
+
+        # needs: [unit-tests-llvm-15-debug, unit-tests-llvm-15-release, unit-tests-llvm-16-debug, unit-tests-llvm-16-release, unit-tests-llvm-17-debug, unit-tests-llvm-17-release, unit-tests-llvm-18-debug, unit-tests-llvm-18-release]

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "spirv-extractor/SPIRV-Tools"]
 	path = spirv-extractor/SPIRV-Tools
 	url = https://github.com/KhronosGroup/SPIRV-Tools.git
+[submodule "HeCBench"]
+	path = HeCBench
+	url = https://github.com/franz/HeCBench.git

scripts/compile_libceed.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 source /etc/profile.d/modules.sh &> /dev/null
+module use ~/modulefiles
 HIP_DIR=$1
 echo "Building LibCEED with HIP_DIR: ${HIP_DIR}"
 if which clang >/dev/null 2>&1; then

scripts/test_gamess.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -ex
+
+source /etc/profile.d/modules.sh &> /dev/null
+module use ~/modulefiles
+
+cd /home/pvelesko/GAMESS/gamess_libcchem_hip
+
+export IGC_EnableDPEmulation=1
+export OverrideDefaultFP64Settings=1
+
+module purge
+module load llvm/18.0
+module load HIP/chipStar/testing
+module load hdf5/1.14.4.2
+module load openmpi/4.1.1
+module load HipSolver/2024.05.15-807efe2 
+
+export JSON_ROOT=$PWD/../json
+export MATHLIB_ROOT=$HIP_DIR
+export GPU_BOARD=Intel
+export OMP_NUM_THREADS=1
+
+export MPI_CXX=mpic++
+rm -rf build && mkdir build && cd build
+cmake -DMPI_ROOT=$MPI_ROOT   -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++  -DJSON_ROOT=$JSON_ROOT -DHDF5_ROOT=$HDF5_ROOT  -DHIP=True -DMATHLIB_ROOT=$MATHLIB_ROOT -DHIPSOLVER_ROOT=$HIPSOLVER_ROOT -DBUILD_RIMP2=0 -DMAGMA=False -DGPU_BOARD=$GPU_BOARD -DMKLROOT=$MKLROOT ../
+make -j $(nproc)
+cd ..
+
+export IGC_ForceOCLSIMDWidth=16
+#export ZE_AFFINITY_MASK=0.0
+
+orterun --mca btl vader,self,tcp -np 2 --bind-to core --map-by ppr:2:node ./build/exess ./inputs/json_inputs/scf/w1.json 2>&1 | tee gamess_output.txt
+
+# Extract the floating point number from the output file
+energy=$(grep "Final energy is:" gamess_output.txt | awk '{print $4}')
+correct_energy=-74.9612532341291
+
+# Define the tolerance value
+tol=0.0001
+
+# Calculate the absolute difference between the energy and the correct energy
+diff=$(echo "$energy - $correct_energy" | bc | tr -d -)
+
+# Compare the absolute difference with the tolerance value
+comparison=$(echo "$diff < $tol" | bc)
+
+if [ $comparison -eq 1 ]
+then
+    echo "The energy value is within the tolerance."
+    exit 0
+else
+    echo "The energy value is not within the tolerance."
+    exit 1
+fi

scripts/test_libceed.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+source /etc/profile.d/modules.sh &> /dev/null
+module use ~/modulefiles
+
+export IGC_EnableDPEmulation=1
+export OverrideDefaultFP64Settings=1
+export CHIP_LOGLEVEL=err
+
+module purge
+module load llvm/18.0
+module load HIP/chipStar/testing
+
+cd /home/pvelesko/libCEED
+git co HEAD -f
+git pull
+make clean
+
+make FC= CC=hipcc CXX=hipcc BACKENDS="/gpu/hip/ref /gpu/hip/shared /gpu/hip/gen" -j $(nproc) test | tee libceed_output.txt
+if grep -q "not ok" "libceed_output.txt"; then
+  echo "FAIL"
+  awk '/Test Summary Report/,EOF' "libceed_output.txt"
+  exit 1
+else
+  echo "PASS"
+  exit 0
+fi

scripts/unit_tests.sh

@@ -144,45 +144,34 @@ else
   mkdir build
   cd build
 
+  export CHIPSTAR_INSTALL_DIR=/home/pvelesko/install/HIP/chipStar/testing # set CHIPSTAR_INSTALL_DIR to current build dir
   echo "building with $CLANG"
-  cmake ../ -DCMAKE_BUILD_TYPE="$build_type" -DCHIP_BUILD_HIPBLAS=ON
+  cmake ../ -DCMAKE_BUILD_TYPE="$build_type" -DCHIP_BUILD_HIPBLAS=ON -DCMAKE_INSTALL_PREFIX=${CHIPSTAR_INSTALL_DIR}
   make all build_tests install -j $(nproc) #&> /dev/null
   echo "chipStar build complete." 
 fi
-
 module unload opencl/dgpu
 
-# module load HIP/hipBLAS/main/release # for libCEED NOTE: Must be after build step otherwise it will cause link issues.
-
 # Test Level Zero iGPU
 echo "begin igpu_level0_failed_tests"
-# module load level-zero/igpu
-# module list
 ../scripts/check.py ./ igpu level0 --num-threads=${num_threads} --timeout=$timeout --num-tries=$num_tries --modules=on | tee igpu_level0_make_check_result.txt
 # module unload level-zero/igpu
 echo "end igpu_level0_failed_tests"
 
 # Test Level Zero dGPU
 echo "begin dgpu_level0_failed_tests"
-# module load level-zero/dgpu
-# module list
 ../scripts/check.py ./ dgpu level0 --num-threads=${num_threads} --timeout=$timeout --num-tries=$num_tries --modules=on | tee dgpu_level0_make_check_result.txt
 # module unload level-zero/dgpu
 echo "end dgpu_level0_failed_tests"
 
 # Test OpenCL iGPU
 echo "begin igpu_opencl_failed_tests"
-# module load opencl/igpu
-# module list
 ../scripts/check.py ./ igpu opencl --num-threads=${num_threads} --timeout=$timeout --num-tries=$num_tries --modules=on | tee igpu_opencl_make_check_result.txt
 # module unload opencl/igpu
 echo "end igpu_opencl_failed_tests"
 
 # Test OpenCL dGPU
 echo "begin dgpu_opencl_failed_tests"
-# module load intel/opencl # sets ICD
-# module load opencl/dgpu # sets CHIP_BE
-# module list
 ../scripts/check.py ./ dgpu opencl --num-threads=${num_threads} --timeout=$timeout --num-tries=$num_tries --modules=on | tee dgpu_opencl_make_check_result.txt
 # module unload opencl/dgpu intel/opencl
 echo "end dgpu_opencl_failed_tests"
@@ -198,7 +187,6 @@ function check_tests {
     return 1
   fi
 }
-
 overall_status=0
 set +e
 echo "RESULTS:"

tests/CMakeLists.txt

@@ -35,3 +35,4 @@ add_test(
     -P ${CMAKE_CURRENT_SOURCE_DIR}/run_helgrind_test.cmake
 )
 
+include(HeCBench.cmake)