Support multi-threaded H5VL operations

.github/workflows/multithread_test.yml

@@ -0,0 +1,103 @@
+name: Build and Test Multithreaded HDF5 Library
+
+on:
+    push:
+        branches:
+            - '*'
+
+env:
+  HDF5TestExpress: 1
+
+jobs:
+    build-and-test:
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                api_tests: ["--enable-api-tests", "--disable-api-tests"]
+                multi_thread: ["--enable-multithread --disable-hl", "--disable-multithread --enable-threadsafe --disable-hl", "--disable-multithread --disable-threadsafe"]
+                build_mode: ["production", "debug"]
+            fail-fast: false
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Set up autotools
+              run: |
+                  sudo apt install automake autoconf libtool libtool-bin
+                  sudo apt install libaec0 libaec-dev valgrind
+
+            - name: Configure HDF5
+              working-directory: ${{github.workspace}}
+              run: |
+                    sh ./autogen.sh
+                    ./configure --enable-shared --enable-tests ${{ matrix.multi_thread }} ${{ matrix.api_tests }} --enable-build-mode=${{ matrix.build_mode }}
+
+            - name: Build HDF5
+              working-directory: ${{github.workspace}}
+              run: |
+                    make -j
+
+            - name: API Tests - Native (Single thread)
+              if: matrix.api_tests == '--enable-api-tests'
+              working-directory: ${{github.workspace}}
+              run: |
+                valgrind ./test/API/api_tests -maxthreads 1
+
+            - name: API Tests (Multi thread)
+              if: matrix.api_tests == '--enable-api-tests' && matrix.multi_thread == '--enable-multithread --disable-hl'
+              working-directory: ${{github.workspace}}
+              run: |
+                valgrind ./test/API/api_tests -maxthreads 16
+
+            - name: API Tests - MT Native Wrapper (Single thread)
+              if: matrix.api_tests == '--enable-api-tests' && matrix.multi_thread == '--enable-multithread --disable-hl'
+              working-directory: ${{github.workspace}}
+              env:
+                HDF5_PLUGIN_PATH: ${{github.workspace}}/test/.libs
+                HDF5_VOL_CONNECTOR: "mt_native_wrapper_vol_connector"
+              run: |
+                valgrind ./test/API/api_tests -maxthreads 1
+
+            - name: API Tests - MT Native Wrapper (Multi thread)
+              if: matrix.api_tests == '--enable-api-tests' && matrix.multi_thread == '--enable-multithread --disable-hl'
+              working-directory: ${{github.workspace}}
+              env:
+                HDF5_PLUGIN_PATH: ${{github.workspace}}/test/.libs
+                HDF5_VOL_CONNECTOR: "mt_native_wrapper_vol_connector"
+              run: |
+                valgrind ./test/API/api_tests -maxthreads 16
+
+            - name: API Tests - MT Passthru Wrapper (Single thread)
+              if: matrix.api_tests == '--enable-api-tests' && matrix.multi_thread == '--enable-multithread --disable-hl'
+              working-directory: ${{github.workspace}}
+              env:
+                HDF5_PLUGIN_PATH: ${{github.workspace}}/test/.libs
+                HDF5_VOL_CONNECTOR: "mt_passthru_wrapper_vol_connector under_vol=0\\;under_info={}"
+              run: |
+                valgrind ./test/API/api_tests -maxthreads 1
+
+            - name: API Tests - MT Passthru Wrapper (Multi thread)
+              if: matrix.api_tests == '--enable-api-tests' && matrix.multi_thread == '--enable-multithread --disable-hl'
+              working-directory: ${{github.workspace}}
+              env:
+                HDF5_PLUGIN_PATH: ${{github.workspace}}/test/.libs
+                HDF5_VOL_CONNECTOR: "mt_passthru_wrapper_vol_connector under_vol=0\\;under_info={}"
+              run: |
+                valgrind ./test/API/api_tests -maxthreads 16
+
+            - name: MT VL Tests (Single thread)
+              working-directory: ${{github.workspace}}/test/
+              run: |
+                valgrind ./threads/testmthdf5 -maxthreads 1
+
+            - name: MT VL Tests (Multi thread)
+              if: matrix.multi_thread == '--enable-multithread --disable-hl'
+              working-directory: ${{github.workspace}}/test/
+              run: |
+                valgrind ./threads/testmthdf5 -maxthreads 16
+
+            - name: Other HDF5 Tests
+              working-directory: ${{github.workspace}}
+              run: |
+                make check

README.md

@@ -1,3 +1,5 @@
+HDF5 version 1.14.2 released on 2023-08-11
+
 # Experimental
 
 This branch contains modified source code of HDF5 r1.14.2. The code will be used to

configure.ac

@@ -274,6 +274,14 @@ AC_ARG_ENABLE([warnings-as-errors],
                                ])],
               [WARNINGS_AS_ERRORS=$enableval])
 
+## ----------------------------------------------------------------------
+## Check if we should build the HDF5 API tests
+AC_ARG_ENABLE([api-tests],
+    AS_HELP_STRING([--enable-api-tests], [Enable building of API tests]),
+    [enable_api_tests=$enableval], [enable_api_tests=no])
+
+
+
 ## Set default
 if test "X-$WARNINGS_AS_ERRORS" = X- ; then
   WARNINGS_AS_ERRORS=no
@@ -1655,6 +1663,60 @@ if test "x$HAVE_ZLIB" = "xyes" -a "x$HAVE_ZLIB_H" = "xyes" -a "x$HAVE_COMPRESS2"
     EXTERNAL_FILTERS="${EXTERNAL_FILTERS}deflate(zlib)"
 fi
 
+## TODO - Probably this should be reworked or moved somewhere else
+
+# If the provided name corresponds to a non-empty variable,
+# create a macro with the given name and value. 
+# If the provided name does not exist, create a macro with the 
+# default value if one was provided, otherwise undefine the macro.
+m4_define([EXPORT_VARIABLE],
+  [if test -n "$$1"; then
+    AC_SUBST([$1], ["#define $1 $$1"])
+   else
+     if test -n "$2"; then
+       AC_SUBST([$1], ["#define $1 $2"])
+     else
+       AC_SUBST([$1], ["/*undef $1 */"])
+     fi
+   fi]
+)
+
+EXPORT_VARIABLE([H5_API_TEST_CONFIG_H], ["1"])
+
+# TODO: These flags are copied from a CMake equivalent H5_api_test_config.h.in 
+# and I haven't verified that the autotools define them
+EXPORT_VARIABLE([H5_API_TEST_HAVE_ASYNC], [])
+if test -n "$H5_HAVE_PARALLEL"; then 
+  EXPORT_VARIABLE([MPIEXEC_EXECUTABLE], [])
+  EXPORT_VARIABLE([MPIEXEC], [])
+
+  EXPORT_VARIABLE(MPIEXEC_NUMPROC_FLAG, [])
+  EXPORT_VARIABLE(MPIEXEC_PREFLAGS, [])
+  EXPORT_VARIABLE(MPIEXEC_POSTFLAGS, [])
+
+  EXPORT_VARIABLE(MPIEXEC_SERVER_PREFLAGS, [])
+  EXPORT_VARIABLE(MPIEXEC_SERVER_POSTFLAGS, [])
+  EXPORT_VARIABLE(MPIEXEC_MAX_NUMPROCS, [])
+fi
+
+EXPORT_VARIABLE([DART_TESTING_TIMEOUT], ["1500"])
+
+
+EXPORT_VARIABLE([H5_API_TEST_ENV_VARS], [])
+EXPORT_VARIABLE([H5_API_TEST_INIT_COMMAND], [])
+
+EXPORT_VARIABLE([H5_API_TEST_SERVER_START_MSG], [])
+EXPORT_VARIABLE([H5_API_TEST_SERVER_EXIT_COMMAND], [])
+
+EXPORT_VARIABLE([H5_API_TEST_CLIENT_HELPER_START_MSG], [])
+EXPORT_VARIABLE([H5_API_TEST_CLIENT_HELPER_EXIT_COMMAND], [])
+
+EXPORT_VARIABLE([H5_API_TEST_CLIENT_INIT_TOKEN_REGEX], [])
+EXPORT_VARIABLE([H5_API_TEST_CLIENT_INIT_TOKEN_VAR], [])
+
+
+
+
 
 ## ----------------------------------------------------------------------
 ## Is the szlib present? It has a header file `szlib.h' and a library
@@ -4159,7 +4221,7 @@ AM_CONDITIONAL([BUILD_TESTS_PARALLEL_CONDITIONAL], [test -n "$TESTPARALLEL"])
 AM_CONDITIONAL([BUILD_TOOLS_CONDITIONAL], [test "X$HDF5_TOOLS" = "Xyes"])
 AM_CONDITIONAL([BUILD_TOOLS_HL_GIF_CONDITIONAL], [test "X$HDF5_HL_GIF_TOOLS" = "Xyes"])
 AM_CONDITIONAL([BUILD_DOXYGEN_CONDITIONAL], [test "X$HDF5_DOXYGEN" = "Xyes"])
-
+AM_CONDITIONAL([BUILD_API_TESTS], [test "x$enable_api_tests" = "xyes"])
 ## ----------------------------------------------------------------------
 ## Build the Makefiles.
 ##
@@ -4281,6 +4343,10 @@ AC_CONFIG_FILES([Makefile
                  test/test_use_cases.sh
                  test/test_vds_env.sh
                  test/test_vds_swmr.sh
+                 test/threads/Makefile
+                 test/threads/unit/Makefile
+                 test/API/Makefile
+                 test/API/H5_api_test_config.h:test/API/H5_api_test_config_at.h.in
                  testpar/Makefile
                  testpar/testpflush.sh
                  utils/Makefile