Merge pull request #476 from ValeevGroup/evaleev/fix/host-allocator-i…

…s-serializable

host allocator is serializable

.github/workflows/ci.yml

@@ -12,16 +12,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os : [ macos-latest ]
-        cxx : [ clang++, /opt/homebrew/bin/g++-11 ]
+        os : [ macos-latest, ubuntu-22.04 ]
         build_type : [ Release, Debug ]
         task_backend: [ Pthreads, PaRSEC ]
-        prerequisites : [ gcc@11 boost eigen open-mpi bison scalapack ]
+        include:
+          - os: ubuntu-22.04
+            cc: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
+          - os: macos-latest
+            cc: clang
+            cxx: clang++
 
     name: "${{ matrix.os }}: ${{ matrix.cxx }} ${{ matrix.build_type }} ${{ matrix.task_backend }}"
     runs-on: ${{ matrix.os }}
     env:
       CXX : ${{ matrix.cxx }}
+      CCACHE_DIR : ${{github.workspace}}/build/.ccache
+      CCACHE_COMPRESS : true
+      CCACHE_COMPRESSLEVEL : 6
+      OMPI_MCA_btl_vader_single_copy_mechanism : none
+      PARSEC_MCA_runtime_bind_threads : 0
       BUILD_CONFIG : >
         -DMADNESS_TASK_BACKEND=${{ matrix.task_backend }}
         -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
@@ -33,31 +43,53 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - uses: maxim-lobanov/setup-xcode@v1
-      with:
-        xcode-version: 'latest-stable'
-
     - name: Host system info
       shell: bash
       run: cmake -P ${{github.workspace}}/ci/host_system_info.cmake
 
-    - name: Install ${{matrix.prerequisites}}
+
+    - name: Install prerequisite MacOS packages
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        brew install ninja boost eigen open-mpi bison scalapack ccache
+        echo "MPIEXEC=/opt/homebrew/bin/mpiexec" >> $GITHUB_ENV
+
+    - name: Install prerequisites Ubuntu packages
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
-        brew install ${{matrix.prerequisites}}
-        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+        wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+        sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
+        sudo apt-get update
+        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libboost-random-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison libscalapack-mpi-dev cmake doxygen
+        sudo ln -s /usr/lib/x86_64-linux-gnu/libscalapack-openmpi.so /usr/lib/x86_64-linux-gnu/libscalapack.so
+        echo "MPIEXEC=/usr/bin/mpiexec" >> $GITHUB_ENV
+
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+
+    - name: Setup ccache cache files
+      uses: actions/[email protected]
+      with:
+        path: ${{github.workspace}}/build/.ccache
+        key: ${{ matrix.config.name }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+        restore-keys: |
+          ${{ matrix.config.name }}-ccache-
 
     - name: "Configure build: ${{ env.BUILD_CONFIG }}"
       shell: bash
       run: |
         set -x;
-        cmake -B${{github.workspace}}/build $BUILD_CONFIG || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log)
+        cmake -B${{github.workspace}}/build $BUILD_CONFIG || (cat CMakeFiles/CMakeConfigureLog.yaml)
 
     - name: Build
       working-directory: ${{github.workspace}}/build
       shell: bash
       run: |
-        cmake --build . --target tiledarray
-        cmake --build . --target examples
+        ccache -p && ccache -z && cmake --build . --target tiledarray && cmake --build . --target examples && ccache -s
 
     - name: Test
       working-directory: ${{github.workspace}}/build

.gitlab-ci.yml

@@ -57,22 +57,10 @@ ubuntu:
       metrics: build/metrics.txt
   parallel:
     matrix:
-      - IMAGE : [ "ubuntu:20.04" ]
-        CXX: [ g++ ]
-        BUILD_TYPE : [ "RelWithDebInfo" ]
-        BLA_VENDOR : [ "BLAS_PREFERENCE_LIST=IntelMKL" ]
-        BLA_THREADS : [ "IntelMKL_THREAD_LAYER=tbb" ]
-        # ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
-        TA_PYTHON : [ "TA_PYTHON=OFF" ] # needs to be fixed for MKL
-        RUNNER_TAGS: [ saas-linux-small-amd64 ]
-      - IMAGE : [ "ubuntu:22.04" ]
-        CXX: [ g++, clang++-13 ]
-        BUILD_TYPE : [ "RelWithDebInfo" ]
-        ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
-        RUNNER_TAGS: [ saas-linux-small-amd64 ]
       - IMAGE : [ "ubuntu:22.04" ]
         CXX: [ g++ ]
         BUILD_TYPE : [ "RelWithDebInfo" ]
+        TA_PYTHON : [ "TA_PYTHON=OFF" ]
         ENABLE_CUDA : [ "ENABLE_CUDA=ON" ]
         TA_TARGETS : [ "tiledarray examples-tiledarray check_serial-tiledarray" ]
         RUNNER_TAGS: [ cuda ]

src/CMakeLists.txt

@@ -133,8 +133,8 @@ TiledArray/expressions/index_list.h
 TiledArray/external/btas.h
 TiledArray/external/madness.h
 TiledArray/external/umpire.h
+TiledArray/host/env.cpp
 TiledArray/host/env.h
-TiledArray/host/allocator.h
 TiledArray/math/blas.h
 TiledArray/math/gemm_helper.h
 TiledArray/math/outer.h
@@ -207,11 +207,7 @@ TiledArray/util/vector.h
 if(HIP_FOUND OR CUDA_FOUND)
   list(APPEND TILEDARRAY_HEADER_FILES
           TiledArray/external/device.h
-          TiledArray/external/librett.h)
-endif()
-
-if(CUDA_FOUND OR HIP_FOUND)
-  list(APPEND TILEDARRAY_HEADER_FILES
+          TiledArray/external/librett.h
           TiledArray/device/blas.cpp
           TiledArray/device/blas.h
           TiledArray/device/btas.h
@@ -223,7 +219,6 @@ if(CUDA_FOUND OR HIP_FOUND)
           TiledArray/device/kernel/thrust/reduce_kernel.h
           TiledArray/device/platform.h
           TiledArray/device/thrust.h
-          TiledArray/device/allocators.h
           TiledArray/device/um_storage.h)
   if(CUDA_FOUND)
     list(APPEND TILEDARRAY_HEADER_FILES

src/TiledArray/device/um_storage.cu

@@ -22,7 +22,7 @@
  */
 
 
-#include <TiledArray/device/allocators.h>
+#include <TiledArray/external/device.h>
 #include <TiledArray/device/thrust.h>
 
 #ifdef TILEDARRAY_HAS_CUDA

src/TiledArray/device/um_storage.h

@@ -24,7 +24,7 @@
 #ifndef TILEDARRAY_DEVICE_UM_VECTOR_H__INCLUDED
 #define TILEDARRAY_DEVICE_UM_VECTOR_H__INCLUDED
 
-#include <TiledArray/device/allocators.h>
+#include <TiledArray/external/device.h>
 
 #ifdef TILEDARRAY_HAS_DEVICE
 

src/TiledArray/external/device.h

@@ -41,8 +41,6 @@
 #include <thrust/system_error.h>
 #endif
 
-#include <TiledArray/external/umpire.h>
-
 #include <TiledArray/external/madness.h>
 #include <madness/world/print.h>
 #include <madness/world/safempi.h>
@@ -51,6 +49,20 @@
 #include <TiledArray/error.h>
 #include <TiledArray/initialize.h>
 
+#include <TiledArray/external/umpire.h>
+
+namespace TiledArray::detail {
+
+struct get_um_allocator {
+  inline umpire::Allocator& operator()();
+};
+
+struct get_pinned_allocator {
+  inline umpire::Allocator& operator()();
+};
+
+}  // namespace TiledArray::detail
+
 #if defined(TILEDARRAY_HAS_CUDA)
 
 inline void __DeviceSafeCall(cudaError err, const char* file, const int line) {
@@ -798,9 +810,10 @@ class Env {
     static std::unique_ptr<Env> instance_{nullptr};
     return instance_;
   }
-};
+};  // class Env
 
 namespace detail {
+
 // in a madness device task point to its local optional stream to use by
 // madness_task_stream_opt; set to nullptr after task callable finished
 inline std::optional<Stream>*& madness_task_stream_opt_ptr_accessor() {
@@ -892,6 +905,18 @@ device::Stream stream_for(const Range& range) {
 
 }  // namespace device
 
+namespace detail {
+
+inline umpire::Allocator& get_um_allocator::operator()() {
+  return deviceEnv::instance()->um_allocator();
+}
+
+inline umpire::Allocator& get_pinned_allocator::operator()() {
+  return deviceEnv::instance()->pinned_allocator();
+}
+
+}  // namespace detail
+
 #endif  // TILEDARRAY_HAS_DEVICE
 
 #ifdef TILEDARRAY_HAS_CUDA