Adaptive time-stepping for transient solver using SUNDIALS

CHANGELOG.md

@@ -39,6 +39,9 @@ The format of this changelog is based on
   - Exposed configuration linear solver and eigen solver options for the wave port
     subproblem. These can now be specified as part of the `config["Boundaries"]["WavePort"]`
     configuration. The defaults align with the previously hardcoded values.
+  - Added adaptive time-stepping capability for transient simulations. The new ODE integrators
+    rely on the SUNDIALS library and can be specified by setting the
+    `config["Solver"]["Transient"]["Type"]` option to `"CVODE"` or `"ARKODE"`.
 
 ## [0.13.0] - 2024-05-20
 

CMakeLists.txt

@@ -37,6 +37,8 @@ set(PALACE_WITH_GSLIB ON CACHE BOOL "Build with GSLIB library for high-order fie
 set(PALACE_WITH_STRUMPACK_BUTTERFLYPACK OFF CACHE BOOL "Build with ButterflyPACK support for STRUMPACK solver")
 set(PALACE_WITH_STRUMPACK_ZFP OFF CACHE BOOL "Build with ZFP support for STRUMPACK solver")
 
+set(PALACE_WITH_SUNDIALS ON CACHE BOOL "Build with SUNDIALS differential/algebraic equations solver")
+
 set(ANALYZE_SOURCES_CLANG_TIDY OFF CACHE BOOL "Run static analysis checks using clang-tidy")
 set(ANALYZE_SOURCES_CPPCHECK OFF CACHE BOOL "Run static analysis checks using cppcheck")
 

cmake/ExternalGitTags.cmake

@@ -265,3 +265,17 @@ set(EXTERN_EIGEN_URL
   "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz" CACHE STRING
   "URL for external Eigen build"
 )
+
+# SUNDIALS
+set(EXTERN_SUNDIALS_URL
+  "https://github.com/LLNL/sundials.git" CACHE STRING
+  "URL for external SUNDIALS build"
+)
+set(EXTERN_SUNDIALS_GIT_BRANCH
+  "main" CACHE STRING
+  "Git branch for external SUNDIALS build"
+)
+set(EXTERN_SUNDIALS_GIT_TAG
+  "aaeab8d907c6b7dfca86041401fdc1448f35f826" CACHE STRING
+  "Git tag for external SUNDIALS build"
+)

cmake/ExternalMFEM.cmake

@@ -17,6 +17,9 @@ if(PALACE_BUILD_EXTERNAL_DEPS)
   if(PALACE_WITH_SUPERLU)
     list(APPEND MFEM_DEPENDENCIES superlu_dist)
   endif()
+  if(PALACE_WITH_SUNDIALS)
+    list(APPEND MFEM_DEPENDENCIES sundials)
+  endif()
 else()
   set(MFEM_DEPENDENCIES)
 endif()
@@ -87,6 +90,7 @@ list(APPEND MFEM_OPTIONS
   "-DMFEM_USE_LIBUNWIND=${PALACE_MFEM_WITH_LIBUNWIND}"
   "-DMFEM_USE_METIS_5=YES"
   "-DMFEM_USE_CEED=NO"
+  "-DMFEM_USE_SUNDIALS=${PALACE_WITH_SUNDIALS}"
 )
 if(PALACE_WITH_STRUMPACK OR PALACE_WITH_MUMPS)
   list(APPEND MFEM_OPTIONS
@@ -297,6 +301,32 @@ Intel C++ compiler for MUMPS and STRUMPACK dependencies")
       "-DMUMPS_REQUIRED_LIBRARIES=${SCALAPACK_LIBRARIES}$<SEMICOLON>${STRUMPACK_MUMPS_GFORTRAN_LIBRARY}"
     )
   endif()
+
+  # Configure SUNDIALS
+  if(PALACE_WITH_SUNDIALS)
+    set(SUNDIALS_REQUIRED_PACKAGES "LAPACK" "BLAS" "MPI")
+    if(PALACE_WITH_OPENMP)
+      list(APPEND SUNDIALS_REQUIRED_PACKAGES "OpenMP")
+    endif()
+    if(PALACE_WITH_CUDA)
+      list(APPEND SUNDIALS_REQUIRED_PACKAGES "CUDAToolkit")
+      list(APPEND SUNDIALS_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_CUDA_LIBRARIES})
+    endif()
+    string(REPLACE ";" "$<SEMICOLON>" SUNDIALS_REQUIRED_PACKAGES "${SUNDIALS_REQUIRED_PACKAGES}")
+    string(REPLACE ";" "$<SEMICOLON>" SUNDIALS_REQUIRED_LIBRARIES "${SUNDIALS_REQUIRED_LIBRARIES}")
+    list(APPEND MFEM_OPTIONS
+      "-DSUNDIALS_DIR=${CMAKE_INSTALL_PREFIX}"
+      "-DSUNDIALS_OPT=-I${CMAKE_INSTALL_PREFIX}/include"
+      "-DSUNDIALS_LIB=-Wl,-rpath,${CMAKE_INSTALL_PREFIX}/lib64 -L${CMAKE_INSTALL_PREFIX}/lib64 -lsundials_arkode, -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol -lsundials_nvecparhyp -lsundials_nvecparallel"
+      "-DSUNDIALS_REQUIRED_PACKAGES=${SUNDIALS_REQUIRED_PACKAGES}"
+    )
+    if(NOT "${SUNDIALS_REQUIRED_LIBRARIES}" STREQUAL "")
+      list(APPEND MFEM_OPTIONS
+        "-DSUNDIALS_REQUIRED_LIBRARIES=${SUNDIALS_REQUIRED_LIBRARIES}"
+      )
+    endif()
+  endif()
+
 else()
   # Help find dependencies for the internal MFEM build
   # If we trust MFEM's Find<PACKAGE>.cmake module, we can just set <PACKAGE>_DIR and, if
@@ -309,6 +339,7 @@ else()
     "SuperLUDist"
     "STRUMPACK"
     "MUMPS"
+    "SUNDIALS"
   )
   foreach(DEP IN LISTS PALACE_MFEM_DEPS)
     set(${DEP}_DIR "" CACHE STRING "Path to ${DEP} build or installation directory")

cmake/ExternalSUNDIALS.cmake

@@ -0,0 +1,73 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+#
+# Build SUNDIALS
+#
+
+# Force build order
+set(SUNDIALS_DEPENDENCIES )
+
+set(SUNDIALS_OPTIONS ${PALACE_SUPERBUILD_DEFAULT_ARGS})
+list(APPEND SUNDIALS_OPTIONS
+  "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+  "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
+  "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
+  "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}"
+  "-DEXAMPLES_ENABLE_C=OFF"
+  "-DEXAMPLES_ENABLE_CXX=OFF"
+  "-DEXAMPLES_ENABLE_CUDA=OFF"
+  "-DEXAMPLES_INSTALL=OFF"
+  "-DENABLE_MPI=ON"
+)
+
+if(PALACE_WITH_OPENMP)
+  list(APPEND SUNDIALS_OPTIONS
+    "-DENABLE_OPENMP=ON"
+  )
+else()
+  list(APPEND SUNDIALS_OPTIONS
+    "-DENABLE_OPENMP=OFF"
+  )
+endif()
+
+# Configure LAPACK dependency
+if(NOT "${BLAS_LAPACK_LIBRARIES}" STREQUAL "")
+  list(APPEND SUNDIALS_OPTIONS
+    "-DENABLE_LAPACK=ON"
+    "-DLAPACK_LIBRARIES=${BLAS_LAPACK_LIBRARIES}"
+    "-DLAPACK_WORKS:BOOL=TRUE"
+  )
+endif()
+
+if(PALACE_WITH_CUDA)
+  list(APPEND SUNDIALS_OPTIONS
+    "-DENABLE_CUDA=ON"
+  )
+endif()
+
+if(PALACE_WITH_MAGMA)
+  list(APPEND SUNDIALS_OPTIONS
+    "-DENABLE_MAGMA=ON"
+    "-DMAGMA_DIR=${MAGMA_DIR}"
+  )
+endif()
+
+
+string(REPLACE ";" "; " SUNDIALS_OPTIONS_PRINT "${SUNDIALS_OPTIONS}")
+message(STATUS "SUNDIALS_OPTIONS: ${SUNDIALS_OPTIONS_PRINT}")
+
+
+include(ExternalProject)
+ExternalProject_Add(sundials
+  DEPENDS           ${SUNDIALS_DEPENDENCIES}
+  GIT_REPOSITORY    ${EXTERN_SUNDIALS_URL}
+  GIT_TAG           ${EXTERN_SUNDIALS_GIT_TAG}
+  SOURCE_DIR        ${CMAKE_BINARY_DIR}/extern/sundials
+  BINARY_DIR        ${CMAKE_BINARY_DIR}/extern/sundials-build
+  INSTALL_DIR       ${CMAKE_INSTALL_PREFIX}
+  PREFIX            ${CMAKE_BINARY_DIR}/extern/sundials-Cmake
+  UPDATE_COMMAND    ""
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} <SOURCE_DIR> "${SUNDIALS_OPTIONS}"
+  TEST_COMMAND      ""
+)

docs/src/config/solver.md

@@ -208,7 +208,10 @@ error tolerance.
     "ExcitationWidth": <float>,
     "MaxTime": <float>,
     "TimeStep": <float>,
-    "SaveStep": <int>
+    "SaveStep": <int>,
+    "Order": <int>,
+    "RelTol": <float>,
+    "AbsTol": <float>
 }
 ```
 
@@ -218,14 +221,13 @@ with
 the second-order system of differential equations. The available options are:
 
   - `"GeneralizedAlpha"` :  The second-order implicit generalized-``\alpha`` method with
-    ``\rho_\inf = 1.0``. This scheme is unconditionally stable.
-  - `"NewmarkBeta"` :  The second-order implicit Newmark-``\beta`` method with
-    ``\beta = 1/4`` and ``\gamma = 1/2``. This scheme is unconditionally stable.
-  - `"CentralDifference"` :  The second-order explicit central difference method, obtained
-    by setting ``\beta = 0`` and ``\gamma = 1/2`` in the Newmark-``\beta`` method. In this
-    case, the maximum eigenvalue of the system operator is estimated at the start of the
-    simulation and used to restrict the simulation time step to below the maximum stability
-    time step.
+    ``\rho_{\inf} = 1.0``. This scheme is unconditionally stable.
+  - `"ARKODE"` :  SUNDIALS ARKode implicit Runge-Kutta scheme applied to the first-order
+    ODE system for the electric field with adaptive time-stepping. This option is only available when *Palace* has been [built with SUNDIALS support](../install.md#Configuration-options).
+  - `"CVODE"` :  SUNDIALS CVODE implicit multistep method scheme applied to the first-order
+    ODE system for the electric field with adaptive time-stepping. This option is only available when *Palace* has been [built with SUNDIALS support](../install.md#Configuration-options).
+  - `"RungeKutta"` : SUNDIALS ARKode implicit Runge-Kutta scheme applied to the first-order
+    ODE system for the electric field with fixed time-stepping. This option is only available when *Palace* has been [built with SUNDIALS support](../install.md#Configuration-options).
   - `"Default"` :  Use the default `"GeneralizedAlpha"` time integration scheme.
 
 `"Excitation" [None]` :  Controls the time dependence of the source excitation. The
@@ -260,6 +262,15 @@ disk for [visualization with ParaView](../guide/postprocessing.md#Visualization)
 saved in the `paraview/` directory under the directory specified by
 [`config["Problem"]["Output"]`](problem.md#config%5B%22Problem%22%5D).
 
+`"Order" [2]` :  Order of the Runge-Kutta integrators or maximum order of the multistep method.
+Only relevant when `"Type"` is `"ARKODE"`, `"Runge-Kutta"` or `"CVODE"`.
+
+`"RelTol" [1e-4]` :  Relative tolerance used in adaptive time-stepping schemes. Only relevant
+ when `"Type"` is `"CVODE"` or `"ARKODE"` .
+
+`"AbsTol" [1e-9]` :  Absolute tolerance used in adaptive time-stepping schemes. Only relevant
+ when `"Type"` is `"CVODE"` or `"ARKODE"`.
+
 ## `solver["Electrostatic"]`
 
 ```json

docs/src/install.md

@@ -133,6 +133,7 @@ Additional build options are (with default values in brackets):
   - `PALACE_WITH_LIBXSMM [ON]` :  Build with LIBXSMM backend for libCEED
   - `PALACE_WITH_MAGMA [ON]` :  Build with MAGMA backend for libCEED
   - `PALACE_WITH_GSLIB [ON]` :  Build with GSLIB library for high-order field interpolation
+  - `PALACE_WITH_SUNDIALS [ON]` : Build with SUNDIALS ODE solver library
 
 The build step is invoked by running (for example with 4 `make` threads)
 
@@ -155,7 +156,7 @@ and LAPACK libraries depending on the system architecture according to the follo
 procedure:
 
   - For `x86_64` systems:
-    
+
       + If the `MKLROOT` environment variable is set, looks for an
         [Intel MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html)
         installation.
@@ -166,7 +167,7 @@ procedure:
         which is permissively licensed and available from most package managers.
 
   - For `aarch64`/`arm64` systems:
-    
+
       + If the `ARMPL_DIR` environment variable is set, looks for an
         [Arm Performance Libraries (PL)](https://www.arm.com/products/development-tools/server-and-hpc/allinea-studio/performance-libraries)
         installation.

extern/CMakeLists.txt

@@ -72,3 +72,9 @@ include(ExternalFmt)
 # Add Eigen
 message(STATUS "===================== Configuring Eigen dependency =====================")
 include(ExternalEigen)
+
+# Add SUNDIALS
+if(PALACE_WITH_SUNDIALS)
+  message(STATUS "===================== Configuring SUNDIALS dependency =====================")
+  include(ExternalSUNDIALS)
+endif()

palace/drivers/transientsolver.cpp

@@ -30,6 +30,7 @@ TransientSolver::Solve(const std::vector<std::unique_ptr<Mesh>> &mesh) const
   std::function<double(double)> dJdt_coef = GetTimeExcitation(true);
   SpaceOperator space_op(iodata, mesh);
   TimeOperator time_op(iodata, space_op, dJdt_coef);
+
   double delta_t = iodata.solver.transient.delta_t;
   if (time_op.isExplicit())
   {
@@ -101,7 +102,7 @@ TransientSolver::Solve(const std::vector<std::unique_ptr<Mesh>> &mesh) const
     {
       Mpi::Print("\n");
       t += delta_t;
-      time_op.Init();  // Initial conditions
+      time_op.Init(delta_t);  // Initial conditions
     }
     else
     {
@@ -138,6 +139,7 @@ TransientSolver::Solve(const std::vector<std::unique_ptr<Mesh>> &mesh) const
     step++;
   }
   BlockTimer bt1(Timer::POSTPRO);
+  time_op.PrintStats();
   SaveMetadata(time_op.GetLinearSolver());
   return {indicator, space_op.GlobalTrueVSize()};
 }