Expand Linux test matrix: (#4454)

This change makes progress on the plan in #4371. It does not replicate the full [matrix] implemented in #3851, but it does replicate the 1.ii section of the Linux matrix. It leverages "heavy" self-hosted runners, and demonstrates a repeatable pattern for future matrices. [matrix]: https://github.com/XRPLF/rippled/blob/d794a0f3f161bb30c74881172fc38f763d7d46e8/.github/README.md#continuous-integration
XRPLF · Apr 24, 2023 · 436de0e · 436de0e
1 parent 8d482d3
commit 436de0e
Show file tree

Hide file tree

Showing 4 changed files with 197 additions and 68 deletions.
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
@@ -0,0 +1,42 @@
+name: build
+inputs:
+  generator:
+    default: null
+  configuration:
+    required: true
+  cmake-args:
+    default: null
+# An implicit input is the environment variable `build_dir`.
+runs:
+  using: composite
+  steps:
+    - name: export custom recipes
+      shell: bash
+      run: conan export external/snappy snappy/1.1.9@
+    - name: install dependencies
+      shell: bash
+      run: |
+        mkdir ${build_dir}
+        cd ${build_dir}
+        conan install \
+          --output-folder . \
+          --build missing \
+          --settings build_type=${{ inputs.configuration }} \
+          ..
+    - name: configure
+      shell: bash
+      run: |
+        cd ${build_dir}
+        cmake \
+          ${{ inputs.generator && format('-G {0}', inputs.generator) || '' }} \
+          -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
+          -DCMAKE_BUILD_TYPE=${{ inputs.configuration }} \
+          ${{ inputs.cmake-args }} \
+          ..
+    - name: build
+      shell: bash
+      run: |
+        cmake \
+          --build ${build_dir} \
+          --config ${{ inputs.configuration }} \
+          --parallel ${NUM_PROCESSORS:-$(nproc)}
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -0,0 +1,69 @@
+name: macos
+on: [push, pull_request]
+
+jobs:
+
+  test:
+    strategy:
+      matrix:
+        platform:
+          - macos-12
+        generator:
+          - Ninja
+        configuration:
+          - Release
+    runs-on: ${{ matrix.platform }}
+    env:
+      # The `build` action requires these variables.
+      build_dir: .build
+      NUM_PROCESSORS: 2
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: install Ninja
+        if: matrix.generator == 'Ninja'
+        run: brew install ninja
+      - name: choose Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+      - name: learn Python cache directory
+        id: pip-cache
+        run: |
+          sudo pip install --upgrade pip
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+      - name: restore Python cache directory
+        uses: actions/cache@v2
+        with:
+            path: ${{ steps.pip-cache.outputs.dir }}
+            key: ${{ runner.os }}-${{ hashFiles('.github/workflows/nix.yml') }}
+      - name: install Conan
+        run: pip install wheel 'conan<2'
+      - name: check environment
+        run: |
+          echo ${PATH} | tr ':' '\n'
+          python --version
+          conan --version
+          cmake --version
+          env
+      - name: configure Conan
+        run: |
+          conan profile new default --detect
+          conan profile update settings.compiler.cppstd=20 default
+      - name: learn Conan cache directory
+        id: conan-cache
+        run: |
+          echo "dir=$(conan config get storage.path)" >> $GITHUB_OUTPUT
+      - name: restore Conan cache directory
+        uses: actions/cache@v2
+        with:
+            path: ${{ steps.conan-cache.outputs.dir }}
+            key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/nix.yml') }}
+      - name: build
+        uses: ./.github/actions/build
+        with:
+          generator: ${{ matrix.generator }}
+          configuration: ${{ matrix.configuration }}
+      - name: test
+        run: |
+          ${build_dir}/rippled --unittest
diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
@@ -3,94 +3,112 @@ on: [push, pull_request]
 
 jobs:
 
-  test:
+  dependencies:
     strategy:
       fail-fast: false
       matrix:
         platform:
-          - ubuntu-latest
-          - macos-12
-        generator:
-          - Ninja
+          - linux
+        compiler:
+          - gcc
+          - clang
         configuration:
+          - Debug
           - Release
-    runs-on: ${{ matrix.platform }}
-    env:
-      build_dir: .build
+        include:
+          - compiler: gcc
+            profile:
+              version: 11
+              cc: /usr/bin/gcc
+              cxx: /usr/bin/g++
+          - compiler: clang
+            profile:
+              version: 14
+              cc: /usr/bin/clang-14
+              cxx: /usr/bin/clang++-14
+    runs-on: [self-hosted, heavy]
+    container: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
-      - name: install Ninja on Linux
-        if: matrix.generator == 'Ninja' && runner.os == 'Linux'
-        run: sudo apt install ninja-build
-      - name: install Ninja on OSX
-        if: matrix.generator == 'Ninja' && runner.os == 'macOS'
-        run: brew install ninja
-      - name: install nproc on OSX
-        if: runner.os == 'macOS'
-        run: brew install coreutils
-      - name: choose Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: 3.9
-      - name: learn Python cache directory
-        id: pip-cache
-        run: |
-          sudo pip install --upgrade pip
-          echo "::set-output name=dir::$(pip cache dir)"
-      - name: restore Python cache directory
-        uses: actions/cache@v2
-        with:
-            path: ${{ steps.pip-cache.outputs.dir }}
-            key: ${{ runner.os }}-${{ hashFiles('.github/workflows/nix.yml') }}
-      - name: install Conan
-        run: pip install wheel 'conan~=1.52'
       - name: check environment
         run: |
           echo ${PATH} | tr ':' '\n'
-          python --version
           conan --version
           cmake --version
           env
       - name: configure Conan
         run: |
           conan profile new default --detect
           conan profile update settings.compiler.cppstd=20 default
-      - name: configure Conan on Linux
-        if: runner.os == 'Linux'
-        run: |
+          conan profile update settings.compiler=${{ matrix.compiler }} default
+          conan profile update settings.compiler.version=${{ matrix.profile.version }} default
           conan profile update settings.compiler.libcxx=libstdc++11 default
-      - name: learn Conan cache directory
-        id: conan-cache
+          conan profile update env.CC=${{ matrix.profile.cc }} default
+          conan profile update env.CXX=${{ matrix.profile.cxx }} default
+          conan profile update conf.tools.build:compiler_executables='{"c": "${{ matrix.profile.cc }}", "cpp": "${{ matrix.profile.cxx }}"}' default
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: build dependencies
         run: |
-          echo "::set-output name=dir::$(conan config get storage.path)"
-      - name: restore Conan cache directory
-        uses: actions/cache@v2
+          mkdir .build
+          cd .build
+          conan install \
+            --output-folder . \
+            --build missing \
+            --settings build_type=${{ matrix.configuration }} \
+            ..
+      - name: archive cache
+        run: tar -czf conan.tar -C ~/.conan .
+      - name: upload cache
+        uses: actions/upload-artifact@v3
         with:
-            path: ${{ steps.conan-cache.outputs.dir }}
-            key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/nix.yml') }}
-      - name: export Snappy
-        run: conan export external/snappy snappy/1.1.9@
-      - name: install dependencies
+          name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
+          path: conan.tar
+
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux
+        compiler:
+          - gcc
+          - clang
+        configuration:
+          - Debug
+          - Release
+        cmake-args:
+          -
+          - "-Dunity=ON"
+    needs: dependencies
+    runs-on: [self-hosted, heavy]
+    container: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b
+    env:
+      build_dir: .build
+    steps:
+      - name: download cache
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
+      - name: extract cache
         run: |
-          mkdir ${build_dir}
-          cd ${build_dir}
-          conan install .. --build missing --settings build_type=${{ matrix.configuration }}
-      - name: configure
+          mkdir -p ~/.conan
+          tar -xzf conan.tar -C ~/.conan
+      - name: check environment
         run: |
-          cd ${build_dir}
-          cmake \
-            -G ${{ matrix.generator }} \
-            -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-            -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} \
-            -Dassert=ON \
-            -Dcoverage=OFF \
-            -Dreporting=OFF \
-            -Dunity=OFF \
-            ..
+          echo ${PATH} | tr ':' '\n'
+          conan --version
+          cmake --version
+          env
+          ls ~/.conan
+      - name: checkout
+        uses: actions/checkout@v3
       - name: build
-        run: |
-          cmake --build ${build_dir} --target rippled --parallel $(nproc)
+        uses: ./.github/actions/build
+        with:
+          generator: Ninja
+          configuration: ${{ matrix.configuration }}
+          cmake-args: ${{ matrix.cmake-args }}
       - name: test
         run: |
           ${build_dir}/rippled --unittest --unittest-jobs $(nproc)
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -39,14 +39,14 @@ jobs:
         id: pip-cache
         run: |
           pip install --upgrade pip
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
       - name: restore Python cache directory
         uses: actions/cache@v2
         with:
             path: ${{ steps.pip-cache.outputs.dir }}
             key: ${{ runner.os }}-${{ hashFiles('.github/workflows/windows.yml') }}
       - name: install Conan
-        run: pip install wheel 'conan~=1.52'
+        run: pip install wheel 'conan<2'
       - name: check environment
         run: |
           $env:PATH -split ';'
@@ -63,7 +63,7 @@ jobs:
       - name: learn Conan cache directory
         id: conan-cache
         run: |
-          echo "::set-output name=dir::$(conan config get storage.path)"
+          echo "dir=$(conan config get storage.path)" >> $GITHUB_OUTPUT
       - name: restore Conan cache directory
         uses: actions/cache@v2
         with: