From 5fe0dc44bd3a34ee46f446652e9f96f158a0ef48 Mon Sep 17 00:00:00 2001 From: Thomas Eizinger Date: Wed, 14 Dec 2022 09:37:19 +1100 Subject: [PATCH] ci(caching): make caching more effective (#3215) Currently, we create a new cache for each workflow run for each crate. That ends up blowing the maximum allowed cache size of 10GB and GitHub deletes the least-recently used cache again. Effectively, this means we don't have any caching. This patch introduces a cache factory workflow that only runs on master and always _saves_ a new cache. The CI workflow run for pull-requests on the other hand only restore these caches but don't save them. --- .github/workflows/cache-factory.yml | 78 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 29 +++++------ 2 files changed, 93 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/cache-factory.yml diff --git a/.github/workflows/cache-factory.yml b/.github/workflows/cache-factory.yml new file mode 100644 index 00000000000..f63e8ad7247 --- /dev/null +++ b/.github/workflows/cache-factory.yml @@ -0,0 +1,78 @@ +# This workflow _produces_ caches which are used to speed up pull request builds. +# The caches are split by Rust version (stable vs MSRV per crate) because those caches cannot share any artifacts. + +name: Cache factory + +on: + push: + branches: + - master # Caches are only created on master branch. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + gather_msrv_versions: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.find-rust-versions.outputs.versions }} + steps: + - uses: actions/checkout@v3 + + - id: find-rust-versions + run: | + RUST_VERSIONS=$(cargo metadata --format-version=1 --no-deps | jq -c '.packages | map(.rust_version) | unique | del(..|nulls)') + echo "versions=${RUST_VERSIONS}" >> $GITHUB_OUTPUT + + make_msrv_cache: + runs-on: ubuntu-latest + needs: gather_msrv_versions + strategy: + fail-fast: false + matrix: + rust: ${{ fromJSON(needs.gather_msrv_versions.outputs.versions) }} + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 + with: + toolchain: ${{ matrix.rust }} + + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 + with: + shared-key: msrv-cache + + - name: Compile all crates which have MSRV ${{ matrix.rust }} + run: | + cargo metadata --format-version=1 --no-deps | \ + jq -r '.packages[] | select(.rust_version == "${{ matrix.rust }}") | "+\(.rust_version) build --all-features --package \(.name)"' | + xargs --verbose -L 1 cargo + + make_stable_rust_cache: + runs-on: ubuntu-latest + steps: + - name: Install Protoc + uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 # v1.1.2 + + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 + with: + toolchain: stable + + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 + with: + shared-key: stable-cache + + - name: Compile workspace with stable Rust + run: cargo test --all-features --all-targets --workspace --no-run + + - name: Render docs + run: cargo doc --all-features --workspace + + - name: Install tools + run: cargo install cargo-semver-checks --locked diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d764ccd0ed..bfce5917a16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,30 +38,33 @@ jobs: with: profile: minimal toolchain: ${{ steps.parse-msrv.outputs.version }} + override: true - - name: Update to latest stable Rust - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 + with: + shared-key: msrv-cache + save-if: false + + - name: Check if ${{ matrix.crate }} compiles on MSRV (Rust ${{ steps.parse-msrv.outputs.version }}) + run: cargo +${{ steps.parse-msrv.outputs.version }} build --package ${{ matrix.crate }} --all-features + + - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 with: profile: minimal toolchain: stable override: true - # By default, this action already includes the active Rust toolchain in the cache key. - # We also install a separate toolchain for the MSRV check so all we need to do is add that to the key to make sure it invalidates when we update the MSRV. - # cargo separates build artifacts by Rust compiler version, meaning we can compile with different versions but cache all artifacts. - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 with: - key: ${{ matrix.crate }}-msrv-${{ steps.parse-msrv.outputs.version }} - - - name: Check if ${{ matrix.crate }} compiles on MSRV (Rust ${{ steps.parse-msrv.outputs.version }}) - run: cargo +${{ steps.parse-msrv.outputs.version }} check --package ${{ matrix.crate }} --all-features - - - name: Check if we compile without any features activated - run: cargo check --package ${{ matrix.crate }} --no-default-features + shared-key: stable-cache + save-if: false - name: Run all tests run: cargo test --package ${{ matrix.crate }} --all-features + - name: Check if we compile without any features activated + run: cargo build --package ${{ matrix.crate }} --no-default-features + - name: Check if crate has been released id: check-released run: | @@ -138,8 +141,6 @@ jobs: override: true - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 - with: - key: ${{ matrix.runtime }} - run: cargo check --package libp2p --features="${{ matrix.features }}"