From 1ac0753c9895ca372989e1a9870fa44057d66c5f Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 29 Nov 2023 06:53:50 +1000 Subject: [PATCH 1/7] Run clippy and build all targets on all crates individually --- .github/workflows/ci-build-crates.yml | 11 +++++++---- .github/workflows/ci-lint.yml | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-build-crates.yml b/.github/workflows/ci-build-crates.yml index 77ea4b44e2c..5b117f36477 100644 --- a/.github/workflows/ci-build-crates.yml +++ b/.github/workflows/ci-build-crates.yml @@ -132,15 +132,18 @@ jobs: # Some Zebra crates do not have any features, and most don't have any default features. - name: Build ${{ matrix.crate }} crate with no default features run: | - cargo build --package ${{ matrix.crate }} --no-default-features + cargo clippy --package ${{ matrix.crate }} --no-default-features --all-targets -- -D warnings + cargo build --package ${{ matrix.crate }} --no-default-features --all-targets - - name: Build ${{ matrix.crate }} crate normally + - name: Build ${{ matrix.crate }} crate with default features run: | - cargo build --package ${{ matrix.crate }} + cargo clippy --package ${{ matrix.crate }} --all-targets -- -D warnings + cargo build --package ${{ matrix.crate }} --all-targets - name: Build ${{ matrix.crate }} crate with all features run: | - cargo build --package ${{ matrix.crate }} --all-features + cargo clippy --package ${{ matrix.crate }} --all-features --all-targets -- -D warnings + cargo build --package ${{ matrix.crate }} --all-features --all-targets failure-issue: name: Open or update issues for building crates individually failures diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index a15ac4b50e5..1b43d15ace0 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -98,11 +98,11 @@ jobs: # GitHub displays the clippy job and its results as separate entries name: Clippy (stable) Results token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features --all-targets -- -D warnings + args: --workspace --all-features --all-targets -- -D warnings - name: Run clippy manually without annotations if: ${{ !steps.check_permissions.outputs.has-permission }} - run: cargo clippy --all-features --all-targets -- -D warnings + run: cargo clippy --workspace --all-features --all-targets -- -D warnings fmt: name: Rustfmt From 6cb7affb4b8448fb0781df3315bfa7e1e8faaebc Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 29 Nov 2023 13:33:27 +1000 Subject: [PATCH 2/7] Fix prod and test features for scanner deps --- zebra-scan/Cargo.toml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index 47709df1a0d..3c4f96ce0d9 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -19,26 +19,30 @@ categories = ["cryptography::cryptocurrencies"] # Production features that activate extra dependencies, or extra features in dependencies [dependencies] -zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.31" } -zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31" } - -zcash_primitives = "0.13.0-rc.1" -zcash_client_backend = "0.10.0-rc.1" color-eyre = "0.6.2" indexmap = { version = "2.0.1", features = ["serde"] } serde = { version = "1.0.193", features = ["serde_derive"] } -tokio = { version = "1.34.0", features = ["test-util"] } +tokio = "1.34.0" tower = "0.4.13" tracing = "0.1.39" +zcash_client_backend = "0.10.0-rc.1" +zcash_primitives = "0.13.0-rc.1" + +zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.31" } +zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31" } + [dev-dependencies] -zcash_note_encryption = "0.4.0" -rand = "0.8.5" bls12_381 = "0.8.0" -jubjub = "0.10.0" ff = "0.13.0" group = "0.13.0" +jubjub = "0.10.0" +rand = "0.8.5" +tokio = { version = "1.34.0", features = ["test-util"] } + +zcash_note_encryption = "0.4.0" +zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test" } From 359cbd374e44a8be835dc3f40262ad1c48f6322e Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 29 Nov 2023 13:36:53 +1000 Subject: [PATCH 3/7] Standardise dependency order --- zebra-scan/src/scan.rs | 4 +++- zebra-scan/src/tests.rs | 12 +++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/zebra-scan/src/scan.rs b/zebra-scan/src/scan.rs index 9b284eedcb1..72577264f7a 100644 --- a/zebra-scan/src/scan.rs +++ b/zebra-scan/src/scan.rs @@ -1,10 +1,11 @@ -//! The scan task. +//! The scanner task and scanning APIs. use std::{sync::Arc, time::Duration}; use color_eyre::{eyre::eyre, Report}; use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt}; use tracing::info; + use zcash_client_backend::{ data_api::ScannedBlock, proto::compact_formats::{ @@ -13,6 +14,7 @@ use zcash_client_backend::{ scanning::{ScanError, ScanningKey}, }; use zcash_primitives::zip32::AccountId; + use zebra_chain::{ block::Block, parameters::Network, serialization::ZcashSerialize, transaction::Transaction, }; diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index fe3be0a6e2c..984abee1bb9 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -5,6 +5,11 @@ use std::sync::Arc; +use color_eyre::Result; +use ff::{Field, PrimeField}; +use group::GroupEncoding; +use rand::{rngs::OsRng, RngCore}; + use zcash_client_backend::{ encoding::decode_extended_full_viewing_key, proto::compact_formats::{ @@ -26,13 +31,6 @@ use zcash_primitives::{ zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, }; -use color_eyre::Result; - -use rand::{rngs::OsRng, RngCore}; - -use ff::{Field, PrimeField}; -use group::GroupEncoding; - use zebra_chain::{ block::Block, chain_tip::ChainTip, parameters::Network, serialization::ZcashDeserializeInto, transaction::Hash, From ea05066ed3c9a56ea7ad09b03d583709f1103794 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 29 Nov 2023 13:37:06 +1000 Subject: [PATCH 4/7] Remove unnecessary async in tests --- zebra-scan/src/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index 984abee1bb9..04c00335949 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -112,8 +112,8 @@ async fn scanning_from_populated_zebra_state() -> Result<()> { /// the transaction and one additional random transaction without it. /// - Verify one relevant transaction is found in the chain when scanning for the pre created fake /// account's nullifier. -#[tokio::test] -async fn scanning_from_fake_generated_blocks() -> Result<()> { +#[test] +fn scanning_from_fake_generated_blocks() -> Result<()> { let account = AccountId::from(12); let extsk = ExtendedSpendingKey::master(&[]); let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); @@ -233,9 +233,9 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { /// In this test we generate a viewing key and manually add it to the database. Also we send results to the Storage database. /// The purpose of this test is to check if our database and our scanning code are compatible. -#[tokio::test] +#[test] #[allow(deprecated)] -async fn scanning_fake_blocks_store_key_and_results() -> Result<()> { +fn scanning_fake_blocks_store_key_and_results() -> Result<()> { // Generate a key let account = AccountId::from(12); let extsk = ExtendedSpendingKey::master(&[]); From 445de4ff6e40a9855aec2bcfaaa1ddb901cca3ce Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 29 Nov 2023 15:33:42 +1000 Subject: [PATCH 5/7] Fix an unused import in a test --- zebrad/tests/common/config.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/common/config.rs b/zebrad/tests/common/config.rs index 38dd75d4a8c..07cb70c4291 100644 --- a/zebrad/tests/common/config.rs +++ b/zebrad/tests/common/config.rs @@ -15,7 +15,7 @@ use std::{ use color_eyre::eyre::Result; use tempfile::TempDir; -use zebra_chain::parameters::Network::{self, *}; +use zebra_chain::parameters::Network; use zebra_test::net::random_known_port; use zebrad::{ components::{mempool, sync, tracing}, @@ -70,10 +70,11 @@ pub fn default_test_config(net: Network) -> Result { #[cfg(feature = "getblocktemplate-rpcs")] { - let miner_address = if network.network == Mainnet { - "t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1" - } else { + let miner_address = if network.network.is_a_test_network() { + // Assume test networks all use the same address prefix and format "t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v" + } else { + "t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1" }; mining.miner_address = Some(miner_address.parse().expect("hard-coded address is valid")); From 4e2192d4f356ddced3a006db458ab63695df85c5 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 30 Nov 2023 06:02:27 +1000 Subject: [PATCH 6/7] Work around a no space left on device error --- .github/workflows/ci-build-crates.patch.yml | 4 ++-- .github/workflows/ci-build-crates.yml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-build-crates.patch.yml b/.github/workflows/ci-build-crates.patch.yml index e173752dd06..58d8f8210d1 100644 --- a/.github/workflows/ci-build-crates.patch.yml +++ b/.github/workflows/ci-build-crates.patch.yml @@ -33,11 +33,11 @@ jobs: # This step is meant to dynamically create a JSON containing the values of each crate # available in this repo in the root directory. We use `cargo tree` to accomplish this task. # - # The result from `cargo tree` is then transform to JSON values between double quotes, + # The result from `cargo tree` is then transform to JSON values between double quotes, # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable. # # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output - # named `matrix`, which is then used as the input in following steps, + # named `matrix`, which is then used as the input in following steps, # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}` - id: set-matrix name: Dynamically build crates JSON diff --git a/.github/workflows/ci-build-crates.yml b/.github/workflows/ci-build-crates.yml index 5b117f36477..e36ab5ddaec 100644 --- a/.github/workflows/ci-build-crates.yml +++ b/.github/workflows/ci-build-crates.yml @@ -142,6 +142,11 @@ jobs: - name: Build ${{ matrix.crate }} crate with all features run: | + # Work around a "No space left on device" error + # TODO: use a runner with a bigger disk + if [[ ${{ matrix.crate }} == "zebrad" ]]; then + cargo clean + fi cargo clippy --package ${{ matrix.crate }} --all-features --all-targets -- -D warnings cargo build --package ${{ matrix.crate }} --all-features --all-targets From 44611a674fecd6cb242566f63812a6da08051ae1 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 30 Nov 2023 06:07:15 +1000 Subject: [PATCH 7/7] Actually just use a larger runner --- .github/workflows/ci-build-crates.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-build-crates.yml b/.github/workflows/ci-build-crates.yml index e36ab5ddaec..a5aa0c6d65c 100644 --- a/.github/workflows/ci-build-crates.yml +++ b/.github/workflows/ci-build-crates.yml @@ -101,7 +101,8 @@ jobs: name: Build ${{ matrix.crate }} crate timeout-minutes: 90 needs: [ matrix, check-matrix ] - runs-on: ubuntu-latest + # Some of these builds take more than 14GB disk space + runs-on: ubuntu-latest-m strategy: # avoid rate-limit errors by only launching a few of these jobs at a time max-parallel: 2 @@ -142,11 +143,6 @@ jobs: - name: Build ${{ matrix.crate }} crate with all features run: | - # Work around a "No space left on device" error - # TODO: use a runner with a bigger disk - if [[ ${{ matrix.crate }} == "zebrad" ]]; then - cargo clean - fi cargo clippy --package ${{ matrix.crate }} --all-features --all-targets -- -D warnings cargo build --package ${{ matrix.crate }} --all-features --all-targets