From 2a2719cc23de85a932aa69e8230e20257fb5f5e0 Mon Sep 17 00:00:00 2001 From: wcampbell Date: Tue, 22 Oct 2024 00:06:38 -0400 Subject: [PATCH] Add backon retry to tests Hopefully fix the following error I've been seeing in CI for downloading test-assets: ---- test_19 stdout ---- Fetching file out.squashfs ...[/home/runner/.cargo/git/checkouts/test-assets-af055f640a5e3329/ebe2e7d/src/lib.rs:137:5] &tfile.url = "https://wcampbell.dev/squashfs/testing/test_19/out.squashfs" thread 'test_19' panicked at backhand-test/tests/test.rs:66:68: called `Result::unwrap()` on an `Err` value: Io(Custom { kind: UnexpectedEof, error: "peer closed connection without sending TLS close_notify: https://docs.rs/rustls/latest/rustls/manual/_03_howto/index.html#unexpected-eof" }) note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace --- Cargo.lock | 94 +++++++++++++++++++++++++++++ backhand-test/Cargo.toml | 1 + backhand-test/tests/add.rs | 4 +- backhand-test/tests/common/mod.rs | 18 ++++++ backhand-test/tests/mutate.rs | 2 +- backhand-test/tests/non_standard.rs | 2 +- backhand-test/tests/raw.rs | 2 +- backhand-test/tests/replace.rs | 2 +- backhand-test/tests/test.rs | 4 +- backhand-test/tests/unsquashfs.rs | 4 +- 10 files changed, 123 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d289152..3c727ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.0" @@ -138,6 +147,32 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "backon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + [[package]] name = "base64" version = "0.21.7" @@ -626,6 +661,21 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + [[package]] name = "generic-array" version = "0.14.7" @@ -647,6 +697,24 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "half" version = "2.4.0" @@ -960,6 +1028,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -1178,6 +1255,12 @@ dependencies = [ "libc", ] +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustix" version = "0.38.32" @@ -1419,6 +1502,7 @@ version = "0.0.0" dependencies = [ "assert_cmd", "backhand", + "backon", "dir-diff", "env_logger", "libdeflater", @@ -1485,6 +1569,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "pin-project-lite", +] + [[package]] name = "toml_datetime" version = "0.6.8" diff --git a/backhand-test/Cargo.toml b/backhand-test/Cargo.toml index a38619e3..eb41b3a1 100644 --- a/backhand-test/Cargo.toml +++ b/backhand-test/Cargo.toml @@ -16,6 +16,7 @@ libdeflater = "1.21.0" env_logger = "0.11.5" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "fmt"] } nix = { version = "0.28.0", default-features = false, features = ["fs"] } +backon = "1.2.0" [lib] bench = false diff --git a/backhand-test/tests/add.rs b/backhand-test/tests/add.rs index 29226925..4cd60cf7 100644 --- a/backhand-test/tests/add.rs +++ b/backhand-test/tests/add.rs @@ -26,7 +26,7 @@ fn test_add() { }]; const TEST_PATH: &str = "test-assets/test_01"; - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let image_path = format!("{TEST_PATH}/{FILE_NAME}"); // Add /test dir @@ -141,7 +141,7 @@ fn test_dont_emit_compression_options() { }]; const TEST_PATH: &str = "test-assets/test_add_compression_options"; - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let image_path = format!("{TEST_PATH}/{FILE_NAME}"); let tmp_dir = tempdir().unwrap(); diff --git a/backhand-test/tests/common/mod.rs b/backhand-test/tests/common/mod.rs index e6ef733f..f24b7936 100644 --- a/backhand-test/tests/common/mod.rs +++ b/backhand-test/tests/common/mod.rs @@ -1,8 +1,26 @@ +use std::error::Error; use std::process::Command; +use std::time::Duration; use assert_cmd::prelude::*; +use backon::BlockingRetryable; +use backon::ExponentialBuilder; use tempfile::tempdir; use tempfile::tempdir_in; +use test_assets::TestAssetDef; + +fn download(assets_defs: &[TestAssetDef], test_path: &str) -> Result<(), Box> { + if test_assets::download_test_files(assets_defs, test_path, true).is_err() { + return Err("falied to download".into()); + } + Ok(()) +} + +pub fn download_backoff(assets_defs: &[TestAssetDef], test_path: &str) { + let strategy = ExponentialBuilder::default().with_max_delay(Duration::from_secs(60)); + + let _result = (|| download(assets_defs, test_path)).retry(strategy).call().unwrap(); +} /// test the new squashfs vs the original squashfs with squashfs-tool/unsquashfs /// by extract diff --git a/backhand-test/tests/mutate.rs b/backhand-test/tests/mutate.rs index 2c99fd6c..8c8492a9 100644 --- a/backhand-test/tests/mutate.rs +++ b/backhand-test/tests/mutate.rs @@ -53,7 +53,7 @@ fn test_add_00() { let og_path = format!("{TEST_PATH}/out.squashfs"); let new_path = format!("{TEST_PATH}/bytes.squashfs"); - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let file = BufReader::new(File::open(&og_path).unwrap()); let og_filesystem = FilesystemReader::from_reader(file).unwrap(); let mut new_filesystem = FilesystemWriter::from_fs_reader(&og_filesystem).unwrap(); diff --git a/backhand-test/tests/non_standard.rs b/backhand-test/tests/non_standard.rs index 839445d0..2d65a581 100644 --- a/backhand-test/tests/non_standard.rs +++ b/backhand-test/tests/non_standard.rs @@ -24,7 +24,7 @@ fn full_test( kind: &Kind, pad: Option, ) { - test_assets::download_test_files(assets_defs, test_path, true).unwrap(); + common::download_backoff(&assets_defs, test_path); let og_path = format!("{test_path}/{filepath}"); let new_path = format!("{test_path}/bytes.squashfs"); { diff --git a/backhand-test/tests/raw.rs b/backhand-test/tests/raw.rs index 1735bbfc..90fe0dff 100644 --- a/backhand-test/tests/raw.rs +++ b/backhand-test/tests/raw.rs @@ -25,7 +25,7 @@ fn test_raw_00() { }]; const TEST_PATH: &str = "test-assets/test_raw_00"; let new_path = format!("{TEST_PATH}/bytes.squashfs"); - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let header = NodeHeader { permissions: 0o755, uid: 1000, gid: 1000, mtime: 0 }; diff --git a/backhand-test/tests/replace.rs b/backhand-test/tests/replace.rs index 01c1672c..59f5f6a5 100644 --- a/backhand-test/tests/replace.rs +++ b/backhand-test/tests/replace.rs @@ -18,7 +18,7 @@ fn test_replace() { }]; const TEST_PATH: &str = "test-assets/test_05"; - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let image_path = format!("{TEST_PATH}/{FILE_NAME}"); // extract single file diff --git a/backhand-test/tests/test.rs b/backhand-test/tests/test.rs index 5a0e99a5..71360746 100644 --- a/backhand-test/tests/test.rs +++ b/backhand-test/tests/test.rs @@ -26,7 +26,7 @@ enum Verify { } fn only_read(assets_defs: &[TestAssetDef], filepath: &str, test_path: &str, offset: u64) { - test_assets::download_test_files(assets_defs, test_path, true).unwrap(); + common::download_backoff(assets_defs, test_path); let og_path = format!("{test_path}/{filepath}"); let file = BufReader::new(File::open(&og_path).unwrap()); @@ -63,7 +63,7 @@ fn full_test_inner( assert_success: bool, run_squashfs_tools_unsquashfs: bool, ) { - test_assets::download_test_files(assets_defs, test_path, true).unwrap(); + common::download_backoff(assets_defs, test_path); let og_path = format!("{test_path}/{filepath}"); let new_path = format!("{test_path}/bytes.squashfs"); diff --git a/backhand-test/tests/unsquashfs.rs b/backhand-test/tests/unsquashfs.rs index 0da32bbf..4e0d3762 100644 --- a/backhand-test/tests/unsquashfs.rs +++ b/backhand-test/tests/unsquashfs.rs @@ -15,7 +15,7 @@ fn test_unsquashfs_cli() { }]; const TEST_PATH: &str = "test-assets/test_re815_xev160"; - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let image_path = format!("{TEST_PATH}/{FILE_NAME}"); // single file @@ -119,7 +119,7 @@ fn test_unsquashfs_cli_auto_offset() { ), }]; const TEST_PATH: &str = "test-assets/test_openwrt_tplink_archera7v5"; - test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap(); + common::download_backoff(&asset_defs, TEST_PATH); let image_path = format!("{TEST_PATH}/{FILE_NAME}"); let tmp_dir = tempdir().unwrap();