Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

cli: introduce host-perf-check command #4342

Merged
merged 31 commits into from
Dec 9, 2021
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
d21db4e
cli: introduce host-perf-check command
slumber Nov 21, 2021
5937ce6
fix build
slumber Nov 22, 2021
ff6d3a1
Review fixes
slumber Nov 22, 2021
71b19d3
Extract perf check into new module
slumber Nov 23, 2021
6770d63
Explicit log level
slumber Nov 23, 2021
e142424
Use a build script for cfg build
slumber Nov 23, 2021
9414d2f
Save dummy file in HOME
slumber Nov 23, 2021
c6a72a8
Refactor
slumber Nov 24, 2021
a4f75a2
Error on any non-release build
slumber Nov 24, 2021
65701ad
Improve naming
slumber Nov 24, 2021
30e225f
Use the base path for caching
slumber Nov 24, 2021
7bb37a2
Reuse wasm binary from kusama_runtime
slumber Nov 24, 2021
11f1a16
Update cli/src/command.rs
slumber Nov 25, 2021
a2b0a72
Add an explanation to gethostname
slumber Nov 25, 2021
2aba480
Make cache path configurable
slumber Nov 30, 2021
2891597
Add erasure-coding check
slumber Dec 1, 2021
a39de72
Green threshold for perf tests
slumber Dec 1, 2021
f89e6b5
Write hostname without zero bytes
slumber Dec 2, 2021
4dd4e89
Add force flag
slumber Dec 2, 2021
de88f4c
Extract performance test into a separate crate
slumber Dec 3, 2021
95bf838
Implement constants generation
slumber Dec 3, 2021
633088d
Add logs
slumber Dec 3, 2021
28ecac3
Simplify fs read/write
slumber Dec 4, 2021
ab46e65
Propagate tracing error
slumber Dec 4, 2021
04d311e
Expect instead of unwrap
slumber Dec 4, 2021
f9ae107
Update headers
slumber Dec 4, 2021
f8739ae
Rename cache_path to base_path
slumber Dec 5, 2021
9bdbddd
Drop caching
slumber Dec 5, 2021
f36888d
Apply suggestions from code review
bkchr Dec 5, 2021
e1e42e4
Merge remote-tracking branch 'origin/master' into slumber-hostperfche…
slumber Dec 9, 2021
66dfa6d
Decrease the number of warm up runs
slumber Dec 9, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cli/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ pub enum Subcommand {
#[structopt(name = "benchmark", about = "Benchmark runtime pallets.")]
Benchmark(frame_benchmarking_cli::BenchmarkCmd),

/// Compiles a sample wasm code in order to measure the machine capabilities
/// of running PVF host.
#[cfg(not(debug_assertions))]
slumber marked this conversation as resolved.
Show resolved Hide resolved
HostPerfCheck,

/// Try some command against runtime state.
#[cfg(feature = "try-runtime")]
TryRuntime(try_runtime_cli::TryRuntimeCmd),
Expand Down
70 changes: 70 additions & 0 deletions cli/src/command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,68 @@ fn ensure_dev(spec: &Box<dyn service::ChainSpec>) -> std::result::Result<(), Str
}
}

/// Runs a performance check via compiling sample wasm code with a timeout.
/// Only available in release build since the check would take too much time otherwise.
/// Returns `Ok` if the check has been passed previously.
#[cfg(not(debug_assertions))]
slumber marked this conversation as resolved.
Show resolved Hide resolved
fn host_perf_check() -> Result<()> {
use polkadot_node_core_pvf::sp_maybe_compressed_blob;
use std::{fs::OpenOptions, path::Path, time::Duration};

const PERF_CHECK_TIME_LIMIT: Duration = Duration::from_secs(20);
slumber marked this conversation as resolved.
Show resolved Hide resolved
const CODE_SIZE_LIMIT: usize = 1024usize.pow(3);
const WASM_CODE: &[u8] = include_bytes!(
"../../target/release/wbuild/kusama-runtime/kusama_runtime.compact.compressed.wasm"
);
const CHECK_PASSED_FILE_NAME: &str = ".perf_check_passed";

// We will try to save a dummy file to the same path as the polkadot binary
// to make it independent from the current directory.
let check_passed_path = std::env::current_exe()
slumber marked this conversation as resolved.
Show resolved Hide resolved
.map(|mut path| {
path.pop();
path
})
.unwrap_or_default()
.join(CHECK_PASSED_FILE_NAME);

// To avoid running the check on every launch we create a dummy dot-file on success.
if Path::new(&check_passed_path).exists() {
info!("Performance check skipped: already passed");
drahnr marked this conversation as resolved.
Show resolved Hide resolved
return Ok(())
}

info!("Running the performance check...");
let start = std::time::Instant::now();

// Recreate the pipeline from the pvf prepare worker.
let code = sp_maybe_compressed_blob::decompress(WASM_CODE, CODE_SIZE_LIMIT).map_err(|err| {
Error::Other(format!("Failed to decompress test wasm code: {}", err.to_string()))
slumber marked this conversation as resolved.
Show resolved Hide resolved
})?;
let blob = polkadot_node_core_pvf::prevalidate(code.as_ref()).map_err(|err| {
Error::Other(format!(
"Failed to create runtime blob from the decompressed code: {}",
err.to_string()
))
})?;
let _ = polkadot_node_core_pvf::prepare(blob).map_err(|err| {
Error::Other(format!("Failed to precompile test wasm code: {}", err.to_string()))
})?;
slumber marked this conversation as resolved.
Show resolved Hide resolved

let elapsed = start.elapsed();
if elapsed <= PERF_CHECK_TIME_LIMIT {
info!("Performance check passed, elapsed: {:?}", start.elapsed());
// `touch` a dummy file.
let _ = OpenOptions::new().create(true).write(true).open(Path::new(&check_passed_path));
Ok(())
} else {
Err(Error::Other(format!(
"Performance check not passed: exceeded the {:?} time limit, elapsed: {:?}",
PERF_CHECK_TIME_LIMIT, elapsed
)))
}
}

/// Launch a node, accepting arguments just like a regular node,
/// accepts an alternative overseer generator, to adjust behavior
/// for integration tests as needed.
Expand Down Expand Up @@ -415,6 +477,14 @@ pub fn run() -> Result<()> {
#[cfg(not(feature = "polkadot-native"))]
panic!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
},
#[cfg(not(debug_assertions))]
slumber marked this conversation as resolved.
Show resolved Hide resolved
Some(Subcommand::HostPerfCheck) => {
let mut builder = sc_cli::LoggerBuilder::new("");
slumber marked this conversation as resolved.
Show resolved Hide resolved
builder.with_colors(true);
let _ = builder.init();
slumber marked this conversation as resolved.
Show resolved Hide resolved

host_perf_check()
},
Some(Subcommand::Key(cmd)) => Ok(cmd.run(&cli)?),
#[cfg(feature = "try-runtime")]
Some(Subcommand::TryRuntime(cmd)) => {
Expand Down
4 changes: 4 additions & 0 deletions node/core/pvf/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,8 @@ pub use metrics::Metrics;
pub use execute::worker_entrypoint as execute_worker_entrypoint;
pub use prepare::worker_entrypoint as prepare_worker_entrypoint;

pub use executor_intf::{prepare, prevalidate};

pub use sp_maybe_compressed_blob;

const LOG_TARGET: &str = "parachain::pvf";