diff --git a/Cargo.lock b/Cargo.lock index 9c05dec404..9b32b541fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,6 +528,7 @@ dependencies = [ "cosmwasm-std", "crc32fast", "criterion", + "criterion-inverted-throughput", "derivative", "dhat", "glob", diff --git a/packages/vm/Cargo.toml b/packages/vm/Cargo.toml index f140107243..edde1116ae 100644 --- a/packages/vm/Cargo.toml +++ b/packages/vm/Cargo.toml @@ -74,6 +74,7 @@ tracing = "0.1.32" [dev-dependencies] criterion = { version = "0.5.1", features = ["html_reports"] } +criterion-inverted-throughput = "0.1.0" glob = "0.3.1" hex-literal = "0.4.1" rand = "0.8" diff --git a/packages/vm/benches/main.rs b/packages/vm/benches/main.rs index 872a5b4607..6224a5d844 100644 --- a/packages/vm/benches/main.rs +++ b/packages/vm/benches/main.rs @@ -1,4 +1,8 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use criterion::{ + black_box, criterion_group, criterion_main, measurement::Measurement, BenchmarkId, Criterion, + Throughput, +}; +use criterion_inverted_throughput::InvertedThroughput; use rand::Rng; use std::sync::Arc; @@ -33,6 +37,7 @@ const CONTRACTS: u64 = 10; const DEFAULT_CAPABILITIES: &str = "cosmwasm_1_1,cosmwasm_1_2,cosmwasm_1_3,iterator,staking"; static HACKATOM: &[u8] = include_bytes!("../testdata/hackatom.wasm"); static CYBERPUNK: &[u8] = include_bytes!("../testdata/cyberpunk.wasm"); +static SHA1: &[u8] = include_bytes!("../testdata/bench_sha1.wasm"); static BENCH_CONTRACTS: &[&str] = &[ "cyberpunk_rust170.wasm", @@ -463,6 +468,89 @@ fn bench_combined(c: &mut Criterion) { group.finish(); } +fn bench_sha1(c: &mut Criterion) { + let mut group = c.benchmark_group("sha1"); + + // bytes, blocks, used_gas + let mut gas_reports_wasm: Vec<(u64, u64, u64)> = vec![]; + let mut gas_reports_api: Vec<(u64, u64, u64)> = vec![]; + + for i in 6..=16 { + // Every 64 bytes needs a hassing and there are + // 8 bytes header (message length) and 1 byte tail (EOF). + let len = u64::pow(2, i) - 9; + let block = (len + 9) / 64; + group.throughput(Throughput::Elements(block)); + group.bench_function(BenchmarkId::new("WASM/blocks", block), |b| { + let backend = mock_backend(&[]); + let much_gas: InstanceOptions = InstanceOptions { + gas_limit: HIGH_GAS_LIMIT, + }; + let mut instance = + Instance::from_code(SHA1, backend, much_gas, Some(DEFAULT_MEMORY_LIMIT)).unwrap(); + + let info = mock_info("creator", &coins(1000, "earth")); + let msg = br#"{}"#; + let contract_result = + call_instantiate::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg).unwrap(); + contract_result.into_result().unwrap(); + + let mut gas_used = 0; + b.iter(|| { + let gas_before = instance.get_gas_left(); + let info = mock_info("foo", &[]); + let msg = format!(r#"{{"wasm":{{"len":{}}}}}"#, len).into_bytes(); + let contract_result = + call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, &msg) + .unwrap(); + contract_result.into_result().unwrap(); + gas_used = gas_before - instance.get_gas_left(); + }); + if gas_reports_wasm.len() < (i - 5) as usize { + gas_reports_wasm.push((len, block, gas_used)) + } + }); + + group.bench_function(BenchmarkId::new("API/blocks", block), |b| { + let backend = mock_backend(&[]); + let much_gas: InstanceOptions = InstanceOptions { + gas_limit: HIGH_GAS_LIMIT, + }; + let mut instance = + Instance::from_code(SHA1, backend, much_gas, Some(DEFAULT_MEMORY_LIMIT)).unwrap(); + + let info = mock_info("creator", &coins(1000, "earth")); + let msg = br#"{}"#; + let contract_result = + call_instantiate::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg).unwrap(); + contract_result.into_result().unwrap(); + + let mut gas_used = 0; + b.iter(|| { + let gas_before = instance.get_gas_left(); + let info = mock_info("foo", &[]); + let msg = format!(r#"{{"api":{{"len":{}}}}}"#, len).into_bytes(); + let contract_result = + call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, &msg) + .unwrap(); + contract_result.into_result().unwrap(); + gas_used = gas_before - instance.get_gas_left(); + }); + if gas_reports_api.len() < (i - 5) as usize { + gas_reports_api.push((len, block, gas_used)) + } + }); + } + for i in 0..gas_reports_wasm.len() { + let wasm = gas_reports_wasm[i]; + let api = gas_reports_api[i]; + println!("gas wasm/{}bytes({}blocks): {}", wasm.0, wasm.1, wasm.2); + println!("gas api /{}bytes({}blocks): {}", api.0, api.1, api.2); + } + + group.finish(); +} + fn make_config(measurement_time_s: u64) -> Criterion { Criterion::default() .without_plots() @@ -471,6 +559,15 @@ fn make_config(measurement_time_s: u64) -> Criterion { .configure_from_args() } +fn make_inversion_throughput_config() -> Criterion { + Criterion::default() + .with_measurement(InvertedThroughput::new()) + .without_plots() + .measurement_time(Duration::new(10, 0)) + .sample_size(12) + .configure_from_args() +} + criterion_group!( name = instance; config = make_config(8); @@ -497,4 +594,9 @@ criterion_group!( .configure_from_args(); targets = bench_instance_threads ); -criterion_main!(instance, cache, combined, multi_threaded_instance); +criterion_group!( + name = sha1; + config = make_inversion_throughput_config(); + targets = bench_sha1 +); +criterion_main!(instance, cache, combined, multi_threaded_instance, sha1); diff --git a/packages/vm/testdata/bench_sha1.wasm b/packages/vm/testdata/bench_sha1.wasm new file mode 100644 index 0000000000..ae579275b0 Binary files /dev/null and b/packages/vm/testdata/bench_sha1.wasm differ