From 5f02129b7357dc764bbf9bcbe34cf56f613250f0 Mon Sep 17 00:00:00 2001 From: Tom French <15848336+TomAFrench@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:28:54 +0100 Subject: [PATCH] chore: add benchmarks for serializing a dummy program (#4813) # Description ## Problem\* Resolves ## Summary\* This PR adds some benchmarks for serializing a (not particularly realistic) ACIR program to give a sense of how this scales with the number of opcodes in a program. As part of this I've fixed some issues in the existing benchmarks which prevented good flamegraphs being generated on linux. I've also smoothed out the experience for creating flamegraphs with new scripts 1. run `sudo ./scripts/benchmark_start.sh` 2. cargo bench --bench serialization -- --profile-time=10 3. run `sudo ./scripts/benchmark_stop.sh` The flamegraphs for all of the serialization benchmarks will then be stored in `target/criterion//profile` The scripts to start and stop benchmarking disable/enable some kernel settings which improve profiling performance at the expense of security. ## Additional Context ## Documentation\* Check one: - [x] No documentation needed. - [ ] Documentation included in this PR. - [ ] **[For Experimental Features]** Documentation to be submitted in a separate PR. # PR Checklist\* - [x] I have tested the changes locally. - [x] I have formatted the changes with [Prettier](https://prettier.io/) and/or `cargo fmt` on default settings. --- Cargo.lock | 28 +++++- Cargo.toml | 8 ++ acvm-repo/acir/Cargo.toml | 6 ++ acvm-repo/acir/benches/serialization.rs | 123 ++++++++++++++++++++++++ scripts/benchmark_start.sh | 3 + scripts/benchmark_stop.sh | 3 + tooling/nargo_cli/Cargo.toml | 8 +- 7 files changed, 171 insertions(+), 8 deletions(-) create mode 100644 acvm-repo/acir/benches/serialization.rs create mode 100755 scripts/benchmark_start.sh create mode 100755 scripts/benchmark_stop.sh diff --git a/Cargo.lock b/Cargo.lock index 754a1c3d1fe..4ded5c40d4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,8 +10,10 @@ dependencies = [ "base64 0.21.2", "bincode", "brillig", + "criterion", "flate2", "fxhash", + "pprof 0.13.0", "serde", "serde-generate", "serde-reflection", @@ -623,7 +625,7 @@ dependencies = [ "lazy_static", "noir_grumpkin", "num-bigint", - "pprof", + "pprof 0.12.1", "thiserror", "wasm-bindgen-futures", "wasmer", @@ -2865,7 +2867,7 @@ dependencies = [ "notify", "notify-debouncer-full", "paste", - "pprof", + "pprof 0.13.0", "predicates 2.1.5", "prettytable-rs", "rayon", @@ -3546,6 +3548,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "pprof" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" +dependencies = [ + "backtrace", + "cfg-if 1.0.0", + "criterion", + "findshlibs", + "inferno", + "libc", + "log", + "nix 0.26.4", + "once_cell", + "parking_lot 0.12.1", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index 5dd453415aa..6a939878f9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,6 +104,14 @@ chumsky = { git = "https://github.com/jfecher/chumsky", rev = "ad9d312", default "ahash", "std", ] } + +# Benchmarking +criterion = "0.5.0" +# Note that using the "frame-pointer" feature breaks framegraphs on linux +# https://github.com/tikv/pprof-rs/pull/172 +pprof = { version = "0.13", features = ["flamegraph","criterion"] } + + dirs = "4" serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0" diff --git a/acvm-repo/acir/Cargo.toml b/acvm-repo/acir/Cargo.toml index 99095ad3f61..1702780ba9b 100644 --- a/acvm-repo/acir/Cargo.toml +++ b/acvm-repo/acir/Cargo.toml @@ -28,8 +28,14 @@ strum_macros = "0.24" serde-reflection = "0.3.6" serde-generate = "0.25.1" fxhash.workspace = true +criterion.workspace = true +pprof.workspace = true [features] default = ["bn254"] bn254 = ["acir_field/bn254", "brillig/bn254"] bls12_381 = ["acir_field/bls12_381", "brillig/bls12_381"] + +[[bench]] +name = "serialization" +harness = false diff --git a/acvm-repo/acir/benches/serialization.rs b/acvm-repo/acir/benches/serialization.rs new file mode 100644 index 00000000000..73e3916a73b --- /dev/null +++ b/acvm-repo/acir/benches/serialization.rs @@ -0,0 +1,123 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use std::{collections::BTreeSet, time::Duration}; + +use acir::{ + circuit::{Circuit, ExpressionWidth, Opcode, Program, PublicInputs}, + native_types::{Expression, Witness}, + FieldElement, +}; + +use pprof::criterion::{Output, PProfProfiler}; + +const SIZES: [usize; 9] = [10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000]; + +fn sample_program(num_opcodes: usize) -> Program { + let assert_zero_opcodes: Vec = (0..num_opcodes) + .map(|i| { + Opcode::AssertZero(Expression { + mul_terms: vec![( + FieldElement::from(2 * i), + Witness(i as u32), + Witness(i as u32 + 10), + )], + linear_combinations: vec![ + (FieldElement::from(2 * i), Witness(i as u32)), + (FieldElement::from(3 * i), Witness(i as u32 + 1)), + ], + q_c: FieldElement::from(i), + }) + }) + .collect(); + + Program { + functions: vec![Circuit { + current_witness_index: 4000, + opcodes: assert_zero_opcodes.to_vec(), + expression_width: ExpressionWidth::Bounded { width: 3 }, + private_parameters: BTreeSet::from([Witness(1), Witness(2), Witness(3), Witness(4)]), + public_parameters: PublicInputs(BTreeSet::from([Witness(5)])), + return_values: PublicInputs(BTreeSet::from([Witness(6)])), + assert_messages: Vec::new(), + recursive: false, + }], + } +} + +fn bench_serialization(c: &mut Criterion) { + let mut group = c.benchmark_group("serialize_program"); + for size in SIZES.iter() { + let program = sample_program(*size); + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), &program, |b, program| { + b.iter(|| Program::serialize_program(program)); + }); + } + group.finish(); + + let mut group = c.benchmark_group("serialize_program_json"); + for size in SIZES.iter() { + let program = sample_program(*size); + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), &program, |b, program| { + b.iter(|| { + let mut bytes = Vec::new(); + let mut serializer = serde_json::Serializer::new(&mut bytes); + Program::serialize_program_base64(program, &mut serializer) + }); + }); + } + group.finish(); +} + +fn bench_deserialization(c: &mut Criterion) { + let mut group = c.benchmark_group("deserialize_program"); + for size in SIZES.iter() { + let program = sample_program(*size); + let serialized_program = Program::serialize_program(&program); + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(size), + &serialized_program, + |b, program| { + b.iter(|| Program::deserialize_program(program)); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("deserialize_program_json"); + for size in SIZES.iter() { + let program = sample_program(*size); + + let serialized_program = { + let mut bytes = Vec::new(); + let mut serializer = serde_json::Serializer::new(&mut bytes); + Program::serialize_program_base64(&program, &mut serializer).expect("should succeed"); + bytes + }; + + group.throughput(Throughput::Elements(*size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(size), + &serialized_program, + |b, program| { + b.iter(|| { + let mut deserializer = serde_json::Deserializer::from_slice(program); + Program::deserialize_program_base64(&mut deserializer) + }); + }, + ); + } + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(40).measurement_time(Duration::from_secs(20)).with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = bench_serialization, bench_deserialization +); + +criterion_main!(benches); diff --git a/scripts/benchmark_start.sh b/scripts/benchmark_start.sh new file mode 100755 index 00000000000..3e69b3d2c65 --- /dev/null +++ b/scripts/benchmark_start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid diff --git a/scripts/benchmark_stop.sh b/scripts/benchmark_stop.sh new file mode 100755 index 00000000000..964e5291817 --- /dev/null +++ b/scripts/benchmark_stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo 4 | sudo tee /proc/sys/kernel/perf_event_paranoid diff --git a/tooling/nargo_cli/Cargo.toml b/tooling/nargo_cli/Cargo.toml index 1629ae86edb..111caaa9c92 100644 --- a/tooling/nargo_cli/Cargo.toml +++ b/tooling/nargo_cli/Cargo.toml @@ -72,13 +72,9 @@ assert_cmd = "2.0.8" assert_fs = "1.0.10" predicates = "2.1.5" fm.workspace = true -criterion = "0.5.0" +criterion.workspace = true +pprof.workspace = true paste = "1.0.14" -pprof = { version = "0.12", features = [ - "flamegraph", - "frame-pointer", - "criterion", -] } iai = "0.1.1" test-binary = "3.0.2"