From 5c22b4de3e2d2ea9039c1516381f1b17679d14bb Mon Sep 17 00:00:00 2001 From: andrejlukacovic <37964423+lukacan@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:01:02 +0100 Subject: [PATCH] Feat/toml params for fuzzer (#106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚧 arguments to hongfuzz within trdelnik toml * 🚧 Working on merge with CLI and give precedence * 🚧 Removed lazy static for config, modified precedence from cli and added logic for merge (may be changed) * ✅ added tests for config merge/precedence with cli, updated template and small changes in order to work with other command * 🐛 moved config * 🐛 fixed review + added tests for short flag without space * ♻️ Removed parse from CLI as later has precedence, renamed structs and modified tests * 🔥 unused extern crate * 🧑‍💻 Simplified vec to string loop and improved function name --------- Co-authored-by: lukacan --- crates/client/src/client.rs | 6 +- crates/client/src/commander.rs | 9 +- crates/client/src/config.rs | 199 +++++++++++++++++- .../client/src/templates/Trdelnik.toml.tmpl | 9 + examples/fuzzer/Trdelnik.toml | 7 + 5 files changed, 221 insertions(+), 9 deletions(-) diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index e5f06c01b..0ddef422e 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -1,4 +1,4 @@ -use crate::{config::CONFIG, Reader, TempClone}; +use crate::{config::Config, Reader, TempClone}; use anchor_client::{ anchor_lang::{ prelude::System, solana_program::program_pack::Pack, AccountDeserialize, Id, @@ -96,6 +96,8 @@ impl Client { /// Set `retry` to `true` when you want to wait for up to 15 seconds until /// the localnet is running (until 30 retries with 500ms delays are performed). pub async fn is_localnet_running(&self, retry: bool) -> bool { + let config = Config::new(); + let rpc_client = self .anchor_client .program(System::id()) @@ -103,7 +105,7 @@ impl Client { .async_rpc(); for _ in 0..(if retry { - CONFIG.test.validator_startup_timeout / RETRY_LOCALNET_EVERY_MILLIS + config.test.validator_startup_timeout / RETRY_LOCALNET_EVERY_MILLIS } else { 1 }) { diff --git a/crates/client/src/commander.rs b/crates/client/src/commander.rs index 6e37a869d..f9a2733d4 100644 --- a/crates/client/src/commander.rs +++ b/crates/client/src/commander.rs @@ -1,3 +1,4 @@ +use crate::config::Config; use crate::{ idl::{self, Idl}, program_client_generator, @@ -150,15 +151,21 @@ impl Commander { throw!(Error::TestingFailed); } } - /// Runs fuzzer on the given target. #[throws] pub async fn run_fuzzer(&self, target: String) { + let config = Config::new(); + + let hfuzz_run_args = std::env::var("HFUZZ_RUN_ARGS").unwrap_or_default(); + let fuzz_args = config.get_fuzz_args(hfuzz_run_args); + let cur_dir = Path::new(&self.root.to_string()).join(TESTS_WORKSPACE); if !cur_dir.try_exists()? { throw!(Error::NotInitialized); } + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) .current_dir(cur_dir) .arg("hfuzz") .arg("run") diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 2d95b0c14..d3712c181 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -1,5 +1,3 @@ -extern crate lazy_static; - use anyhow::Context; use fehler::throw; use serde::Deserialize; @@ -26,13 +24,18 @@ pub enum Error { pub struct Test { pub validator_startup_timeout: u64, } - #[derive(Default, Debug, Deserialize, Clone)] struct _Test { #[serde(default)] pub validator_startup_timeout: Option, } - +impl Default for Test { + fn default() -> Self { + Self { + validator_startup_timeout: 10_000, + } + } +} impl From<_Test> for Test { fn from(_t: _Test) -> Self { Self { @@ -40,22 +43,134 @@ impl From<_Test> for Test { } } } +#[derive(Debug, Deserialize, Clone)] +pub struct FuzzArg { + pub short_opt: Option, + pub long_opt: Option, + pub val: Option, +} +#[derive(Debug, Deserialize, Clone)] +pub struct Fuzz { + pub fuzz_args: Vec, +} +#[derive(Default, Debug, Deserialize, Clone)] +struct _Fuzz { + #[serde(default)] + /// Timeout in seconds (default: 10) + /// -t + pub timeout: Option, + #[serde(default)] + /// Number of fuzzing iterations (default: 0 [no limit]) + /// -N + pub iterations: Option, + #[serde(default)] + /// Don't close children's stdin, stdout, stderr; can be noisy + /// -Q + pub keep_output: Option, + #[serde(default)] + /// Disable ANSI console; use simple log output + /// -v + pub verbose: Option, + #[serde(default)] + /// Exit upon seeing the first crash (default: false) + /// --exit_upon_crash + pub exit_upon_crash: Option, +} +impl Default for Fuzz { + fn default() -> Self { + Self { + fuzz_args: vec![ + FuzzArg::new("-t", "--timeout", &10.to_string()), + FuzzArg::new("-N", "--iterations", &0.to_string()), + ], + } + } +} +impl From<_Fuzz> for Fuzz { + fn from(_f: _Fuzz) -> Self { + let mut _self = Self { fuzz_args: vec![] }; + + // timeout + let timeout = _f.timeout.unwrap_or(10); + _self + .fuzz_args + .push(FuzzArg::new("-t", "--timeout", &timeout.to_string())); + + // iterations + let iterations = _f.iterations.unwrap_or(0); + _self + .fuzz_args + .push(FuzzArg::new("-N", "--iterations", &iterations.to_string())); + + // keep_output + let keep_output = _f.keep_output.unwrap_or(false); + if keep_output { + _self + .fuzz_args + .push(FuzzArg::new("-Q", "--keep_output", "")); + } + + // verbose + let verbose = _f.verbose.unwrap_or(false); + if verbose { + _self.fuzz_args.push(FuzzArg::new("-v", "--verbose", "")); + } + + // exit_upon_crash + let exit_upon_crash = _f.exit_upon_crash.unwrap_or(false); + if exit_upon_crash { + _self + .fuzz_args + .push(FuzzArg::new("", "--exit_upon_crash", "")); + } + _self + } +} + +impl FuzzArg { + fn new(short_opt: &str, long_opt: &str, val: &str) -> Self { + let short_opt = if short_opt.is_empty() { + None + } else { + Some(short_opt.to_owned()) + }; + let long_opt = if long_opt.is_empty() { + None + } else { + Some(long_opt.to_owned()) + }; + let val = if val.is_empty() { + None + } else { + Some(val.to_owned()) + }; + Self { + short_opt, + long_opt, + val, + } + } +} #[derive(Debug, Deserialize, Clone)] pub struct Config { pub test: Test, + pub fuzz: Fuzz, } #[derive(Default, Debug, Deserialize, Clone)] struct _Config { #[serde(default)] pub test: Option<_Test>, + #[serde(default)] + pub fuzz: Option<_Fuzz>, } impl From<_Config> for Config { fn from(_c: _Config) -> Self { Self { test: _c.test.unwrap_or_default().into(), + fuzz: _c.fuzz.unwrap_or_default().into(), } } } @@ -94,8 +209,80 @@ impl Config { } throw!(Error::BadWorkspace) } + pub fn get_fuzz_args(self, cli_input: String) -> String { + // Tested on a few examples, HFUZZ_RUN_ARGS give precedence to the later arguments. + // so if HFUZZ_RUN_ARGS="-t 10 -t 15" -> timeout 15s is applied. + // That means we do not need to parse the arguments from the CLI; + // thus, we can simply append them at the end, and the CLI will have precedence. + + let mut args: Vec = self + .fuzz + .fuzz_args + .iter() + .map(|a| { + let val = a.val.to_owned().unwrap_or("".to_string()); + if let Some(o) = &a.short_opt { + format!("{} {}", o, val) + } else if let Some(o) = &a.long_opt { + format!("{} {}", o, val) + } else { + "".to_string() + } + }) + .collect(); + args.push(cli_input); + args.join(" ") + } } -lazy_static::lazy_static! { - pub static ref CONFIG: Config = Config::new(); +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_merge_and_precedence1() { + let config = Config { + test: Test::default(), + fuzz: Fuzz::default(), + }; + + let env_var_string = config.get_fuzz_args(String::default()); + assert_eq!(env_var_string, "-t 10 -N 0 "); + } + #[test] + fn test_merge_and_precedence2() { + let config = Config { + test: Test::default(), + fuzz: Fuzz::default(), + }; + + let env_var_string = config.get_fuzz_args("-t 0 -N10 --exit_upon_crash".to_string()); + + assert_eq!(env_var_string, "-t 10 -N 0 -t 0 -N10 --exit_upon_crash"); + } + #[test] + fn test_merge_and_precedence3() { + let config = Config { + test: Test::default(), + fuzz: Fuzz::default(), + }; + let env_var_string = + config.get_fuzz_args("-t 100 -N 5000 -Q -v --exit_upon_crash".to_string()); + assert_eq!( + env_var_string, + "-t 10 -N 0 -t 100 -N 5000 -Q -v --exit_upon_crash" + ); + } + #[test] + fn test_merge_and_precedence4() { + let config = Config { + test: Test::default(), + fuzz: Fuzz::default(), + }; + + let env_var_string = config.get_fuzz_args("-t 10 -N 500 -Q -v --exit_upon_crash -n 15 --mutations_per_run 8 --verifier -W random_dir --crashdir random_dir5 --run_time 666".to_string()); + assert_eq!( + env_var_string, + "-t 10 -N 0 -t 10 -N 500 -Q -v --exit_upon_crash -n 15 --mutations_per_run 8 --verifier -W random_dir --crashdir random_dir5 --run_time 666" + ); + } } diff --git a/crates/client/src/templates/Trdelnik.toml.tmpl b/crates/client/src/templates/Trdelnik.toml.tmpl index 0428ca653..023cf8a3e 100644 --- a/crates/client/src/templates/Trdelnik.toml.tmpl +++ b/crates/client/src/templates/Trdelnik.toml.tmpl @@ -1,2 +1,11 @@ [test] validator_startup_timeout = 15000 + + +# set for default values +[fuzz] +timeout = 10 +iterations = 0 +keep_output = false +verbose = false +exit_upon_crash = false diff --git a/examples/fuzzer/Trdelnik.toml b/examples/fuzzer/Trdelnik.toml index 0428ca653..357052e94 100644 --- a/examples/fuzzer/Trdelnik.toml +++ b/examples/fuzzer/Trdelnik.toml @@ -1,2 +1,9 @@ [test] validator_startup_timeout = 15000 + +[fuzz] +iterations = 500 +timeout = 25 +keep_output = false +verbose = false +exit_upon_crash = true