Skip to content

Commit

Permalink
chore(ci): use random seed for sqlsmith cron test (#3688)
Browse files Browse the repository at this point in the history 
* add env RW_RANDOM_SEED_SQLSMITH

* test

* fix

* add random seed on main cron

* cargo fmt
  • Loading branch information
@huangjw806
huangjw806 authored Jul 6, 2022
1 parent 7f9ad22 commit 4952c5c
Show file tree
Hide file tree
Showing 2 changed files with 126 additions and 120 deletions.
1 change: 1 addition & 0 deletions ci/workflows/main-cron.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ steps:
config: ci/docker-compose.yml
environment:
- CODECOV_TOKEN
- RW_RANDOM_SEED_SQLSMITH: true
timeout_in_minutes: 15
retry: *auto-retry

Expand Down
245 changes: 125 additions & 120 deletions src/tests/sqlsmith/tests/test_runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,141 +12,146 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#[cfg(test)]
mod tests {
use std::panic;
use std::sync::Arc;
#![feature(let_chains)]

use rand::SeedableRng;
use risingwave_frontend::binder::Binder;
use risingwave_frontend::handler::create_table;
use risingwave_frontend::planner::Planner;
use risingwave_frontend::session::{OptimizerContext, OptimizerContextRef, SessionImpl};
use risingwave_frontend::test_utils::LocalFrontend;
use risingwave_frontend::FrontendOpts;
use risingwave_sqlparser::ast::Statement;
use risingwave_sqlparser::parser::Parser;
use risingwave_sqlsmith::{sql_gen, Table};
use std::sync::Arc;
use std::{env, panic};

/// Create the tables defined in testdata.
async fn create_tables(session: Arc<SessionImpl>) -> Vec<Table> {
let sql = std::fs::read_to_string("tests/testdata/tpch.sql").unwrap();
let statements =
Parser::parse_sql(&sql).unwrap_or_else(|_| panic!("Failed to parse SQL: {}", sql));
use rand::SeedableRng;
use risingwave_frontend::binder::Binder;
use risingwave_frontend::handler::create_table;
use risingwave_frontend::planner::Planner;
use risingwave_frontend::session::{OptimizerContext, OptimizerContextRef, SessionImpl};
use risingwave_frontend::test_utils::LocalFrontend;
use risingwave_frontend::FrontendOpts;
use risingwave_sqlparser::ast::Statement;
use risingwave_sqlparser::parser::Parser;
use risingwave_sqlsmith::{sql_gen, Table};

let mut tables = vec![];
for s in statements.into_iter() {
match s {
Statement::CreateTable {
name,
columns,
/// Create the tables defined in testdata.
async fn create_tables(session: Arc<SessionImpl>) -> Vec<Table> {
let sql = std::fs::read_to_string("tests/testdata/tpch.sql").unwrap();
let statements =
Parser::parse_sql(&sql).unwrap_or_else(|_| panic!("Failed to parse SQL: {}", sql));

let mut tables = vec![];
for s in statements.into_iter() {
match s {
Statement::CreateTable {
name,
columns,
with_options,
..
} => {
let context = OptimizerContext::new(session.clone(), Arc::from(sql.clone()));
create_table::handle_create_table(
context,
name.clone(),
columns.clone(),
with_options,
..
} => {
let context = OptimizerContext::new(session.clone(), Arc::from(sql.clone()));
create_table::handle_create_table(
context,
name.clone(),
columns.clone(),
with_options,
)
.await
.unwrap();
tables.push(Table {
name: name.0[0].value.clone(),
columns: columns.iter().map(|c| c.clone().into()).collect(),
})
}
_ => panic!("Unexpected statement: {}", s),
)
.await
.unwrap();
tables.push(Table {
name: name.0[0].value.clone(),
columns: columns.iter().map(|c| c.clone().into()).collect(),
})
}
_ => panic!("Unexpected statement: {}", s),
}
tables
}
tables
}

async fn run_sqlsmith_with_seed(seed: u64) {
let frontend = LocalFrontend::new(FrontendOpts::default()).await;
let session = frontend.session_ref();
let tables = create_tables(session.clone()).await;
let mut rng = rand::rngs::SmallRng::seed_from_u64(seed);
async fn run_sqlsmith_with_seed(seed: u64) {
let frontend = LocalFrontend::new(FrontendOpts::default()).await;
let session = frontend.session_ref();
let tables = create_tables(session.clone()).await;

for _ in 0..512 {
let sql = sql_gen(&mut rng, tables.clone());
let mut rng;
if let Ok(x) = env::var("RW_RANDOM_SEED_SQLSMITH") && x == "true" {
rng = rand::rngs::SmallRng::from_entropy();
} else {
rng = rand::rngs::SmallRng::seed_from_u64(seed);
}

let sql_copy = sql.clone();
panic::set_hook(Box::new(move |e| {
println!("Panic on SQL:\n{}\nReason:\n{}", sql_copy.clone(), e);
}));
for _ in 0..512 {
let sql = sql_gen(&mut rng, tables.clone());

// The generated SQL must be parsable.
let statements =
Parser::parse_sql(&sql).unwrap_or_else(|_| panic!("Failed to parse SQL: {}", sql));
let stmt = statements[0].clone();
let context: OptimizerContextRef =
OptimizerContext::new(session.clone(), Arc::from(sql.clone())).into();
let sql_copy = sql.clone();
panic::set_hook(Box::new(move |e| {
println!("Panic on SQL:\n{}\nReason:\n{}", sql_copy.clone(), e);
}));

match stmt.clone() {
Statement::Query(_) => {
let mut binder = Binder::new(
session.env().catalog_reader().read_guard(),
session.database().to_string(),
);
let bound = binder
.bind(stmt.clone())
.unwrap_or_else(|e| panic!("Failed to bind:\n{}\nReason:\n{}", sql, e));
let mut planner = Planner::new(context.clone());
let logical_plan = planner.plan(bound).unwrap_or_else(|e| {
panic!("Failed to generate logical plan:\n{}\nReason:\n{}", sql, e)
});
logical_plan.gen_batch_query_plan().unwrap_or_else(|e| {
panic!("Failed to generate batch plan:\n{}\nReason:\n{}", sql, e)
});
}
_ => unreachable!(),
// The generated SQL must be parsable.
let statements =
Parser::parse_sql(&sql).unwrap_or_else(|_| panic!("Failed to parse SQL: {}", sql));
let stmt = statements[0].clone();
let context: OptimizerContextRef =
OptimizerContext::new(session.clone(), Arc::from(sql.clone())).into();

match stmt.clone() {
Statement::Query(_) => {
let mut binder = Binder::new(
session.env().catalog_reader().read_guard(),
session.database().to_string(),
);
let bound = binder
.bind(stmt.clone())
.unwrap_or_else(|e| panic!("Failed to bind:\n{}\nReason:\n{}", sql, e));
let mut planner = Planner::new(context.clone());
let logical_plan = planner.plan(bound).unwrap_or_else(|e| {
panic!("Failed to generate logical plan:\n{}\nReason:\n{}", sql, e)
});
logical_plan.gen_batch_query_plan().unwrap_or_else(|e| {
panic!("Failed to generate batch plan:\n{}\nReason:\n{}", sql, e)
});
}
_ => unreachable!(),
}
}
}

macro_rules! generate_sqlsmith_test {
($seed:expr) => {
paste::paste! {
#[tokio::test]
async fn [<run_sqlsmith_on_frontend_ $seed>]() {
run_sqlsmith_with_seed($seed).await;
}
macro_rules! generate_sqlsmith_test {
($seed:expr) => {
paste::paste! {
#[tokio::test]
async fn [<run_sqlsmith_on_frontend_ $seed>]() {
run_sqlsmith_with_seed($seed).await;
}
};
}

generate_sqlsmith_test! { 0 }
generate_sqlsmith_test! { 1 }
generate_sqlsmith_test! { 2 }
generate_sqlsmith_test! { 3 }
generate_sqlsmith_test! { 4 }
generate_sqlsmith_test! { 5 }
generate_sqlsmith_test! { 6 }
generate_sqlsmith_test! { 7 }
generate_sqlsmith_test! { 8 }
generate_sqlsmith_test! { 9 }
generate_sqlsmith_test! { 10 }
generate_sqlsmith_test! { 11 }
generate_sqlsmith_test! { 12 }
generate_sqlsmith_test! { 13 }
generate_sqlsmith_test! { 14 }
generate_sqlsmith_test! { 15 }
generate_sqlsmith_test! { 16 }
generate_sqlsmith_test! { 17 }
generate_sqlsmith_test! { 18 }
generate_sqlsmith_test! { 19 }
generate_sqlsmith_test! { 20 }
generate_sqlsmith_test! { 21 }
generate_sqlsmith_test! { 22 }
generate_sqlsmith_test! { 23 }
generate_sqlsmith_test! { 24 }
generate_sqlsmith_test! { 25 }
generate_sqlsmith_test! { 26 }
generate_sqlsmith_test! { 27 }
generate_sqlsmith_test! { 28 }
generate_sqlsmith_test! { 29 }
generate_sqlsmith_test! { 30 }
generate_sqlsmith_test! { 31 }
}
};
}

generate_sqlsmith_test! { 0 }
generate_sqlsmith_test! { 1 }
generate_sqlsmith_test! { 2 }
generate_sqlsmith_test! { 3 }
generate_sqlsmith_test! { 4 }
generate_sqlsmith_test! { 5 }
generate_sqlsmith_test! { 6 }
generate_sqlsmith_test! { 7 }
generate_sqlsmith_test! { 8 }
generate_sqlsmith_test! { 9 }
generate_sqlsmith_test! { 10 }
generate_sqlsmith_test! { 11 }
generate_sqlsmith_test! { 12 }
generate_sqlsmith_test! { 13 }
generate_sqlsmith_test! { 14 }
generate_sqlsmith_test! { 15 }
generate_sqlsmith_test! { 16 }
generate_sqlsmith_test! { 17 }
generate_sqlsmith_test! { 18 }
generate_sqlsmith_test! { 19 }
generate_sqlsmith_test! { 20 }
generate_sqlsmith_test! { 21 }
generate_sqlsmith_test! { 22 }
generate_sqlsmith_test! { 23 }
generate_sqlsmith_test! { 24 }
generate_sqlsmith_test! { 25 }
generate_sqlsmith_test! { 26 }
generate_sqlsmith_test! { 27 }
generate_sqlsmith_test! { 28 }
generate_sqlsmith_test! { 29 }
generate_sqlsmith_test! { 30 }
generate_sqlsmith_test! { 31 }

0 comments on commit 4952c5c

Please sign in to comment.