Skip to content

Commit

Permalink
Detect which reference examples should check/codegen/verify. (rust-la…
Browse files Browse the repository at this point in the history 
…ng#363)

* Copy options from the rust code blocks in the markdown files.

* Fix typos and add comments.
  • Loading branch information
@bdalrhm @tedinski
bdalrhm authored and tedinski committed Jul 28, 2021
1 parent ca36771 commit c550d95
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 85 deletions.
2 changes: 2 additions & 0 deletions scripts/rmc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ def compile_single_rust_file(input_filename, output_filename, verbose=False, deb
"-Z", f"symbol-mangling-version={mangler}",
"-Z", f"symbol_table_passes={' '.join(symbol_table_passes)}",
f"--cfg={RMC_CFG}", "-o", output_filename, input_filename]
if "RUSTFLAGS" in os.environ:
build_cmd += os.environ["RUSTFLAGS"].split(" ")
build_env = os.environ
if debug:
add_rmc_rustc_debug_to_env(build_env)
Expand Down
17 changes: 14 additions & 3 deletions src/tools/compiletest/src/runtest.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2394,12 +2394,12 @@ impl<'test> TestCx<'test> {

/// Adds rmc scripts directory to the `PATH` environment variable.
fn add_rmc_dir_to_path(&self, command: &mut Command) {
// If the PATH enviornment variable is already defined,
// If the PATH environment variable is already defined,
if let Some((key, val)) = env::vars().find(|(key, _)| key == "PATH") {
// Add the RMC scripts directory to the PATH.
command.env(key, format!("{}:{}", self.config.rmc_dir_path.to_str().unwrap(), val));
} else {
// Otherwise, insert PATH as a new enviornment variable and set its value to the RMC scripts directory.
// Otherwise, insert PATH as a new environment variable and set its value to the RMC scripts directory.
command.env(
String::from("PATH"),
String::from(self.config.rmc_dir_path.to_str().unwrap()),
Expand All @@ -2411,7 +2411,10 @@ impl<'test> TestCx<'test> {
/// error message is printed to stdout if the check result is not expected.
fn check(&self) {
let mut rustc = Command::new("rmc-rustc");
rustc.args(["-Z", "no-codegen"]).arg(&self.testpaths.file);
rustc
.args(self.props.compile_flags.clone())
.args(["-Z", "no-codegen"])
.arg(&self.testpaths.file);
self.add_rmc_dir_to_path(&mut rustc);
let proc_res = self.compose_and_run_compiler(rustc, None);
if self.props.rmc_panic_step == Some(RMCFailStep::Check) {
Expand All @@ -2431,6 +2434,7 @@ impl<'test> TestCx<'test> {
fn codegen(&self) {
let mut rustc = Command::new("rmc-rustc");
rustc
.args(self.props.compile_flags.clone())
.args(["-Z", "codegen-backend=gotoc", "--cfg=rmc", "--out-dir"])
.arg(self.output_base_dir())
.arg(&self.testpaths.file);
Expand Down Expand Up @@ -2461,6 +2465,13 @@ impl<'test> TestCx<'test> {
// 2. It may pass some options that do not make sense for RMC
// So we create our own command to execute RMC and pass it to self.compose_and_run_compiler(...) directly.
let mut rmc = Command::new("rmc");
// We cannot pass rustc flags directly to RMC. Instead, we add them
// to the current environment through the `RUSTFLAGS` environment
// variable. RMC recognizes the variable and adds those flags to its
// internal call to rustc.
if !self.props.compile_flags.is_empty() {
rmc.env("RUSTFLAGS", self.props.compile_flags.join(" "));
}
// Pass the test path along with RMC and CBMC flags parsed from comments at the top of the test file.
rmc.args(&self.props.rmc_flags)
.arg("--input")
Expand Down
210 changes: 128 additions & 82 deletions src/tools/dashboard/src/reference.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ use crate::dashboard;
use pulldown_cmark::{Parser, Tag};
use std::{
collections::HashMap,
env, fs,
env,
fmt::{Debug, Formatter, Result},
fs::{self, File},
hash::Hash,
io::{BufRead, BufReader},
path::{Path, PathBuf},
process::{Command, Stdio},
Expand All @@ -19,17 +22,18 @@ use std::{
/// code to corresponding directories where the extracted rust code should
/// reside.
fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
let summary_dir = summary_path.parent().unwrap().to_path_buf();
let start = "# The Rust Reference\n\n[Introduction](introduction.md)";
let summary = fs::read_to_string(summary_path).unwrap();
assert!(summary.starts_with(start), "Error: The start of the summary file changed.");
// Skip the title and introduction.
let n = Parser::new(start).count();
let parser = Parser::new(&summary).skip(n);
// Set "ref" as the root of the hierarchical path.
let mut hierarchy = PathBuf::from("ref");
let mut hierarchy: PathBuf = ["src", "test", "ref"].iter().collect();
let mut map = HashMap::new();
// Introduction is a especial case, so handle it separately.
map.insert(PathBuf::from("introduction.md"), hierarchy.join("Introduction"));
map.insert(summary_dir.join("introduction.md"), hierarchy.join("Introduction"));
for event in parser {
match event {
pulldown_cmark::Event::End(Tag::Item) => {
Expand All @@ -42,7 +46,9 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
// contain the title of the current chapter/section. So, we wait
// for the end of the link tag before adding the path and
// hierarchy of the current chapter/section to the map.
map.insert(path.split('/').collect(), hierarchy.clone());
let mut full_path = summary_dir.clone();
full_path.extend(path.split('/'));
map.insert(full_path, hierarchy.clone());
}
pulldown_cmark::Event::Text(text) => {
// Add the current chapter/section title to the hierarchy.
Expand All @@ -54,87 +60,131 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
map
}

/// Extracts examples from the given relative `paths` in the `book_dir` and
/// saves them in `gen_dir`.
fn extract_examples(paths: Vec<&PathBuf>, book_dir: &Path, gen_dir: &Path) {
for path in paths {
let mut cmd = Command::new("rustdoc");
cmd.args([
"+nightly",
"--test",
"-Z",
"unstable-options",
book_dir.join(path).to_str().unwrap(),
"--test-builder",
&["src", "tools", "dashboard", "print.sh"]
.iter()
.collect::<PathBuf>()
.to_str()
.unwrap(),
"--persist-doctests",
gen_dir.to_str().unwrap(),
"--no-run",
]);
cmd.stdout(Stdio::null());
cmd.spawn().unwrap().wait().unwrap();
/// The data structure represents the "full" path to examples in the Rust books.
#[derive(PartialEq, Eq, Hash)]
struct Example {
/// Path to the markdown file containing the example.
path: PathBuf,
/// Line number of the code block introducing the example.
line: usize,
}

impl Example {
/// Creates a new [`Example`] instance representing "full" path to the
/// Rust example.
fn new(path: PathBuf, line: usize) -> Example {
Example { path, line }
}
}

/// Copies the extracted rust code in `from_dir` to `src/test` following the
/// hierarchy specified by `map`.
fn organize_examples(map: &HashMap<PathBuf, PathBuf>, book_dir: &Path, from_dir: &Path) {
// The names of the extracted examples generated by `rustdoc` have the
// format `<path>_<line-num>_<test-num>` where occurrences of '/', '-', and
// '.' in <path> are replaced by '_'. This transformation is not injective,
// so we cannot map those names back to the original markdown file path.
// Instead, we apply the same transformation on the keys of `map` in the for
// loop below and lookup <path> in those modified keys.
let mut modified_map = HashMap::new();
for (path, hierarchy) in map.iter() {
modified_map.insert(
book_dir.join(path).to_str().unwrap().replace(&['\\', '/', '-', '.'][..], "_"),
hierarchy.clone(),
);
impl Debug for Example {
fn fmt(&self, f: &mut Formatter<'_>) -> Result {
f.write_fmt(format_args!("{}:{}", self.path.to_str().unwrap(), self.line))
}
for dir in from_dir.read_dir().unwrap() {
let dir = dir.unwrap().path();
}

/// Extracts examples from the markdown files specified by each key in the given
/// `map` and saves them in the directory specified by the corresponding value.
/// Returns a mapping from the original location of **_each_** example to the
/// path it was extracted to.
fn extract_examples(par_map: HashMap<PathBuf, PathBuf>) -> HashMap<Example, PathBuf> {
let mut full_map = HashMap::new();
for (par_from, par_to) in par_map {
let pairs = extract(&par_from, &par_to);
for (key, val) in pairs {
full_map.insert(key, val);
}
}
full_map
}

/// Extracts examples from the markdown files specified by `par_from` and saves
/// them in the directory specified by `par_to`. Returns a mapping from the
/// original location of **_each_** example to the path it was extracted to.
fn extract(par_from: &Path, par_to: &Path) -> Vec<(Example, PathBuf)> {
let build_dir = &env::var("BUILD_DIR").unwrap();
let triple = &env::var("TRIPLE").unwrap();
// Create a temporary directory to save the files generated by `rustdoc`.
let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect();
fs::create_dir_all(&gen_dir).unwrap();
let mut cmd = Command::new("rustdoc");
cmd.args([
"+nightly",
"--test",
"-Z",
"unstable-options",
par_from.to_str().unwrap(),
"--test-builder",
&["src", "tools", "dashboard", "print.sh"].iter().collect::<PathBuf>().to_str().unwrap(),
"--persist-doctests",
gen_dir.to_str().unwrap(),
"--no-run",
]);
cmd.stdout(Stdio::null());
cmd.spawn().unwrap().wait().unwrap();
// Mapping from path and line number of rust example to where it was extracted to.
let mut pairs = Vec::new();

for dir in gen_dir.read_dir().unwrap() {
// Some directories do not contain tests because the markdown file
// instructs `rustdoc` to ignore those tests.
if let Some(example) = dir.read_dir().unwrap().next() {
let example = example.unwrap().path();
copy(&example, &modified_map);
// instructs `rustdoc` to "ignore" those tests.
let dir = dir.unwrap().path();
if let Some(from) = dir.read_dir().unwrap().next() {
// The path to each example extracted by `rustdoc` has the form:
// <from> = `<gen_dir>/<par_from>_<line>_<test-num>/rust_out`
// where occurrences of '/', '-', and '.' in <par_from> are replaced
// by '_'. We copy the file in this path to a new path of the form:
// <to> = `<par_to>/<line>.rs`
// We omit <test-num> because all tests have the same number, 0.
let from = from.unwrap().path();
let path_line_test = dir.file_name().unwrap().to_str().unwrap();
let splits: Vec<_> = path_line_test.rsplitn(3, '_').collect();
let line: usize = splits[1].parse().unwrap();
let to = par_to.join(format!("{}.rs", line));
fs::create_dir_all(par_to).unwrap();
fs::copy(&from, &to).unwrap();
pairs.push((Example::new(par_from.to_path_buf(), line), to));
}
}
// Delete the temporary directory.
fs::remove_dir_all(gen_dir).unwrap();
pairs
}

/// Copy the file specified by `from` to the corresponding location specified by
/// `map`.
fn copy(from: &Path, map: &HashMap<String, PathBuf>) {
// The path specified by `from` has the form:
// `build/<triple>/dashboard/ref/<key>_<line-num>_<test-num>/rust_out`
// We copy the file in this path to a new path of the form:
// `src/test/<val>/<line-num>.rs
// where `map[<key>] == <val>`. We omit <test-num> because all tests have
// the same number, 0.
// Extract `<key>_<line-num>_<test-num>`.
let key_line_test = from.parent().unwrap().file_name().unwrap().to_str().unwrap();
// Extract <key> and <line-num> from `key_line_test` to get <val> and
// construct destination path.
let splits: Vec<_> = key_line_test.rsplitn(3, '_').collect();
let key = splits[2];
let line = splits[1];
let val = &map[key];
let name = &format!("{}.rs", line);
let to = Path::new("src").join("test").join(val).join(name);
fs::create_dir_all(to.parent().unwrap()).unwrap();
fs::copy(&from, &to).unwrap();
/// Prepends the text in `path` with the given `text`.
fn prepend_text(path: &Path, text: &str) {
let code = fs::read_to_string(&path).unwrap();
let code = format!("{}\n{}", text, code);
fs::write(&path, code).unwrap();
}

/// Pre-processes the tests in the specified `paths` before running them with
/// `compiletest`.
fn preprocess_examples(_paths: Vec<&PathBuf>) {
// For now, we will only pre-process the tests that cause infinite loops.
// TODO: properly implement this step (see issue #324).
/// Pre-processes the examples in `map` before running them with `compiletest`.
fn preprocess_examples(map: &HashMap<Example, PathBuf>) {
// Copy compiler configurations specified in the original markdown code
// block.
for (from, to) in map.iter() {
let file = File::open(&from.path).unwrap();
// Skip to the first line of the example code block.
// Line numbers in files start with 1 but `nth(...)` starts with 0.
// Subtract 1 to account for the difference.
let line = BufReader::new(file).lines().nth(from.line - 1).unwrap().unwrap();
if line.contains("edition2015") {
prepend_text(to, "// compile-flags: --edition 2015");
} else {
prepend_text(to, "// compile-flags: --edition 2018");
}
// Most examples with `compile_fail` configuration fail because of
// check errors.
if line.contains("compile_fail") {
prepend_text(to, "// rmc-check-fail");
}
// RMC should catch run-time errors.
if line.contains("should_panic") {
prepend_text(to, "// rmc-verify-fail");
}
}
// For now, we will only manually pre-process the tests that cause infinite loops.
// TODO: Add support for manually adding options and assertions (see issue #324).
let loop_tests: [PathBuf; 4] = [
["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(),
["src", "test", "ref", "Linkage", "190.rs"].iter().collect(),
Expand Down Expand Up @@ -193,7 +243,6 @@ fn run_examples(suite: &str, log_path: &Path) {
]);
cmd.env_clear().envs(filtered_env);
cmd.stdout(Stdio::null());

cmd.spawn().unwrap().wait().unwrap();
}

Expand Down Expand Up @@ -258,20 +307,17 @@ fn display_dashboard(dashboard: dashboard::Tree) {
/// displays their results in a terminal dashboard.
pub fn display_reference_dashboard() {
let summary_path: PathBuf = ["src", "doc", "reference", "src", "SUMMARY.md"].iter().collect();
let ref_dir: PathBuf = ["src", "doc", "reference", "src"].iter().collect();
let build_dir = &env::var("BUILD_DIR").unwrap();
let triple = &env::var("TRIPLE").unwrap();
let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect();
let log_path: PathBuf = [build_dir, triple, "dashboard", "ref.log"].iter().collect();
// Parse the chapter/section hierarchy from the table of contents in The
// Rust Reference.
let map = parse_hierarchy(&summary_path);
// Extract examples from The Rust Reference.
extract_examples(map.keys().collect(), &ref_dir, &gen_dir);
// Reorganize those examples following the The Rust Reference hierarchy.
organize_examples(&map, &ref_dir, &gen_dir);
// Extract examples from The Rust Reference, organize them following the
// partial hierarchy in map, and return the full hierarchy map.
let map = extract_examples(map);
// Pre-process the examples before running them through `compiletest`.
preprocess_examples(map.values().collect());
preprocess_examples(&map);
// Run `compiletest` on the reference examples.
run_examples("ref", &log_path);
// Parse `compiletest` log file.
Expand Down

0 comments on commit c550d95

Please sign in to comment.