From abd73e16d4e9d2738695b91d6071765194e8cb0c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 18 Nov 2024 15:52:03 +0100 Subject: [PATCH 1/4] Revert "Revert "Rust: allow to specify more cargo configuration options"" --- Cargo.lock | 2 + rust/codeql-extractor.yml | 19 ++++ rust/extractor/Cargo.toml | 2 + rust/extractor/macros/src/lib.rs | 67 +++++++++---- rust/extractor/src/config.rs | 97 ++++++++++++++++++- rust/extractor/src/main.rs | 6 +- rust/extractor/src/rust_analyzer.rs | 11 +-- .../integration-tests/options/cfg/Cargo.toml | 5 + .../options/cfg/functions.expected | 3 + .../options/cfg/functions.override.expected | 4 + .../options/cfg/functions.ql | 5 + .../integration-tests/options/cfg/src/lib.rs | 20 ++++ .../options/cfg/test_cfg_overrides_option.py | 19 ++++ .../options/features/Cargo.toml | 9 ++ .../options/features/functions.all.expected | 3 + .../options/features/functions.bar.expected | 2 + .../options/features/functions.expected | 1 + .../options/features/functions.foo.expected | 2 + .../options/features/functions.ql | 5 + .../options/features/src/lib.rs | 7 ++ .../options/features/test_features_option.py | 17 ++++ .../options/target/Cargo.toml | 5 + .../options/target/functions.Darwin.expected | 1 + .../options/target/functions.Linux.expected | 1 + .../options/target/functions.Windows.expected | 1 + .../options/target/functions.ql | 5 + .../options/target/src/lib.rs | 8 ++ .../options/target/test_target_option.py | 19 ++++ 28 files changed, 313 insertions(+), 33 deletions(-) create mode 100644 rust/ql/integration-tests/options/cfg/Cargo.toml create mode 100644 rust/ql/integration-tests/options/cfg/functions.expected create mode 100644 rust/ql/integration-tests/options/cfg/functions.override.expected create mode 100644 rust/ql/integration-tests/options/cfg/functions.ql create mode 100644 rust/ql/integration-tests/options/cfg/src/lib.rs create mode 100644 rust/ql/integration-tests/options/cfg/test_cfg_overrides_option.py create mode 100644 rust/ql/integration-tests/options/features/Cargo.toml create mode 100644 rust/ql/integration-tests/options/features/functions.all.expected create mode 100644 rust/ql/integration-tests/options/features/functions.bar.expected create mode 100644 rust/ql/integration-tests/options/features/functions.expected create mode 100644 rust/ql/integration-tests/options/features/functions.foo.expected create mode 100644 rust/ql/integration-tests/options/features/functions.ql create mode 100644 rust/ql/integration-tests/options/features/src/lib.rs create mode 100644 rust/ql/integration-tests/options/features/test_features_option.py create mode 100644 rust/ql/integration-tests/options/target/Cargo.toml create mode 100644 rust/ql/integration-tests/options/target/functions.Darwin.expected create mode 100644 rust/ql/integration-tests/options/target/functions.Linux.expected create mode 100644 rust/ql/integration-tests/options/target/functions.Windows.expected create mode 100644 rust/ql/integration-tests/options/target/functions.ql create mode 100644 rust/ql/integration-tests/options/target/src/lib.rs create mode 100644 rust/ql/integration-tests/options/target/test_target_option.py diff --git a/Cargo.lock b/Cargo.lock index ea1bca33cd36..1d5b8824c84a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,10 +389,12 @@ dependencies = [ "log", "num-traits", "ra_ap_base_db", + "ra_ap_cfg", "ra_ap_hir", "ra_ap_hir_def", "ra_ap_hir_expand", "ra_ap_ide_db", + "ra_ap_intern", "ra_ap_load-cargo", "ra_ap_parser", "ra_ap_paths", diff --git a/rust/codeql-extractor.yml b/rust/codeql-extractor.yml index c7558c17efef..2f726a10adac 100644 --- a/rust/codeql-extractor.yml +++ b/rust/codeql-extractor.yml @@ -35,3 +35,22 @@ options: reduce execution time of consecutive extractor runs. By default, a new scratch directory is used for each run. type: string + cargo_target: + title: Target architecture + description: > + Target architecture to use for analysis, analogous to `cargo --target`. By + default the host architecture is used. + type: string + cargo_features: + title: Cargo features to turn on + description: > + Comma-separated list of features to turn on. If any value is `*` all features + are turned on. By default only default cargo features are enabled. Can be + repeated. + type: array + cargo_cfg_overrides: + title: Cargo cfg overrides + description: > + Comma-separated list of cfg settings to enable, or disable if prefixed with `-`. + Can be repeated. + type: array diff --git a/rust/extractor/Cargo.toml b/rust/extractor/Cargo.toml index d6d8519b1a84..787ad7fda844 100644 --- a/rust/extractor/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -22,6 +22,8 @@ ra_ap_syntax = "0.0.232" ra_ap_vfs = "0.0.232" ra_ap_parser = "0.0.232" ra_ap_span = "0.0.232" +ra_ap_cfg = "0.0.232" +ra_ap_intern = "0.0.232" serde = "1.0.209" serde_with = "3.9.0" stderrlog = "0.6.0" diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs index c9771a181de5..c70856aad9f3 100644 --- a/rust/extractor/macros/src/lib.rs +++ b/rust/extractor/macros/src/lib.rs @@ -1,11 +1,36 @@ use proc_macro::TokenStream; use quote::{format_ident, quote}; +use syn::{Ident, Type}; + +fn get_type_tip(t: &Type) -> Option<&Ident> { + let syn::Type::Path(path) = t else { + return None; + }; + let segment = path.path.segments.last()?; + Some(&segment.ident) +} /// Allow all fields in the extractor config to be also overrideable by extractor CLI flags #[proc_macro_attribute] pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream { let ast = syn::parse_macro_input!(item as syn::ItemStruct); let name = &ast.ident; + let fields = ast + .fields + .iter() + .map(|f| { + if f.ident.as_ref().is_some_and(|i| i != "inputs") + && get_type_tip(&f.ty).is_some_and(|i| i == "Vec") + { + quote! { + #[serde(deserialize_with="deserialize_newline_or_comma_separated")] + #f + } + } else { + quote! { #f } + } + }) + .collect::>(); let cli_name = format_ident!("Cli{}", name); let cli_fields = ast .fields @@ -13,35 +38,37 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea .map(|f| { let id = f.ident.as_ref().unwrap(); let ty = &f.ty; - if let syn::Type::Path(p) = ty { - if p.path.is_ident(&format_ident!("bool")) { - return quote! { - #[arg(long)] - #[serde(skip_serializing_if="<&bool>::not")] - #id: bool, - }; + let type_tip = get_type_tip(ty); + if type_tip.is_some_and(|i| i == "bool") { + quote! { + #[arg(long)] + #[serde(skip_serializing_if="<&bool>::not")] + #id: bool } - if p.path.segments.len() == 1 && p.path.segments[0].ident == "Option" { - return quote! { - #[arg(long)] - #id: #ty, - }; + } else if type_tip.is_some_and(|i| i == "Option") { + quote! { + #[arg(long)] + #f } - } - if id == &format_ident!("verbose") { + } else if id == &format_ident!("verbose") { quote! { #[arg(long, short, action=clap::ArgAction::Count)] #[serde(skip_serializing_if="u8::is_zero")] - #id: u8, + #id: u8 } } else if id == &format_ident!("inputs") { quote! { - #id: #ty, + #f + } + } else if type_tip.is_some_and(|i| i == "Vec") { + quote! { + #[arg(long)] + #id: Option } } else { quote! { #[arg(long)] - #id: Option<#ty>, + #id: Option<#ty> } } }) @@ -66,7 +93,9 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea let gen = quote! { #[serde_with::apply(_ => #[serde(default)])] #[derive(Deserialize, Default)] - #ast + pub struct #name { + #(#fields),* + } impl Debug for #name { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -80,7 +109,7 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea #[derive(clap::Parser, Serialize)] #[command(about, long_about = None)] struct #cli_name { - #(#cli_fields)* + #(#cli_fields),* } }; gen.into() diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 8520df189304..dfa9f5d37d20 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -7,9 +7,14 @@ use figment::{ Figment, }; use itertools::Itertools; +use log::warn; use num_traits::Zero; +use ra_ap_cfg::{CfgAtom, CfgDiff}; +use ra_ap_intern::Symbol; +use ra_ap_paths::Utf8PathBuf; +use ra_ap_project_model::{CargoConfig, CargoFeatures, CfgOverrides, RustLibSource}; use rust_extractor_macros::extractor_cli_config; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use std::fmt::Debug; use std::ops::Not; use std::path::PathBuf; @@ -32,12 +37,23 @@ impl From for trap::Compression { } } +// required by the extractor_cli_config macro. +fn deserialize_newline_or_comma_separated<'a, D: Deserializer<'a>, T: for<'b> From<&'b str>>( + deserializer: D, +) -> Result, D::Error> { + let value = String::deserialize(deserializer)?; + Ok(value.split(['\n', ',']).map(T::from).collect()) +} + #[extractor_cli_config] pub struct Config { pub scratch_dir: PathBuf, pub trap_dir: PathBuf, pub source_archive_dir: PathBuf, pub cargo_target_dir: Option, + pub cargo_target: Option, + pub cargo_features: Vec, + pub cargo_cfg_overrides: Vec, pub verbose: u8, pub compression: Compression, pub inputs: Vec, @@ -52,7 +68,7 @@ impl Config { .context("expanding parameter files")?; let cli_args = CliConfig::parse_from(args); let mut figment = Figment::new() - .merge(Env::prefixed("CODEQL_")) + .merge(Env::raw().only(["CODEQL_VERBOSE"].as_slice())) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_")) .merge(Serialized::defaults(cli_args)); @@ -78,4 +94,81 @@ impl Config { } figment.extract().context("loading configuration") } + + pub fn to_cargo_config(&self) -> CargoConfig { + let sysroot = Some(RustLibSource::Discover); + + let target_dir = self + .cargo_target_dir + .clone() + .unwrap_or_else(|| self.scratch_dir.join("target")); + let target_dir = Utf8PathBuf::from_path_buf(target_dir).ok(); + + let features = if self.cargo_features.is_empty() { + Default::default() + } else if self.cargo_features.contains(&"*".to_string()) { + CargoFeatures::All + } else { + CargoFeatures::Selected { + features: self.cargo_features.clone(), + no_default_features: false, + } + }; + + let target = self.cargo_target.clone(); + + let cfg_overrides = to_cfg_overrides(&self.cargo_cfg_overrides); + + CargoConfig { + sysroot, + target_dir, + features, + target, + cfg_overrides, + ..Default::default() + } + } +} + +fn to_cfg_override(spec: &str) -> CfgAtom { + if let Some((key, value)) = spec.split_once("=") { + CfgAtom::KeyValue { + key: Symbol::intern(key), + value: Symbol::intern(value), + } + } else { + CfgAtom::Flag(Symbol::intern(spec)) + } +} + +fn to_cfg_overrides(specs: &Vec) -> CfgOverrides { + let mut enabled_cfgs = Vec::new(); + let mut disabled_cfgs = Vec::new(); + let mut has_test_explicitly_enabled = false; + for spec in specs { + if spec.starts_with("-") { + disabled_cfgs.push(to_cfg_override(&spec[1..])); + } else { + enabled_cfgs.push(to_cfg_override(spec)); + if spec == "test" { + has_test_explicitly_enabled = true; + } + } + } + if !has_test_explicitly_enabled { + disabled_cfgs.push(to_cfg_override("test")); + } + if let Some(global) = CfgDiff::new(enabled_cfgs, disabled_cfgs) { + CfgOverrides { + global, + ..Default::default() + } + } else { + warn!("non-disjoint cfg overrides, ignoring: {}", specs.join(", ")); + CfgOverrides { + global: CfgDiff::new(Vec::new(), vec![to_cfg_override("test")]) + .expect("disabling one cfg should always succeed"), + ..Default::default() + } + } } diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index 1f963a0990e5..ecbdc965f8e7 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -130,11 +130,9 @@ fn main() -> anyhow::Result<()> { } extractor.extract_without_semantics(file, "no manifest found"); } - let target_dir = &cfg - .cargo_target_dir - .unwrap_or_else(|| cfg.scratch_dir.join("target")); + let cargo_config = cfg.to_cargo_config(); for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) { - if let Some((ref db, ref vfs)) = RustAnalyzer::load_workspace(manifest, target_dir) { + if let Some((ref db, ref vfs)) = RustAnalyzer::load_workspace(manifest, &cargo_config) { let semantics = Semantics::new(db); for file in files { let Some(id) = path_to_file_id(file, vfs) else { diff --git a/rust/extractor/src/rust_analyzer.rs b/rust/extractor/src/rust_analyzer.rs index ddbc16ab6a3a..735bacb27c12 100644 --- a/rust/extractor/src/rust_analyzer.rs +++ b/rust/extractor/src/rust_analyzer.rs @@ -7,7 +7,6 @@ use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice use ra_ap_paths::Utf8PathBuf; use ra_ap_project_model::CargoConfig; use ra_ap_project_model::ProjectManifest; -use ra_ap_project_model::RustLibSource; use ra_ap_span::Edition; use ra_ap_span::EditionedFileId; use ra_ap_span::TextRange; @@ -20,6 +19,7 @@ use ra_ap_vfs::{AbsPathBuf, FileId}; use std::borrow::Cow; use std::path::{Path, PathBuf}; use triomphe::Arc; + pub enum RustAnalyzer<'a> { WithSemantics { vfs: &'a Vfs, @@ -45,13 +45,8 @@ pub struct ParseResult<'a> { impl<'a> RustAnalyzer<'a> { pub fn load_workspace( project: &ProjectManifest, - target_dir: &Path, + config: &CargoConfig, ) -> Option<(RootDatabase, Vfs)> { - let config = CargoConfig { - sysroot: Some(RustLibSource::Discover), - target_dir: ra_ap_paths::Utf8PathBuf::from_path_buf(target_dir.to_path_buf()).ok(), - ..Default::default() - }; let progress = |t| (log::trace!("progress: {}", t)); let load_config = LoadCargoConfig { load_out_dirs_from_check: true, @@ -60,7 +55,7 @@ impl<'a> RustAnalyzer<'a> { }; let manifest = project.manifest_path(); - match load_workspace_at(manifest.as_ref(), &config, &load_config, &progress) { + match load_workspace_at(manifest.as_ref(), config, &load_config, &progress) { Ok((db, vfs, _macro_server)) => Some((db, vfs)), Err(err) => { log::error!("failed to load workspace for {}: {}", manifest, err); diff --git a/rust/ql/integration-tests/options/cfg/Cargo.toml b/rust/ql/integration-tests/options/cfg/Cargo.toml new file mode 100644 index 000000000000..c5cbf8722f1f --- /dev/null +++ b/rust/ql/integration-tests/options/cfg/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +[package] +name = "cfg" +version = "0.1.0" +edition = "2021" diff --git a/rust/ql/integration-tests/options/cfg/functions.expected b/rust/ql/integration-tests/options/cfg/functions.expected new file mode 100644 index 000000000000..a2042c5e4c49 --- /dev/null +++ b/rust/ql/integration-tests/options/cfg/functions.expected @@ -0,0 +1,3 @@ +| src/lib.rs:7:1:8:19 | cfg_no_flag | +| src/lib.rs:10:1:11:18 | cfg_no_key | +| src/lib.rs:16:1:17:24 | pointer_width_64 | diff --git a/rust/ql/integration-tests/options/cfg/functions.override.expected b/rust/ql/integration-tests/options/cfg/functions.override.expected new file mode 100644 index 000000000000..4a19db56a741 --- /dev/null +++ b/rust/ql/integration-tests/options/cfg/functions.override.expected @@ -0,0 +1,4 @@ +| src/lib.rs:1:1:2:16 | cfg_flag | +| src/lib.rs:4:1:5:15 | cfg_key | +| src/lib.rs:13:1:14:12 | test | +| src/lib.rs:19:1:20:24 | pointer_width_32 | diff --git a/rust/ql/integration-tests/options/cfg/functions.ql b/rust/ql/integration-tests/options/cfg/functions.ql new file mode 100644 index 000000000000..da7b22cf5843 --- /dev/null +++ b/rust/ql/integration-tests/options/cfg/functions.ql @@ -0,0 +1,5 @@ +import rust + +from Function f +where f.hasExtendedCanonicalPath() +select f diff --git a/rust/ql/integration-tests/options/cfg/src/lib.rs b/rust/ql/integration-tests/options/cfg/src/lib.rs new file mode 100644 index 000000000000..1643ffbf2adf --- /dev/null +++ b/rust/ql/integration-tests/options/cfg/src/lib.rs @@ -0,0 +1,20 @@ +#[cfg(cfg_flag)] +fn cfg_flag() {} + +#[cfg(cfg_key = "value")] +fn cfg_key() {} + +#[cfg(not(cfg_flag))] +fn cfg_no_flag() {} + +#[cfg(not(cfg_key = "value"))] +fn cfg_no_key() {} + +#[cfg(test)] +fn test() {} + +#[cfg(target_pointer_width = "64")] +fn pointer_width_64() {} + +#[cfg(target_pointer_width = "32")] +fn pointer_width_32() {} diff --git a/rust/ql/integration-tests/options/cfg/test_cfg_overrides_option.py b/rust/ql/integration-tests/options/cfg/test_cfg_overrides_option.py new file mode 100644 index 000000000000..aa5f3cd3b2ed --- /dev/null +++ b/rust/ql/integration-tests/options/cfg/test_cfg_overrides_option.py @@ -0,0 +1,19 @@ +import pytest +import platform +import os + + +def test_default(codeql, rust): + codeql.database.create() + + +@pytest.mark.ql_test(expected=".override.expected") +def test_cfg_overrides(codeql, rust): + overrides = ",".join(( + "cfg_flag", + "cfg_key=value", + "-target_pointer_width=64", + "target_pointer_width=32", + "test", + )) + codeql.database.create(extractor_option=f"cargo_cfg_overrides={overrides}") diff --git a/rust/ql/integration-tests/options/features/Cargo.toml b/rust/ql/integration-tests/options/features/Cargo.toml new file mode 100644 index 000000000000..7faf9f167bec --- /dev/null +++ b/rust/ql/integration-tests/options/features/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +[package] +name = "features" +version = "0.1.0" +edition = "2021" + +[features] +foo = [] +bar = [] diff --git a/rust/ql/integration-tests/options/features/functions.all.expected b/rust/ql/integration-tests/options/features/functions.all.expected new file mode 100644 index 000000000000..5ce4cae2ab0d --- /dev/null +++ b/rust/ql/integration-tests/options/features/functions.all.expected @@ -0,0 +1,3 @@ +| src/lib.rs:1:1:2:11 | foo | +| src/lib.rs:4:1:5:11 | bar | +| src/lib.rs:7:1:7:14 | always | diff --git a/rust/ql/integration-tests/options/features/functions.bar.expected b/rust/ql/integration-tests/options/features/functions.bar.expected new file mode 100644 index 000000000000..cad441d6749b --- /dev/null +++ b/rust/ql/integration-tests/options/features/functions.bar.expected @@ -0,0 +1,2 @@ +| src/lib.rs:4:1:5:11 | bar | +| src/lib.rs:7:1:7:14 | always | diff --git a/rust/ql/integration-tests/options/features/functions.expected b/rust/ql/integration-tests/options/features/functions.expected new file mode 100644 index 000000000000..523c21876884 --- /dev/null +++ b/rust/ql/integration-tests/options/features/functions.expected @@ -0,0 +1 @@ +| src/lib.rs:7:1:7:14 | always | diff --git a/rust/ql/integration-tests/options/features/functions.foo.expected b/rust/ql/integration-tests/options/features/functions.foo.expected new file mode 100644 index 000000000000..b78754230833 --- /dev/null +++ b/rust/ql/integration-tests/options/features/functions.foo.expected @@ -0,0 +1,2 @@ +| src/lib.rs:1:1:2:11 | foo | +| src/lib.rs:7:1:7:14 | always | diff --git a/rust/ql/integration-tests/options/features/functions.ql b/rust/ql/integration-tests/options/features/functions.ql new file mode 100644 index 000000000000..da7b22cf5843 --- /dev/null +++ b/rust/ql/integration-tests/options/features/functions.ql @@ -0,0 +1,5 @@ +import rust + +from Function f +where f.hasExtendedCanonicalPath() +select f diff --git a/rust/ql/integration-tests/options/features/src/lib.rs b/rust/ql/integration-tests/options/features/src/lib.rs new file mode 100644 index 000000000000..c1e97e05fd7f --- /dev/null +++ b/rust/ql/integration-tests/options/features/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(feature = "foo")] +fn foo() {} + +#[cfg(feature = "bar")] +fn bar() {} + +fn always() {} diff --git a/rust/ql/integration-tests/options/features/test_features_option.py b/rust/ql/integration-tests/options/features/test_features_option.py new file mode 100644 index 000000000000..75c5058be078 --- /dev/null +++ b/rust/ql/integration-tests/options/features/test_features_option.py @@ -0,0 +1,17 @@ +import pytest + +def test_default(codeql, rust): + codeql.database.create() + +@pytest.mark.parametrize("features", + [ + pytest.param(p, + marks=pytest.mark.ql_test(expected=f".{e}.expected")) + for p, e in ( + ("foo", "foo"), + ("bar", "bar"), + ("*", "all"), + ("foo,bar", "all")) + ]) +def test_features(codeql, rust, features): + codeql.database.create(extractor_option=f"cargo_features={features}") diff --git a/rust/ql/integration-tests/options/target/Cargo.toml b/rust/ql/integration-tests/options/target/Cargo.toml new file mode 100644 index 000000000000..90a82fb6a07b --- /dev/null +++ b/rust/ql/integration-tests/options/target/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +[package] +name = "target" +version = "0.1.0" +edition = "2021" diff --git a/rust/ql/integration-tests/options/target/functions.Darwin.expected b/rust/ql/integration-tests/options/target/functions.Darwin.expected new file mode 100644 index 000000000000..217974bb7c63 --- /dev/null +++ b/rust/ql/integration-tests/options/target/functions.Darwin.expected @@ -0,0 +1 @@ +| src/lib.rs:7:1:8:13 | macos | diff --git a/rust/ql/integration-tests/options/target/functions.Linux.expected b/rust/ql/integration-tests/options/target/functions.Linux.expected new file mode 100644 index 000000000000..57662579f1f5 --- /dev/null +++ b/rust/ql/integration-tests/options/target/functions.Linux.expected @@ -0,0 +1 @@ +| src/lib.rs:1:1:2:13 | linux | diff --git a/rust/ql/integration-tests/options/target/functions.Windows.expected b/rust/ql/integration-tests/options/target/functions.Windows.expected new file mode 100644 index 000000000000..8ae2d8c86ccf --- /dev/null +++ b/rust/ql/integration-tests/options/target/functions.Windows.expected @@ -0,0 +1 @@ +| src/lib.rs:4:1:5:15 | windows | diff --git a/rust/ql/integration-tests/options/target/functions.ql b/rust/ql/integration-tests/options/target/functions.ql new file mode 100644 index 000000000000..da7b22cf5843 --- /dev/null +++ b/rust/ql/integration-tests/options/target/functions.ql @@ -0,0 +1,5 @@ +import rust + +from Function f +where f.hasExtendedCanonicalPath() +select f diff --git a/rust/ql/integration-tests/options/target/src/lib.rs b/rust/ql/integration-tests/options/target/src/lib.rs new file mode 100644 index 000000000000..fc9ebcbb616d --- /dev/null +++ b/rust/ql/integration-tests/options/target/src/lib.rs @@ -0,0 +1,8 @@ +#[cfg(target_os = "linux")] +fn linux() {} + +#[cfg(target_os = "windows")] +fn windows() {} + +#[cfg(target_os = "macos")] +fn macos() {} diff --git a/rust/ql/integration-tests/options/target/test_target_option.py b/rust/ql/integration-tests/options/target/test_target_option.py new file mode 100644 index 000000000000..3cb1db7513d1 --- /dev/null +++ b/rust/ql/integration-tests/options/target/test_target_option.py @@ -0,0 +1,19 @@ +import pytest +import platform + + +@pytest.mark.ql_test(expected=f".{platform.system()}.expected") +def test_default(codeql, rust): + codeql.database.create() + +@pytest.mark.ql_test(expected=".Windows.expected") +def test_target_windows(codeql, rust): + codeql.database.create(extractor_option="cargo_target=x86_64-pc-windows-msvc") + +@pytest.mark.ql_test(expected=".Darwin.expected") +def test_target_macos(codeql, rust): + codeql.database.create(extractor_option="cargo_target=aarch64-apple-darwin") + +@pytest.mark.ql_test(expected=".Linux.expected") +def test_target_linux(codeql, rust): + codeql.database.create(extractor_option="cargo_target=x86_64-unknown-linux-gnu") From e077bf07327c32eed1355ae113a1613aa7385cf8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 18 Nov 2024 16:06:27 +0100 Subject: [PATCH 2/4] Rust: update vendored dependencies --- MODULE.bazel | 2 +- .../3rdparty/tree_sitter_extractors_deps/BUILD.bazel | 12 ++++++++++++ .../3rdparty/tree_sitter_extractors_deps/defs.bzl | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 666ed89be437..500916c057f3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -68,7 +68,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2. # deps for ruby+rust # keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh` tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r") -use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1") +use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1") dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") dotnet.toolchain(dotnet_version = "8.0.101") diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index a1fc9e105123..844d385f8a41 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -139,6 +139,12 @@ alias( tags = ["manual"], ) +alias( + name = "ra_ap_cfg", + actual = "@vendor__ra_ap_cfg-0.0.232//:ra_ap_cfg", + tags = ["manual"], +) + alias( name = "ra_ap_hir", actual = "@vendor__ra_ap_hir-0.0.232//:ra_ap_hir", @@ -163,6 +169,12 @@ alias( tags = ["manual"], ) +alias( + name = "ra_ap_intern", + actual = "@vendor__ra_ap_intern-0.0.232//:ra_ap_intern", + tags = ["manual"], +) + alias( name = "ra_ap_load-cargo", actual = "@vendor__ra_ap_load-cargo-0.0.232//:ra_ap_load_cargo", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index 86344386015e..1d0b825c2356 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -327,10 +327,12 @@ _NORMAL_DEPENDENCIES = { "log": Label("@vendor__log-0.4.22//:log"), "num-traits": Label("@vendor__num-traits-0.2.19//:num_traits"), "ra_ap_base_db": Label("@vendor__ra_ap_base_db-0.0.232//:ra_ap_base_db"), + "ra_ap_cfg": Label("@vendor__ra_ap_cfg-0.0.232//:ra_ap_cfg"), "ra_ap_hir": Label("@vendor__ra_ap_hir-0.0.232//:ra_ap_hir"), "ra_ap_hir_def": Label("@vendor__ra_ap_hir_def-0.0.232//:ra_ap_hir_def"), "ra_ap_hir_expand": Label("@vendor__ra_ap_hir_expand-0.0.232//:ra_ap_hir_expand"), "ra_ap_ide_db": Label("@vendor__ra_ap_ide_db-0.0.232//:ra_ap_ide_db"), + "ra_ap_intern": Label("@vendor__ra_ap_intern-0.0.232//:ra_ap_intern"), "ra_ap_load-cargo": Label("@vendor__ra_ap_load-cargo-0.0.232//:ra_ap_load_cargo"), "ra_ap_parser": Label("@vendor__ra_ap_parser-0.0.232//:ra_ap_parser"), "ra_ap_paths": Label("@vendor__ra_ap_paths-0.0.232//:ra_ap_paths"), @@ -3323,10 +3325,12 @@ def crate_repositories(): struct(repo = "vendor__proc-macro2-1.0.89", is_dev_dep = False), struct(repo = "vendor__quote-1.0.37", is_dev_dep = False), struct(repo = "vendor__ra_ap_base_db-0.0.232", is_dev_dep = False), + struct(repo = "vendor__ra_ap_cfg-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_hir-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_hir_def-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_hir_expand-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_ide_db-0.0.232", is_dev_dep = False), + struct(repo = "vendor__ra_ap_intern-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_load-cargo-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_parser-0.0.232", is_dev_dep = False), struct(repo = "vendor__ra_ap_paths-0.0.232", is_dev_dep = False), From 0943389ca13636c17bedd172a2a31acc4021e927 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 18 Nov 2024 16:16:54 +0100 Subject: [PATCH 3/4] Rust: add rust-specific deps updater script --- rust/ast-generator/Cargo.toml | 2 +- rust/extractor/Cargo.toml | 2 +- rust/extractor/macros/Cargo.toml | 2 +- rust/update_cargo_deps.sh | 9 +++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100755 rust/update_cargo_deps.sh diff --git a/rust/ast-generator/Cargo.toml b/rust/ast-generator/Cargo.toml index e52aac450ab4..5eaff201b209 100644 --- a/rust/ast-generator/Cargo.toml +++ b/rust/ast-generator/Cargo.toml @@ -3,7 +3,7 @@ name = "ast-generator" version = "0.1.0" edition = "2021" -# When updating these dependencies, run `misc/bazel/3rdparty/update_cargo_deps.sh` +# When updating these dependencies, run `rust/update_cargo_deps.sh` [dependencies] ungrammar = "1.16.1" proc-macro2 = "1.0.33" diff --git a/rust/extractor/Cargo.toml b/rust/extractor/Cargo.toml index 787ad7fda844..8b58898d3cf3 100644 --- a/rust/extractor/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -3,7 +3,7 @@ name = "codeql-rust" version = "0.1.0" edition = "2021" -# When updating these dependencies, run `misc/bazel/3rdparty/update_cargo_deps.sh` +# When updating these dependencies, run `rust/update_cargo_deps.sh` [dependencies] anyhow = "1.0.86" clap = { version = "4.5.16", features = ["derive"] } diff --git a/rust/extractor/macros/Cargo.toml b/rust/extractor/macros/Cargo.toml index 50c865a63730..42c0a0c6d3d5 100644 --- a/rust/extractor/macros/Cargo.toml +++ b/rust/extractor/macros/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [lib] proc-macro = true -# When updating these dependencies, run `misc/bazel/3rdparty/update_cargo_deps.sh` +# When updating these dependencies, run `rust/update_cargo_deps.sh` [dependencies] quote = "1.0.37" syn = { version = "2.0.77", features = ["full"] } diff --git a/rust/update_cargo_deps.sh b/rust/update_cargo_deps.sh new file mode 100755 index 000000000000..c60670c24041 --- /dev/null +++ b/rust/update_cargo_deps.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +cd "$SCRIPT_DIR/.." +time bazel run //misc/bazel/3rdparty:vendor_tree_sitter_extractors +bazel mod tidy From 8377ee545f53daca0adffc88daa07ed2da752acb Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 18 Nov 2024 17:22:23 +0100 Subject: [PATCH 4/4] Rust: fix reading lists from `options.yml` --- rust/extractor/src/config.rs | 13 ++---- rust/extractor/src/config/deserialize_vec.rs | 49 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 rust/extractor/src/config/deserialize_vec.rs diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index dfa9f5d37d20..66edb94c7f85 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -1,6 +1,9 @@ +mod deserialize_vec; + use anyhow::Context; use clap::Parser; use codeql_extractor::trap; +use deserialize_vec::deserialize_newline_or_comma_separated; use figment::{ providers::{Env, Format, Serialized, Yaml}, value::Value, @@ -14,7 +17,7 @@ use ra_ap_intern::Symbol; use ra_ap_paths::Utf8PathBuf; use ra_ap_project_model::{CargoConfig, CargoFeatures, CfgOverrides, RustLibSource}; use rust_extractor_macros::extractor_cli_config; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use std::fmt::Debug; use std::ops::Not; use std::path::PathBuf; @@ -37,14 +40,6 @@ impl From for trap::Compression { } } -// required by the extractor_cli_config macro. -fn deserialize_newline_or_comma_separated<'a, D: Deserializer<'a>, T: for<'b> From<&'b str>>( - deserializer: D, -) -> Result, D::Error> { - let value = String::deserialize(deserializer)?; - Ok(value.split(['\n', ',']).map(T::from).collect()) -} - #[extractor_cli_config] pub struct Config { pub scratch_dir: PathBuf, diff --git a/rust/extractor/src/config/deserialize_vec.rs b/rust/extractor/src/config/deserialize_vec.rs new file mode 100644 index 000000000000..eebe413ca298 --- /dev/null +++ b/rust/extractor/src/config/deserialize_vec.rs @@ -0,0 +1,49 @@ +use serde::de::Visitor; +use serde::Deserializer; +use std::fmt::Formatter; +use std::marker::PhantomData; + +// phantom data ise required to allow parametrizing on `T` without actual `T` data +struct VectorVisitor>(PhantomData); + +impl> VectorVisitor { + fn new() -> Self { + VectorVisitor(PhantomData) + } +} + +impl<'de, T: From> Visitor<'de> for VectorVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("either a sequence, or a comma or newline separated string") + } + + fn visit_str(self, value: &str) -> Result, E> { + Ok(value + .split(['\n', ',']) + .map(|s| T::from(s.to_owned())) + .collect()) + } + + fn visit_seq(self, mut seq: A) -> Result, A::Error> + where + A: serde::de::SeqAccess<'de>, + { + let mut ret = Vec::new(); + while let Some(el) = seq.next_element::()? { + ret.push(T::from(el)); + } + Ok(ret) + } +} + +/// deserialize into a vector of `T` either of: +/// * a sequence of elements serializable into `String`s, or +/// * a single element serializable into `String`, then split on `,` and `\n` +/// This is required to be in scope when the `extractor_cli_config` macro is used. +pub(crate) fn deserialize_newline_or_comma_separated<'a, D: Deserializer<'a>, T: From>( + deserializer: D, +) -> Result, D::Error> { + deserializer.deserialize_seq(VectorVisitor::new()) +}