From 1b3a5cdab138855838b41c75998e59b7d8ee191d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 11 Sep 2024 17:09:18 +0200 Subject: [PATCH 01/57] Rust: make the cli flags override automatic This makes the clap flags overlay over `Config` entirely derived via an attribute macro. Also, the `--intputs-file` option is replaced by a more standard and versatile `@` parameter file mechanism. --- Cargo.lock | 38 +++++++++++++++++++++++ Cargo.toml | 1 + rust/extractor/Cargo.toml | 5 ++- rust/extractor/macros/Cargo.toml | 11 +++++++ rust/extractor/macros/src/lib.rs | 52 ++++++++++++++++++++++++++++++++ rust/extractor/src/config.rs | 41 ++++--------------------- rust/tools/index-files.sh | 2 +- 7 files changed, 111 insertions(+), 39 deletions(-) create mode 100644 rust/extractor/macros/Cargo.toml create mode 100644 rust/extractor/macros/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index fcd7d8068439..d3b543434997 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,6 +96,16 @@ version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" +[[package]] +name = "argfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1cc0ba69de57db40674c66f7cf2caee3981ddef084388482c95c0e2133e5e8" +dependencies = [ + "fs-err", + "os_str_bytes", +] + [[package]] name = "arrayvec" version = "0.7.6" @@ -360,6 +370,7 @@ name = "codeql-rust" version = "0.1.0" dependencies = [ "anyhow", + "argfile", "clap", "codeql-extractor", "figment", @@ -374,6 +385,7 @@ dependencies = [ "ra_ap_project_model", "ra_ap_syntax", "ra_ap_vfs", + "rust-extractor-macros", "serde", "serde_with", "stderrlog", @@ -643,6 +655,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1064,6 +1085,15 @@ version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" +[[package]] +name = "os_str_bytes" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ac44c994af577c799b1b4bd80dc214701e349873ad894d6cdf96f4f7526e0b9" +dependencies = [ + "memchr", +] + [[package]] name = "overload" version = "0.1.1" @@ -1875,6 +1905,14 @@ dependencies = [ "text-size", ] +[[package]] +name = "rust-extractor-macros" +version = "0.1.0" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5f095736c8a7..4aacef79adc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "shared/tree-sitter-extractor", "ruby/extractor", "rust/extractor", + "rust/extractor/macros", ] [patch.crates-io] diff --git a/rust/extractor/Cargo.toml b/rust/extractor/Cargo.toml index c849ea4aa464..3b474f90f984 100644 --- a/rust/extractor/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -22,7 +22,6 @@ serde = "1.0.209" serde_with = "3.9.0" stderrlog = "0.6.0" triomphe = "0.1.13" -# Ideally, we'd like to pull this in via a relative path. -# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525 -# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory. +argfile = "0.2.1" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } +rust-extractor-macros = { path = "macros" } diff --git a/rust/extractor/macros/Cargo.toml b/rust/extractor/macros/Cargo.toml new file mode 100644 index 000000000000..d4d10bc3bde2 --- /dev/null +++ b/rust/extractor/macros/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rust-extractor-macros" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +quote = "1.0.37" +syn = { version = "2.0.77", features = ["full"] } diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs new file mode 100644 index 000000000000..13472665454a --- /dev/null +++ b/rust/extractor/macros/src/lib.rs @@ -0,0 +1,52 @@ +use proc_macro::TokenStream; +use quote::{quote, format_ident}; +use syn; + + +/// Allow all fields in the extractor config to be also overrideable by extractor CLI flags +#[proc_macro_attribute] +pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream { + let ast = syn::parse_macro_input!(item as syn::ItemStruct); + let name = &ast.ident; + let new_name = format_ident!("Cli{}", name); + let fields: Vec<_> = ast.fields.iter().map(|f| { + let id = f.ident.as_ref().unwrap(); + let ty = &f.ty; + if let syn::Type::Path(p) = ty { + if p.path.is_ident(&format_ident!("bool")) { + return quote! { + #[arg(long)] + #id: bool, + }; + } + } + if id == &format_ident!("verbose") { + quote! { + #[arg(long, short, action=clap::ArgAction::Count)] + #id: u8, + } + } else if id == &format_ident!("inputs") { + quote! { + #id: #ty, + } + } else { + quote! { + #[arg(long)] + #id: Option<#ty>, + } + } + }).collect(); + let gen = quote! { + #[serde_with::apply(_ => #[serde(default)])] + #[derive(Debug, Deserialize, Default)] + #ast + + #[serde_with::skip_serializing_none] + #[derive(clap::Parser, Serialize)] + #[command(about, long_about = None)] + struct #new_name { + #(#fields)* + } + }; + gen.into() +} diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 399c2bb9e7e7..310ca2c3649b 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -1,14 +1,15 @@ use anyhow::Context; -use clap::{ArgAction, Parser, ValueEnum}; +use clap::Parser; use codeql_extractor::trap; use figment::{ providers::{Env, Serialized}, Figment, }; +use rust_extractor_macros::extractor_cli_config; use serde::{Deserialize, Serialize}; use std::path::PathBuf; -#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, ValueEnum)] +#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)] #[serde(rename_all = "lowercase")] #[clap(rename_all = "lowercase")] pub enum Compression { @@ -26,8 +27,7 @@ impl From for trap::Compression { } } -#[serde_with::apply(_ => #[serde(default)])] -#[derive(Debug, Deserialize, Default)] +#[extractor_cli_config] pub struct Config { pub scratch_dir: PathBuf, pub trap_dir: PathBuf, @@ -38,39 +38,10 @@ pub struct Config { pub inputs: Vec, } -#[serde_with::apply(_ => #[serde(skip_serializing_if = "is_default")])] -#[derive(clap::Parser, Serialize)] -#[command(about, long_about = None)] -struct CliArgs { - #[arg(long)] - scratch_dir: Option, - #[arg(long)] - trap_dir: Option, - #[arg(long)] - source_archive_dir: Option, - #[arg(long)] - compression: Option, - #[arg(short, long, action = ArgAction::Count)] - verbose: u8, - #[arg(long)] - inputs_file: Option, - - inputs: Vec, -} - -fn is_default(t: &T) -> bool { - *t == Default::default() -} - impl Config { pub fn extract() -> anyhow::Result { - let mut cli_args = CliArgs::parse(); - if let Some(inputs_file) = cli_args.inputs_file.take() { - let inputs_list = std::fs::read_to_string(inputs_file).context("reading file list")?; - cli_args - .inputs - .extend(inputs_list.split_terminator("\n").map(PathBuf::from)); - } + let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)?; + let cli_args = CliConfig::parse_from(args); Figment::new() .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_")) diff --git a/rust/tools/index-files.sh b/rust/tools/index-files.sh index da4b841b692f..f3d93fbaf4a9 100755 --- a/rust/tools/index-files.sh +++ b/rust/tools/index-files.sh @@ -2,4 +2,4 @@ set -eu -exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --inputs-file="$1" +exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" @"$1" From 0a8c0f5ab40cfc681913ce20ae1328c195ef53fa Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 12 Sep 2024 08:46:50 +0200 Subject: [PATCH 02/57] Rust: fix bazel build --- MODULE.bazel | 1 + ruby/extractor/BUILD.bazel | 2 +- rust/extractor/BUILD.bazel | 6 ++++-- rust/extractor/macros/BUILD.bazel | 20 ++++++++++++++++++++ shared/tree-sitter-extractor/BUILD.bazel | 8 ++++++-- 5 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 rust/extractor/macros/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 20efc5390396..020d5b0fbaea 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -60,6 +60,7 @@ r.from_cargo( "//:Cargo.toml", "//ruby/extractor:Cargo.toml", "//rust/extractor:Cargo.toml", + "//rust/extractor/macros:Cargo.toml", "//shared/tree-sitter-extractor:Cargo.toml", ], ) diff --git a/ruby/extractor/BUILD.bazel b/ruby/extractor/BUILD.bazel index 203f90310fa7..158f1c91e4c4 100644 --- a/ruby/extractor/BUILD.bazel +++ b/ruby/extractor/BUILD.bazel @@ -12,6 +12,6 @@ codeql_rust_binary( deps = all_crate_deps( normal = True, ) + [ - "//shared/tree-sitter-extractor:codeql-extractor", + "//shared/tree-sitter-extractor", ], ) diff --git a/rust/extractor/BUILD.bazel b/rust/extractor/BUILD.bazel index 924d5e01497c..a5d99e82584e 100644 --- a/rust/extractor/BUILD.bazel +++ b/rust/extractor/BUILD.bazel @@ -7,11 +7,13 @@ codeql_rust_binary( aliases = aliases(), proc_macro_deps = all_crate_deps( proc_macro = True, - ), + ) + [ + "//rust/extractor/macros", + ], visibility = ["//rust:__subpackages__"], deps = all_crate_deps( normal = True, ) + [ - "//shared/tree-sitter-extractor:codeql-extractor", + "//shared/tree-sitter-extractor", ], ) diff --git a/rust/extractor/macros/BUILD.bazel b/rust/extractor/macros/BUILD.bazel new file mode 100644 index 000000000000..4ddfb14a171f --- /dev/null +++ b/rust/extractor/macros/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_rust//rust:defs.bzl", "rust_proc_macro") +load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps") + +rust_proc_macro( + name = "rust_extractor_macros", + srcs = glob(["src/**/*.rs"]), + aliases = aliases(), + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + deps = all_crate_deps( + normal = True, + ), +) + +alias( + name = "macros", + actual = "rust_extractor_macros", + visibility = ["//rust:__subpackages__"], +) diff --git a/shared/tree-sitter-extractor/BUILD.bazel b/shared/tree-sitter-extractor/BUILD.bazel index dc9001a32d25..0ebc189954be 100644 --- a/shared/tree-sitter-extractor/BUILD.bazel +++ b/shared/tree-sitter-extractor/BUILD.bazel @@ -1,8 +1,6 @@ load("@rules_rust//rust:defs.bzl", "rust_library") load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps") -package(default_visibility = ["//visibility:public"]) - rust_library( name = "codeql-extractor", srcs = glob([ @@ -14,3 +12,9 @@ rust_library( ], deps = all_crate_deps(), ) + +alias( + name = "tree-sitter-extractor", + actual = ":codeql-extractor", + visibility = ["//visibility:public"], +) From 6adf88542e4477f7cab6ff02ca965b293f85ce50 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 12 Sep 2024 08:53:08 +0200 Subject: [PATCH 03/57] Rust: fix linting script --- rust/extractor/macros/src/lib.rs | 58 +++++++++++++++++--------------- rust/lint.py | 10 +++--- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs index 13472665454a..1905d40f9b21 100644 --- a/rust/extractor/macros/src/lib.rs +++ b/rust/extractor/macros/src/lib.rs @@ -1,7 +1,5 @@ use proc_macro::TokenStream; -use quote::{quote, format_ident}; -use syn; - +use quote::{format_ident, quote}; /// Allow all fields in the extractor config to be also overrideable by extractor CLI flags #[proc_macro_attribute] @@ -9,33 +7,37 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea let ast = syn::parse_macro_input!(item as syn::ItemStruct); let name = &ast.ident; let new_name = format_ident!("Cli{}", name); - let fields: Vec<_> = ast.fields.iter().map(|f| { - let id = f.ident.as_ref().unwrap(); - let ty = &f.ty; - if let syn::Type::Path(p) = ty { - if p.path.is_ident(&format_ident!("bool")) { - return quote! { - #[arg(long)] - #id: bool, - }; - } - } - if id == &format_ident!("verbose") { - quote! { - #[arg(long, short, action=clap::ArgAction::Count)] - #id: u8, + let fields: Vec<_> = ast + .fields + .iter() + .map(|f| { + let id = f.ident.as_ref().unwrap(); + let ty = &f.ty; + if let syn::Type::Path(p) = ty { + if p.path.is_ident(&format_ident!("bool")) { + return quote! { + #[arg(long)] + #id: bool, + }; + } } - } else if id == &format_ident!("inputs") { - quote! { - #id: #ty, - } - } else { - quote! { - #[arg(long)] - #id: Option<#ty>, + if id == &format_ident!("verbose") { + quote! { + #[arg(long, short, action=clap::ArgAction::Count)] + #id: u8, + } + } else if id == &format_ident!("inputs") { + quote! { + #id: #ty, + } + } else { + quote! { + #[arg(long)] + #id: Option<#ty>, + } } - } - }).collect(); + }) + .collect(); let gen = quote! { #[serde_with::apply(_ => #[serde(default)])] #[derive(Debug, Deserialize, Default)] diff --git a/rust/lint.py b/rust/lint.py index 1af2470dbbcf..eb71cbd0b3dd 100755 --- a/rust/lint.py +++ b/rust/lint.py @@ -5,12 +5,14 @@ import shutil import sys -extractor_dir = pathlib.Path(__file__).resolve().parent / "extractor" +this_dir = pathlib.Path(__file__).resolve().parent cargo = shutil.which("cargo") assert cargo, "no cargo binary found on `PATH`" -fmt = subprocess.run([cargo, "fmt", "--quiet"], cwd=extractor_dir) -clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"], - cwd=extractor_dir) +fmt = subprocess.run([cargo, "fmt", "--all", "--quiet"], cwd=this_dir) +for manifest in this_dir.rglob("Cargo.toml"): + if not manifest.is_relative_to(this_dir / "ql"): + clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"], + cwd=manifest.parent) sys.exit(fmt.returncode or clippy.returncode) From 5ae88243033c013bf1fd473e1e9e0648493e7aed Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 12 Sep 2024 08:56:07 +0200 Subject: [PATCH 04/57] Rust: add context to parameter file expansion errors --- rust/extractor/src/config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 310ca2c3649b..74c9f4b20a5b 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -40,7 +40,8 @@ pub struct Config { impl Config { pub fn extract() -> anyhow::Result { - let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)?; + let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX) + .context("expanding parameter files")?; let cli_args = CliConfig::parse_from(args); Figment::new() .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) From 403cc3df9013c3a278b6b8b9b8b5a52f2dd2d91b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 13 Sep 2024 06:50:12 +0200 Subject: [PATCH 05/57] Rust: avoid cli flag defaults overriding env settings --- rust/extractor/macros/src/lib.rs | 2 ++ rust/extractor/src/config.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs index 1905d40f9b21..781d53bd8513 100644 --- a/rust/extractor/macros/src/lib.rs +++ b/rust/extractor/macros/src/lib.rs @@ -17,6 +17,7 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea if p.path.is_ident(&format_ident!("bool")) { return quote! { #[arg(long)] + #[serde(skip_serializing_if="<&bool>::not")] #id: bool, }; } @@ -24,6 +25,7 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea if id == &format_ident!("verbose") { quote! { #[arg(long, short, action=clap::ArgAction::Count)] + #[serde(skip_serializing_if="u8::is_zero")] #id: u8, } } else if id == &format_ident!("inputs") { diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 74c9f4b20a5b..98dac7fce326 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -5,8 +5,10 @@ use figment::{ providers::{Env, Serialized}, Figment, }; +use num_traits::Zero; use rust_extractor_macros::extractor_cli_config; use serde::{Deserialize, Serialize}; +use std::ops::Not; use std::path::PathBuf; #[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)] From 194c2fa9c48907ccf0ca75ecd3c3007d114d6ce7 Mon Sep 17 00:00:00 2001 From: Florin Coada Date: Fri, 13 Sep 2024 10:18:04 +0100 Subject: [PATCH 06/57] Add changedocs for 2.18.4 --- .../codeql-changelog/codeql-cli-2.18.4.rst | 45 +++++++++++++++++++ .../codeql-changelog/index.rst | 1 + 2 files changed, 46 insertions(+) create mode 100644 docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst diff --git a/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst new file mode 100644 index 000000000000..14f7cb966478 --- /dev/null +++ b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst @@ -0,0 +1,45 @@ +.. _codeql-cli-2.18.4: + +========================== +CodeQL 2.18.4 (2024-09-12) +========================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: none + +This is an overview of changes in the CodeQL CLI and relevant CodeQL query and library packs. For additional updates on changes to the CodeQL code scanning experience, check out the `code scanning section on the GitHub blog `__, `relevant GitHub Changelog updates `__, `changes in the CodeQL extension for Visual Studio Code `__, and the `CodeQL Action changelog `__. + +Security Coverage +----------------- + +CodeQL 2.18.4 runs a total of 425 security queries when configured with the Default suite (covering 164 CWE). The Extended suite enables an additional 128 queries (covering 34 more CWE). + +CodeQL CLI +---------- + +New Features +~~~~~~~~~~~~ + +* C# support for :code:`build-mode: none` is now out of beta, and generally available. +* Go 1.23 is now supported. + +Language Libraries +------------------ + +Major Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Golang +"""""" + +* Go 1.23 is now supported. + +New Features +~~~~~~~~~~~~ + +C# +"" + +* C# support for :code:`build-mode: none` is now out of beta, and generally available. diff --git a/docs/codeql/codeql-overview/codeql-changelog/index.rst b/docs/codeql/codeql-overview/codeql-changelog/index.rst index d8baa12e9958..67722290b364 100644 --- a/docs/codeql/codeql-overview/codeql-changelog/index.rst +++ b/docs/codeql/codeql-overview/codeql-changelog/index.rst @@ -11,6 +11,7 @@ A list of queries for each suite and language `is available here Date: Fri, 13 Sep 2024 13:39:39 +0200 Subject: [PATCH 07/57] Rust: add `CODEQL_` base env layer --- rust/extractor/src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 98dac7fce326..0625968a4af7 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -46,6 +46,7 @@ impl Config { .context("expanding parameter files")?; let cli_args = CliConfig::parse_from(args); Figment::new() + .merge(Env::prefixed("CODEQL_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_")) .merge(Serialized::defaults(cli_args)) From faf1eeeb0db2c8d38be608283e1c550fad258bdc Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 13 Sep 2024 13:18:33 +0200 Subject: [PATCH 08/57] Rust: introduce typed labels --- misc/codegen/generators/rustgen.py | 16 +- misc/codegen/lib/rust.py | 3 +- misc/codegen/templates/rust_classes.mustache | 75 +- rust/extractor/src/generated/.generated.list | 2 +- rust/extractor/src/generated/top.rs | 5326 ++++++++++++++--- rust/extractor/src/translate.rs | 723 ++- rust/extractor/src/trap.rs | 65 +- rust/ql/.generated.list | 13 +- rust/ql/.gitattributes | 1 - .../rust/elements/TypeRefConstructor.qll | 14 - .../lib/codeql/rust/generated/ParentChild.qll | 41 +- rust/ql/lib/codeql/rust/generated/Raw.qll | 18 +- rust/ql/lib/codeql/rust/generated/Synth.qll | 49 +- .../rust/generated/SynthConstructors.qll | 1 - rust/ql/lib/codeql/rust/generated/TypeRef.qll | 4 +- .../codeql/rust/generated/Unimplemented.qll | 5 +- rust/ql/lib/rust.dbscheme | 16 +- rust/schema.py | 22 +- 18 files changed, 5175 insertions(+), 1219 deletions(-) delete mode 100644 rust/ql/lib/codeql/rust/elements/TypeRefConstructor.qll diff --git a/misc/codegen/generators/rustgen.py b/misc/codegen/generators/rustgen.py index 9b850c3cf1be..6fa5d4a05ec7 100644 --- a/misc/codegen/generators/rustgen.py +++ b/misc/codegen/generators/rustgen.py @@ -20,7 +20,7 @@ def _get_type(t: str) -> str: case "int": return "usize" case _ if t[0].isupper(): - return "trap::Label" + return f"{t}TrapLabel" case "boolean": assert False, "boolean unsupported" case _: @@ -57,6 +57,15 @@ def _get_properties( yield cls, p +def _get_ancestors( + cls: schema.Class, lookup: dict[str, schema.Class] +) -> typing.Iterable[schema.Class]: + for b in cls.bases: + base = lookup[b] + yield base + yield from _get_ancestors(base, lookup) + + class Processor: def __init__(self, data: schema.Schema): self._classmap = data.classes @@ -69,14 +78,15 @@ def _get_class(self, name: str) -> rust.Class: _get_field(c, p) for c, p in _get_properties(cls, self._classmap) if "rust_skip" not in p.pragmas and not p.synth - ], + ] if not cls.derived else [], + ancestors=sorted(set(a.name for a in _get_ancestors(cls, self._classmap))), table_name=inflection.tableize(cls.name), ) def get_classes(self): ret = {"": []} for k, cls in self._classmap.items(): - if not cls.synth and not cls.derived: + if not cls.synth: ret.setdefault(cls.group, []).append(self._get_class(cls.name)) return ret diff --git a/misc/codegen/lib/rust.py b/misc/codegen/lib/rust.py index ac7bf4313d3f..0f4b410db70b 100644 --- a/misc/codegen/lib/rust.py +++ b/misc/codegen/lib/rust.py @@ -110,8 +110,9 @@ def is_label(self): @dataclasses.dataclass class Class: name: str - table_name: str + table_name: str | None = None fields: list[Field] = dataclasses.field(default_factory=list) + ancestors: list[str] = dataclasses.field(default_factory=list) @property def single_field_entries(self): diff --git a/misc/codegen/templates/rust_classes.mustache b/misc/codegen/templates/rust_classes.mustache index 3b415683d5f3..f749733b2bd0 100644 --- a/misc/codegen/templates/rust_classes.mustache +++ b/misc/codegen/templates/rust_classes.mustache @@ -2,48 +2,77 @@ #![cfg_attr(any(), rustfmt::skip)] -use crate::trap::{TrapId, TrapEntry}; -use codeql_extractor::trap; +use crate::trap; {{#classes}} +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct {{name}}TrapLabel(trap::UntypedLabel); + +impl From for {{name}}TrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From<{{name}}TrapLabel> for trap::TrapId<{{name}}> { + fn from(value: {{name}}TrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for {{name}}TrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From<{{name}}TrapLabel> for trap::Arg { + fn from(value: {{name}}TrapLabel) -> Self { + value.0.into() + } +} + +{{#table_name}} #[derive(Debug)] pub struct {{name}} { - pub id: TrapId, + pub id: trap::TrapId<{{name}}>, {{#fields}} pub {{field_name}}: {{type}}, {{/fields}} } -impl TrapEntry for {{name}} { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for {{name}} { + fn class_name() -> &'static str { "{{name}}" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { + fn emit(self, id: Self::Label, out: &mut trap::Writer) { {{#single_field_entries}} - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{#fields}}, self.{{field_name}}.into(){{/fields}}]); + out.add_tuple("{{table_name}}", vec![id.into(){{#fields}}, self.{{field_name}}.into(){{/fields}}]); {{/single_field_entries}} {{#fields}} {{#is_predicate}} if self.{{field_name}} { - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id)]); + out.add_tuple("{{table_name}}", vec![id.into()]); } {{/is_predicate}} {{#is_optional}} {{^is_repeated}} if let Some(v) = self.{{field_name}} { - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id), v.into()]); + out.add_tuple("{{table_name}}", vec![id.into(), v.into()]); } {{/is_repeated}} {{/is_optional}} {{#is_repeated}} for (i, v) in self.{{field_name}}.into_iter().enumerate() { {{^is_optional}} - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); + out.add_tuple("{{table_name}}", vec![id.into(){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); {{/is_optional}} {{#is_optional}} if let Some(v) = v { - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); + out.add_tuple("{{table_name}}", vec![id.into(){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); } {{/is_optional}} } @@ -51,4 +80,26 @@ impl TrapEntry for {{name}} { {{/fields}} } } +{{/table_name}} +{{^table_name}} +{{! virtual class, make it unbuildable }} +pub struct {{name}} { + unused: () +} +{{/table_name}} + +impl trap::TrapClass for {{name}} { + type Label = {{name}}TrapLabel; +} +{{/classes}} + +// Conversions +{{#classes}} +{{#ancestors}} +impl From<{{name}}TrapLabel> for {{.}}TrapLabel { + fn from(value: {{name}}TrapLabel) -> Self { + value.0.into() + } +} +{{/ancestors}} {{/classes}} diff --git a/rust/extractor/src/generated/.generated.list b/rust/extractor/src/generated/.generated.list index b404b2e7541c..fd7cc5c6614a 100644 --- a/rust/extractor/src/generated/.generated.list +++ b/rust/extractor/src/generated/.generated.list @@ -1,2 +1,2 @@ mod.rs 7cdfedcd68cf8e41134daf810c1af78624082b0c3e8be6570339b1a69a5d457e 7cdfedcd68cf8e41134daf810c1af78624082b0c3e8be6570339b1a69a5d457e -top.rs 7150acaeab0b57039ca9f2ed20311229aab5fd48b533f13410ecc34fd8e3bda0 7150acaeab0b57039ca9f2ed20311229aab5fd48b533f13410ecc34fd8e3bda0 +top.rs e06dc90de4abd57719786fd5e49e6ea3089ec3ec167c64446e25d95a16b1714c e06dc90de4abd57719786fd5e49e6ea3089ec3ec167c64446e25d95a16b1714c diff --git a/rust/extractor/src/generated/top.rs b/rust/extractor/src/generated/top.rs index de6d2106c1d7..084fdbb61f02 100644 --- a/rust/extractor/src/generated/top.rs +++ b/rust/extractor/src/generated/top.rs @@ -2,1222 +2,5008 @@ #![cfg_attr(any(), rustfmt::skip)] -use crate::trap::{TrapId, TrapEntry}; -use codeql_extractor::trap; +use crate::trap; -#[derive(Debug)] -pub struct Label { - pub id: TrapId, - pub name: String, +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ElementTrapLabel(trap::UntypedLabel); + +impl From for ElementTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId { + fn from(value: ElementTrapLabel) -> Self { + Self::Label(value) + } } -impl TrapEntry for Label { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::Label for ElementTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 } +} - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("labels", vec![trap::Arg::Label(id), self.name.into()]); +impl From for trap::Arg { + fn from(value: ElementTrapLabel) -> Self { + value.0.into() } } #[derive(Debug)] -pub struct MatchArm { - pub id: TrapId, - pub pat: trap::Label, - pub guard: Option, - pub expr: trap::Label, +pub struct Element { + pub id: trap::TrapId, } -impl TrapEntry for MatchArm { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for Element { + fn class_name() -> &'static str { "Element" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("match_arms", vec![trap::Arg::Label(id), self.pat.into(), self.expr.into()]); - if let Some(v) = self.guard { - out.add_tuple("match_arm_guards", vec![trap::Arg::Label(id), v.into()]); - } + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("elements", vec![id.into()]); } } -#[derive(Debug)] -pub struct RecordFieldPat { - pub id: TrapId, - pub name: String, - pub pat: trap::Label, +impl trap::TrapClass for Element { + type Label = ElementTrapLabel; } -impl TrapEntry for RecordFieldPat { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) - } +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct LocatableTrapLabel(trap::UntypedLabel); - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("record_field_pats", vec![trap::Arg::Label(id), self.name.into(), self.pat.into()]); +impl From for LocatableTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) } } -#[derive(Debug)] -pub struct RecordLitField { - pub id: TrapId, - pub name: String, - pub expr: trap::Label, +impl From for trap::TrapId { + fn from(value: LocatableTrapLabel) -> Self { + Self::Label(value) + } } -impl TrapEntry for RecordLitField { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::Label for LocatableTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 } +} - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("record_lit_fields", vec![trap::Arg::Label(id), self.name.into(), self.expr.into()]); +impl From for trap::Arg { + fn from(value: LocatableTrapLabel) -> Self { + value.0.into() } } #[derive(Debug)] -pub struct TypeRef { - pub id: TrapId, +pub struct Locatable { + pub id: trap::TrapId, } -impl TrapEntry for TypeRef { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for Locatable { + fn class_name() -> &'static str { "Locatable" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("type_refs", vec![trap::Arg::Label(id)]); + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("locatables", vec![id.into()]); } } -#[derive(Debug)] -pub struct Unimplemented { - pub id: TrapId, +impl trap::TrapClass for Locatable { + type Label = LocatableTrapLabel; } -impl TrapEntry for Unimplemented { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) - } +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct AstNodeTrapLabel(trap::UntypedLabel); - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("unimplementeds", vec![trap::Arg::Label(id)]); +impl From for AstNodeTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) } } -#[derive(Debug)] -pub struct AwaitExpr { - pub id: TrapId, - pub expr: trap::Label, +impl From for trap::TrapId { + fn from(value: AstNodeTrapLabel) -> Self { + Self::Label(value) + } } -impl TrapEntry for AwaitExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::Label for AstNodeTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 } +} - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("await_exprs", vec![trap::Arg::Label(id), self.expr.into()]); +impl From for trap::Arg { + fn from(value: AstNodeTrapLabel) -> Self { + value.0.into() } } #[derive(Debug)] -pub struct BecomeExpr { - pub id: TrapId, - pub expr: trap::Label, +pub struct AstNode { + pub id: trap::TrapId, } -impl TrapEntry for BecomeExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for AstNode { + fn class_name() -> &'static str { "AstNode" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("become_exprs", vec![trap::Arg::Label(id), self.expr.into()]); + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("ast_nodes", vec![id.into()]); } } -#[derive(Debug)] -pub struct BinaryOpExpr { - pub id: TrapId, - pub lhs: trap::Label, - pub rhs: trap::Label, - pub op: Option, +impl trap::TrapClass for AstNode { + type Label = AstNodeTrapLabel; } -impl TrapEntry for BinaryOpExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) - } +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct DeclarationTrapLabel(trap::UntypedLabel); - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("binary_op_exprs", vec![trap::Arg::Label(id), self.lhs.into(), self.rhs.into()]); - if let Some(v) = self.op { - out.add_tuple("binary_op_expr_ops", vec![trap::Arg::Label(id), v.into()]); - } +impl From for DeclarationTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) } } -#[derive(Debug)] -pub struct BindPat { - pub id: TrapId, - pub binding_id: String, - pub subpat: Option, +impl From for trap::TrapId { + fn from(value: DeclarationTrapLabel) -> Self { + Self::Label(value) + } } -impl TrapEntry for BindPat { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::Label for DeclarationTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 } +} - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("bind_pats", vec![trap::Arg::Label(id), self.binding_id.into()]); - if let Some(v) = self.subpat { - out.add_tuple("bind_pat_subpats", vec![trap::Arg::Label(id), v.into()]); - } +impl From for trap::Arg { + fn from(value: DeclarationTrapLabel) -> Self { + value.0.into() } } #[derive(Debug)] -pub struct BoxExpr { - pub id: TrapId, - pub expr: trap::Label, +pub struct Declaration { + pub id: trap::TrapId, } -impl TrapEntry for BoxExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for Declaration { + fn class_name() -> &'static str { "Declaration" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("box_exprs", vec![trap::Arg::Label(id), self.expr.into()]); + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("declarations", vec![id.into()]); } } -#[derive(Debug)] -pub struct BoxPat { - pub id: TrapId, - pub inner: trap::Label, +impl trap::TrapClass for Declaration { + type Label = DeclarationTrapLabel; } -impl TrapEntry for BoxPat { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) - } +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ExprTrapLabel(trap::UntypedLabel); - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("box_pats", vec![trap::Arg::Label(id), self.inner.into()]); +impl From for ExprTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) } } -#[derive(Debug)] -pub struct BreakExpr { - pub id: TrapId, - pub expr: Option, - pub label: Option, +impl From for trap::TrapId { + fn from(value: ExprTrapLabel) -> Self { + Self::Label(value) + } } -impl TrapEntry for BreakExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::Label for ExprTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 } +} - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("break_exprs", vec![trap::Arg::Label(id)]); - if let Some(v) = self.expr { - out.add_tuple("break_expr_exprs", vec![trap::Arg::Label(id), v.into()]); - } - if let Some(v) = self.label { - out.add_tuple("break_expr_labels", vec![trap::Arg::Label(id), v.into()]); - } +impl From for trap::Arg { + fn from(value: ExprTrapLabel) -> Self { + value.0.into() } } #[derive(Debug)] -pub struct CallExpr { - pub id: TrapId, - pub callee: trap::Label, - pub args: Vec, - pub is_assignee_expr: bool, +pub struct Expr { + pub id: trap::TrapId, } -impl TrapEntry for CallExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for Expr { + fn class_name() -> &'static str { "Expr" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("call_exprs", vec![trap::Arg::Label(id), self.callee.into()]); - for (i, v) in self.args.into_iter().enumerate() { - out.add_tuple("call_expr_args", vec![trap::Arg::Label(id), i.into(), v.into()]); - } - if self.is_assignee_expr { - out.add_tuple("call_expr_is_assignee_expr", vec![trap::Arg::Label(id)]); - } + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("exprs", vec![id.into()]); } } -#[derive(Debug)] -pub struct CastExpr { - pub id: TrapId, - pub expr: trap::Label, - pub type_ref: trap::Label, +impl trap::TrapClass for Expr { + type Label = ExprTrapLabel; } -impl TrapEntry for CastExpr { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) - } +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct LabelTrapLabel(trap::UntypedLabel); - fn emit(self, id: trap::Label, out: &mut trap::Writer) { - out.add_tuple("cast_exprs", vec![trap::Arg::Label(id), self.expr.into(), self.type_ref.into()]); +impl From for LabelTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) } } -#[derive(Debug)] -pub struct ClosureExpr { - pub id: TrapId, - pub args: Vec, - pub arg_types: Vec>, - pub ret_type: Option, - pub body: trap::Label, - pub closure_kind: String, - pub is_move: bool, +impl From for trap::TrapId