From 53b2e9708cddc30c8f71134b772cce533493e3b3 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Wed, 30 Apr 2025 13:21:56 +0200 Subject: [PATCH 1/3] Rust: extract declarations of builtin types --- rust/BUILD.bazel | 1 + rust/extractor/src/main.rs | 36 +++++++++++++++++++++++---------- rust/tools/builtins/BUILD.bazel | 8 ++++++++ rust/tools/builtins/types.rs | 25 +++++++++++++++++++++++ 4 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 rust/tools/builtins/BUILD.bazel create mode 100644 rust/tools/builtins/types.rs diff --git a/rust/BUILD.bazel b/rust/BUILD.bazel index 9cdc89dd52fd..6536a4b6fd9e 100644 --- a/rust/BUILD.bazel +++ b/rust/BUILD.bazel @@ -36,6 +36,7 @@ pkg_filegroup( srcs = [ ":tools-arch", "//rust/tools", + "//rust/tools/builtins", ], prefix = "tools", ) diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index 0ec1769f1d17..b983a3217a3a 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -17,6 +17,7 @@ use std::{ collections::HashMap, path::{Path, PathBuf}, }; +use std::{env, fs}; use tracing::{error, info, warn}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; @@ -77,17 +78,19 @@ impl<'a> Extractor<'a> { } let no_location = (LineCol { line: 0, col: 0 }, LineCol { line: 0, col: 0 }); if let Err(reason) = semantics_info { - let message = format!("semantic analyzer unavailable ({reason})"); - let full_message = format!( - "{message}: macro expansion, call graph, and type inference will be skipped." - ); - translator.emit_diagnostic( - trap::DiagnosticSeverity::Warning, - "semantics".to_owned(), - message, - full_message, - no_location, - ); + if !reason.is_empty() { + let message = format!("semantic analyzer unavailable ({reason})"); + let full_message = format!( + "{message}: macro expansion, call graph, and type inference will be skipped." + ); + translator.emit_diagnostic( + trap::DiagnosticSeverity::Warning, + "semantics".to_owned(), + message, + full_message, + no_location, + ); + } } translator.emit_source_file(ast); translator.trap.commit().unwrap_or_else(|err| { @@ -276,5 +279,16 @@ fn main() -> anyhow::Result<()> { } } } + let builtins_dir = env::var("CODEQL_EXTRACTOR_RUST_ROOT") + .map(|path| Path::new(&path).join("tools").join("builtins"))?; + let builtins = fs::read_dir(builtins_dir).context("failed to read builtins directory")?; + for entry in builtins { + let entry = entry.context("failed to read builtins directory")?; + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "rs") { + extractor.extract_without_semantics(&path, ""); + } + } + extractor.emit_extraction_diagnostics(start, &cfg) } diff --git a/rust/tools/builtins/BUILD.bazel b/rust/tools/builtins/BUILD.bazel new file mode 100644 index 000000000000..2c7da705dd20 --- /dev/null +++ b/rust/tools/builtins/BUILD.bazel @@ -0,0 +1,8 @@ +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") + +codeql_pkg_files( + name = "builtins", + srcs = glob(["*.rs"]), + prefix = "builtins", + visibility = ["//rust:__subpackages__"], +) diff --git a/rust/tools/builtins/types.rs b/rust/tools/builtins/types.rs new file mode 100644 index 000000000000..91989b5262b5 --- /dev/null +++ b/rust/tools/builtins/types.rs @@ -0,0 +1,25 @@ +// The Language Prelude: https://doc.rust-lang.org/reference/names/preludes.html#language-prelude + +// Type namespace +// Boolean type +pub struct bool; +// Textual types +pub struct char; +pub struct str; +// Integer types +pub struct i8; +pub struct i16; +pub struct i32; +pub struct i64; +pub struct i128; +pub struct u8; +pub struct u16; +pub struct u32; +pub struct u64; +pub struct u128; +// Machine-dependent integer types +pub struct usize; +pub struct isize; +// floating-point types +pub struct f32; +pub struct f64; From 8e8efedb42ddd87de4519b34f47d77b238aa3577 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Wed, 30 Apr 2025 17:10:33 +0200 Subject: [PATCH 2/3] Rust: adjust integration tests and expected output --- .../hello-project/diagnostics.expected | 2 +- .../hello-project/steps.cargo.expected | 2 ++ .../integration-tests/hello-project/steps.ql | 25 ++++++++++++++++++- .../hello-project/steps.rust-project.expected | 2 ++ .../hello-project/summary.expected | 2 +- .../diagnostics.cargo.expected | 2 +- .../diagnostics.rust-project.expected | 2 +- .../hello-workspace/steps.cargo.expected | 2 ++ .../hello-workspace/steps.ql | 25 ++++++++++++++++++- .../steps.rust-project.expected | 2 ++ .../hello-workspace/summary.cargo.expected | 2 +- .../summary.rust-project.expected | 2 +- .../workspace-with-glob/steps.expected | 2 ++ .../workspace-with-glob/steps.ql | 25 ++++++++++++++++++- 14 files changed, 88 insertions(+), 9 deletions(-) diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected index 146d8514488e..f45877f26d06 100644 --- a/rust/ql/integration-tests/hello-project/diagnostics.expected +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -30,7 +30,7 @@ "pretty": "__REDACTED__" } }, - "numberOfFiles": 5, + "numberOfFiles": 6, "numberOfManifests": 1 }, "severity": "note", diff --git a/rust/ql/integration-tests/hello-project/steps.cargo.expected b/rust/ql/integration-tests/hello-project/steps.cargo.expected index 4deec0653daf..ca256c4f8569 100644 --- a/rust/ql/integration-tests/hello-project/steps.cargo.expected +++ b/rust/ql/integration-tests/hello-project/steps.cargo.expected @@ -1,4 +1,6 @@ | Cargo.toml:0:0:0:0 | LoadManifest(Cargo.toml) | +| file:///types.rs:0:0:0:0 | Extract(/types.rs) | +| file:///types.rs:0:0:0:0 | Parse(/types.rs) | | file://:0:0:0:0 | CrateGraph | | file://:0:0:0:0 | FindManifests | | src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) | diff --git a/rust/ql/integration-tests/hello-project/steps.ql b/rust/ql/integration-tests/hello-project/steps.ql index 17358a1c1000..fe45fc4b6dc8 100644 --- a/rust/ql/integration-tests/hello-project/steps.ql +++ b/rust/ql/integration-tests/hello-project/steps.ql @@ -1,4 +1,27 @@ import codeql.rust.elements.internal.ExtractorStep -from ExtractorStep step +private class Step instanceof ExtractorStep { + string toString() { + result = super.getAction() + "(" + this.getFilePath() + ")" + or + not super.hasFile() and result = super.getAction() + } + + private string getFilePath() { + exists(File file | file = super.getFile() | + exists(file.getRelativePath()) and result = file.getAbsolutePath() + or + not exists(file.getRelativePath()) and result = "/" + file.getBaseName() + ) + } + + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + super.hasLocationInfo(_, startline, startcolumn, endline, endcolumn) and + filepath = this.getFilePath() + } +} + +from Step step select step diff --git a/rust/ql/integration-tests/hello-project/steps.rust-project.expected b/rust/ql/integration-tests/hello-project/steps.rust-project.expected index fa790e6cd7fd..165a770e1cba 100644 --- a/rust/ql/integration-tests/hello-project/steps.rust-project.expected +++ b/rust/ql/integration-tests/hello-project/steps.rust-project.expected @@ -1,3 +1,5 @@ +| file:///types.rs:0:0:0:0 | Extract(/types.rs) | +| file:///types.rs:0:0:0:0 | Parse(/types.rs) | | file://:0:0:0:0 | CrateGraph | | file://:0:0:0:0 | FindManifests | | rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) | diff --git a/rust/ql/integration-tests/hello-project/summary.expected b/rust/ql/integration-tests/hello-project/summary.expected index 1f343b197c0f..15ee83de7adc 100644 --- a/rust/ql/integration-tests/hello-project/summary.expected +++ b/rust/ql/integration-tests/hello-project/summary.expected @@ -9,7 +9,7 @@ | Inconsistencies - Path resolution | 0 | | Inconsistencies - SSA | 0 | | Inconsistencies - data flow | 0 | -| Lines of code extracted | 6 | +| Lines of code extracted | 23 | | Lines of user code extracted | 6 | | Macro calls - resolved | 2 | | Macro calls - total | 2 | diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected index 9429dada005e..146d8514488e 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -30,7 +30,7 @@ "pretty": "__REDACTED__" } }, - "numberOfFiles": 4, + "numberOfFiles": 5, "numberOfManifests": 1 }, "severity": "note", diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected index 9429dada005e..146d8514488e 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -30,7 +30,7 @@ "pretty": "__REDACTED__" } }, - "numberOfFiles": 4, + "numberOfFiles": 5, "numberOfManifests": 1 }, "severity": "note", diff --git a/rust/ql/integration-tests/hello-workspace/steps.cargo.expected b/rust/ql/integration-tests/hello-workspace/steps.cargo.expected index 32a3b1110247..03c81ea6fb71 100644 --- a/rust/ql/integration-tests/hello-workspace/steps.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/steps.cargo.expected @@ -5,6 +5,8 @@ | exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | | exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) | | exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| file:///types.rs:0:0:0:0 | Extract(/types.rs) | +| file:///types.rs:0:0:0:0 | Parse(/types.rs) | | file://:0:0:0:0 | CrateGraph | | file://:0:0:0:0 | FindManifests | | lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) | diff --git a/rust/ql/integration-tests/hello-workspace/steps.ql b/rust/ql/integration-tests/hello-workspace/steps.ql index 17358a1c1000..fe45fc4b6dc8 100644 --- a/rust/ql/integration-tests/hello-workspace/steps.ql +++ b/rust/ql/integration-tests/hello-workspace/steps.ql @@ -1,4 +1,27 @@ import codeql.rust.elements.internal.ExtractorStep -from ExtractorStep step +private class Step instanceof ExtractorStep { + string toString() { + result = super.getAction() + "(" + this.getFilePath() + ")" + or + not super.hasFile() and result = super.getAction() + } + + private string getFilePath() { + exists(File file | file = super.getFile() | + exists(file.getRelativePath()) and result = file.getAbsolutePath() + or + not exists(file.getRelativePath()) and result = "/" + file.getBaseName() + ) + } + + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + super.hasLocationInfo(_, startline, startcolumn, endline, endcolumn) and + filepath = this.getFilePath() + } +} + +from Step step select step diff --git a/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected b/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected index e9a65e0c7be5..0cf90cf71e02 100644 --- a/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected @@ -4,6 +4,8 @@ | exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | | exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) | | exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| file:///types.rs:0:0:0:0 | Extract(/types.rs) | +| file:///types.rs:0:0:0:0 | Parse(/types.rs) | | file://:0:0:0:0 | CrateGraph | | file://:0:0:0:0 | FindManifests | | lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) | diff --git a/rust/ql/integration-tests/hello-workspace/summary.cargo.expected b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected index 5912f7d69baf..c845417a6244 100644 --- a/rust/ql/integration-tests/hello-workspace/summary.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected @@ -9,7 +9,7 @@ | Inconsistencies - Path resolution | 0 | | Inconsistencies - SSA | 0 | | Inconsistencies - data flow | 0 | -| Lines of code extracted | 9 | +| Lines of code extracted | 26 | | Lines of user code extracted | 9 | | Macro calls - resolved | 2 | | Macro calls - total | 2 | diff --git a/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected index 5912f7d69baf..c845417a6244 100644 --- a/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected @@ -9,7 +9,7 @@ | Inconsistencies - Path resolution | 0 | | Inconsistencies - SSA | 0 | | Inconsistencies - data flow | 0 | -| Lines of code extracted | 9 | +| Lines of code extracted | 26 | | Lines of user code extracted | 9 | | Macro calls - resolved | 2 | | Macro calls - total | 2 | diff --git a/rust/ql/integration-tests/workspace-with-glob/steps.expected b/rust/ql/integration-tests/workspace-with-glob/steps.expected index 4b0e6ed828b6..0ee55e79623f 100644 --- a/rust/ql/integration-tests/workspace-with-glob/steps.expected +++ b/rust/ql/integration-tests/workspace-with-glob/steps.expected @@ -1,6 +1,8 @@ | Cargo.toml:0:0:0:0 | LoadManifest(Cargo.toml) | | exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | | exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| file:///types.rs:0:0:0:0 | Extract(/types.rs) | +| file:///types.rs:0:0:0:0 | Parse(/types.rs) | | file://:0:0:0:0 | CrateGraph | | file://:0:0:0:0 | FindManifests | | lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) | diff --git a/rust/ql/integration-tests/workspace-with-glob/steps.ql b/rust/ql/integration-tests/workspace-with-glob/steps.ql index 17358a1c1000..fe45fc4b6dc8 100644 --- a/rust/ql/integration-tests/workspace-with-glob/steps.ql +++ b/rust/ql/integration-tests/workspace-with-glob/steps.ql @@ -1,4 +1,27 @@ import codeql.rust.elements.internal.ExtractorStep -from ExtractorStep step +private class Step instanceof ExtractorStep { + string toString() { + result = super.getAction() + "(" + this.getFilePath() + ")" + or + not super.hasFile() and result = super.getAction() + } + + private string getFilePath() { + exists(File file | file = super.getFile() | + exists(file.getRelativePath()) and result = file.getAbsolutePath() + or + not exists(file.getRelativePath()) and result = "/" + file.getBaseName() + ) + } + + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + super.hasLocationInfo(_, startline, startcolumn, endline, endcolumn) and + filepath = this.getFilePath() + } +} + +from Step step select step From 778f46d5be80ff1657da988c5a041fb5b74e06c3 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Thu, 1 May 2025 22:21:48 +0200 Subject: [PATCH 3/3] Rust: adjust unit tests and expected output --- rust/ql/test/TestUtils.qll | 6 +++++- rust/ql/test/query-tests/diagnostics/LinesOfCode.expected | 2 +- .../query-tests/diagnostics/SummaryStatsReduced.expected | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/ql/test/TestUtils.qll b/rust/ql/test/TestUtils.qll index ce494ff5a608..dc75d109a339 100644 --- a/rust/ql/test/TestUtils.qll +++ b/rust/ql/test/TestUtils.qll @@ -1,6 +1,6 @@ private import rust -predicate toBeTested(Element e) { not e instanceof CrateElement } +predicate toBeTested(Element e) { not e instanceof CrateElement and not e instanceof Builtin } class CrateElement extends Element { CrateElement() { @@ -9,3 +9,7 @@ class CrateElement extends Element { any(Crate c).getModule() = this.(AstNode).getParentNode*() } } + +class Builtin extends AstNode { + Builtin() { this.getFile().getAbsolutePath().matches("%/builtins/%.rs") } +} diff --git a/rust/ql/test/query-tests/diagnostics/LinesOfCode.expected b/rust/ql/test/query-tests/diagnostics/LinesOfCode.expected index 5fa7b20e01bb..76e48043d0d4 100644 --- a/rust/ql/test/query-tests/diagnostics/LinesOfCode.expected +++ b/rust/ql/test/query-tests/diagnostics/LinesOfCode.expected @@ -1 +1 @@ -| 60 | +| 77 | diff --git a/rust/ql/test/query-tests/diagnostics/SummaryStatsReduced.expected b/rust/ql/test/query-tests/diagnostics/SummaryStatsReduced.expected index ed21d9772fce..793ed90a482a 100644 --- a/rust/ql/test/query-tests/diagnostics/SummaryStatsReduced.expected +++ b/rust/ql/test/query-tests/diagnostics/SummaryStatsReduced.expected @@ -9,7 +9,7 @@ | Inconsistencies - Path resolution | 0 | | Inconsistencies - SSA | 0 | | Inconsistencies - data flow | 0 | -| Lines of code extracted | 60 | +| Lines of code extracted | 77 | | Lines of user code extracted | 60 | | Macro calls - resolved | 8 | | Macro calls - total | 9 |