Skip to content

Make rule-preprocessor process a single directory at a time #372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nunoplopes merged 2 commits into from
Sep 18, 2026
Merged

Make rule-preprocessor process a single directory at a time #372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f94decf
Make rule-preprocessor process a single directory at a time
joaotgouveia Sep 16, 2026
bcdf914
Split independent commands
joaotgouveia Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 66 additions & 26 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,18 +141,53 @@ set(RULES_IR_DIR "${CMAKE_BINARY_DIR}/rules")

set(RUSTLS_HEADER "${PROJECT_SOURCE_DIR}/rules/rustls/rustls.h")

set(PREPROCESSOR_BIN "${CMAKE_BINARY_DIR}/target_preprocessor/release/rule-preprocessor")
set(PREPROCESSOR_STAMP "${CMAKE_BINARY_DIR}/.rule-preprocessor.stamp")

file(GLOB_RECURSE rule_preprocessor_sources
${PROJECT_SOURCE_DIR}/rule-preprocessor/src/*.rs)

execute_process(
COMMAND ${CMAKE_COMMAND} -E env RUSTUP_TOOLCHAIN=${RUST_NIGHTLY_VERSION}
rustc --print sysroot
OUTPUT_VARIABLE NIGHTLY_SYSROOT
OUTPUT_STRIP_TRAILING_WHITESPACE
)

add_custom_command(

@nunoplopes nunoplopes Sep 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be split in two rules, each output its own binary. Then there's no need for this PREPROCESSOR_STAMP

@joaotgouveia joaotgouveia Sep 17, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll split building the rules crate and the rule-preprocessor, keeping both commands together is not necessary since they are independent. However, preprocessing rust rules depends on both commands, one ensures the binary itself is build, and the other ensures that the rules crate dependencies are built.

Since preprocessing rust rules depends on the built dependencies, and not on the RUSTLS_HEADER the rules crate command outputs, won't it be clearer to keep the PREPROCESSOR_STAMP as an output of this command as well?

OUTPUT ${RUSTLS_HEADER} ${PREPROCESSOR_STAMP}
COMMAND cargo +${RUST_STABLE_VERSION} build --release
--manifest-path "${PROJECT_SOURCE_DIR}/rules/Cargo.toml"
COMMAND ${CMAKE_COMMAND} -E touch "${PREPROCESSOR_STAMP}"
DEPENDS ${rule_preprocessor_sources}
"${PROJECT_SOURCE_DIR}/rules/build.rs"
"${PROJECT_SOURCE_DIR}/rules/Cargo.toml"
"${RUST_STAMP_FILE}"
VERBATIM
)

add_custom_command(
OUTPUT ${PREPROCESSOR_BIN}
COMMAND ${CMAKE_COMMAND} -E env
CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target_preprocessor
cargo +${RUST_NIGHTLY_VERSION} build --release
--manifest-path "${PROJECT_SOURCE_DIR}/rule-preprocessor/Cargo.toml"
DEPENDS ${rule_preprocessor_sources}
"${RUST_STAMP_FILE}"
VERBATIM
)

file(GLOB rule_subdirs ${PROJECT_SOURCE_DIR}/rules/*)
set(cpp_rules_ir_outputs)
set(rust_rules_ir_outputs)
set(rust_rules_inputs)
foreach(_rule_dir IN LISTS rule_subdirs)
file(GLOB _srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
if(NOT _srcs)
file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
if(NOT _cpp_srcs)
continue()
endif()
get_filename_component(_rule_name ${_rule_dir} NAME)
set(_out_dir ${RULES_IR_DIR}/${_rule_name})
set(_out ${_out_dir}/ir_src.json)
set(_cpp_out ${_out_dir}/ir_src.json)
set(_cxxflags)
set(_cxxflags_file)
if(EXISTS ${_rule_dir}/cxxflags)
Expand All @@ -163,39 +198,44 @@ foreach(_rule_dir IN LISTS rule_subdirs)
endforeach()
endif()
add_custom_command(
OUTPUT ${_out}
OUTPUT ${_cpp_out}
COMMAND ${CMAKE_COMMAND} -E make_directory ${_out_dir}
COMMAND $<TARGET_FILE:cpp-rule-preprocessor> --dir ${_rule_dir} --out ${_out} ${_cxxflags}
DEPENDS ${_srcs} ${_cxxflags_file} ${PROJECT_SOURCE_DIR}/cpp2rust/cpp_rule_preprocessor.cpp ${RUSTLS_HEADER}
COMMAND $<TARGET_FILE:cpp-rule-preprocessor> --dir ${_rule_dir} --out ${_cpp_out} ${_cxxflags}
DEPENDS ${_cpp_srcs} ${_cxxflags_file} ${PROJECT_SOURCE_DIR}/cpp2rust/cpp_rule_preprocessor.cpp ${RUSTLS_HEADER}
VERBATIM
)
list(APPEND cpp_rules_ir_outputs ${_out})
list(APPEND cpp_rules_ir_outputs ${_cpp_out})

set(_rust_srcs)
set(_rust_out)
if(EXISTS ${_rule_dir}/tgt_unsafe.rs)
list(APPEND rust_rules_ir_outputs ${_out_dir}/ir_unsafe.json)
list(APPEND rust_rules_inputs ${_rule_dir}/tgt_unsafe.rs)
list(APPEND _rust_out ${_out_dir}/ir_unsafe.json)
list(APPEND _rust_srcs ${_rule_dir}/tgt_unsafe.rs)
endif()
if(EXISTS ${_rule_dir}/tgt_refcount.rs)
list(APPEND rust_rules_ir_outputs ${_out_dir}/ir_refcount.json)
list(APPEND rust_rules_inputs ${_rule_dir}/tgt_refcount.rs)
list(APPEND _rust_out ${_out_dir}/ir_refcount.json)
list(APPEND _rust_srcs ${_rule_dir}/tgt_refcount.rs)
endif()
if(NOT _rust_srcs)
continue()
endif()
add_custom_command(
OUTPUT ${_rust_out}
COMMAND ${CMAKE_COMMAND} -E make_directory ${_out_dir}
COMMAND ${CMAKE_COMMAND} -E env
CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target_preprocessor
RUSTUP_TOOLCHAIN=${RUST_NIGHTLY_VERSION}
LD_LIBRARY_PATH=${NIGHTLY_SYSROOT}/lib
DYLD_LIBRARY_PATH=${NIGHTLY_SYSROOT}/lib
${PREPROCESSOR_BIN} ${_out_dir} ${_rule_dir}
DEPENDS ${_rust_srcs} ${PREPROCESSOR_BIN} ${PREPROCESSOR_STAMP}
VERBATIM
)
list(APPEND rust_rules_ir_outputs ${_rust_out})
endforeach()

add_custom_target("preprocess-cpp-rules" ALL DEPENDS ${cpp_rules_ir_outputs})

file(GLOB_RECURSE rule_preprocessor_sources
${PROJECT_SOURCE_DIR}/rule-preprocessor/src/*.rs)

add_custom_command(
OUTPUT ${rust_rules_ir_outputs} ${RUSTLS_HEADER}
COMMAND cargo +${RUST_STABLE_VERSION} build --release --manifest-path "${PROJECT_SOURCE_DIR}/rules/Cargo.toml"
COMMAND ${CMAKE_COMMAND} -E env
CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target_preprocessor
cargo +${RUST_NIGHTLY_VERSION} run --release --manifest-path "${PROJECT_SOURCE_DIR}/rule-preprocessor/Cargo.toml"
-- "${RULES_IR_DIR}"
DEPENDS ${rust_rules_inputs} ${rule_preprocessor_sources} "${RUST_STAMP_FILE}"
VERBATIM
)

add_custom_target("preprocess-rust-rules" ALL
DEPENDS ${rust_rules_ir_outputs})

Expand Down
70 changes: 54 additions & 16 deletions rule-preprocessor/src/ir.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,32 @@

use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
use std::iter::Flatten;
use std::path::{Path, PathBuf};

pub enum Model {
Refcount,
Unsafe,
}

impl Model {
pub const ALL: [Model; 2] = [Model::Refcount, Model::Unsafe];

pub fn src_filename(self) -> &'static str {
match self {
Model::Refcount => "tgt_refcount.rs",
Model::Unsafe => "tgt_unsafe.rs",
}
}

pub fn ir_filename(self) -> &'static str {
match self {
Model::Refcount => "ir_refcount.json",
Model::Unsafe => "ir_unsafe.json",
}
}
}

fn validate_consecutive_keys<'a>(
keys: impl Iterator<Item = &'a String>,
prefix: char,
Expand Down Expand Up @@ -231,30 +255,44 @@ pub enum RuleIr {
/// Per-file IR: rule name -> FnIr or TypeIr
pub type FileIr = BTreeMap<String, RuleIr>;

/// All IR for all rule files.
/// Per-dir IR
pub struct RulesIR {
pub all_ir: HashMap<String, FileIr>,
pub crate_root: PathBuf,
pub dir: PathBuf,
pub refcount_ir: Option<FileIr>,
pub unsafe_ir: Option<FileIr>,
}

impl<'a> IntoIterator for &'a RulesIR {
type Item = (Model, &'a FileIr);
type IntoIter = Flatten<std::array::IntoIter<Option<Self::Item>, 2>>;

fn into_iter(self) -> Self::IntoIter {
[
self.refcount_ir.as_ref().map(|ir| (Model::Refcount, ir)),
self.unsafe_ir.as_ref().map(|ir| (Model::Unsafe, ir)),
]
.into_iter()
.flatten()
}
}

impl RulesIR {
pub fn write_ir(&self, out_dir: &Path) {
for (rule_path, file_ir) in &self.all_ir {
let rule_path = Path::new(rule_path);
let rule_name = rule_path.parent().unwrap().file_name().unwrap();
let json_name = rule_path
.file_name()
.unwrap()
.to_str()
.unwrap()
.replace("tgt_", "ir_")
.replace(".rs", ".json");
let json_path = out_dir.join(rule_name).join(json_name);

std::fs::create_dir_all(json_path.parent().unwrap()).unwrap();
for (model, file_ir) in self {
let json_path = out_dir.join(model.ir_filename());
let json = serde_json::to_string_pretty(file_ir).unwrap();
std::fs::write(&json_path, format!("{json}\n")).unwrap();
println!("{}", json_path.display());
}
}

pub fn get_mut(&mut self, file: &str) -> Option<&mut FileIr> {
if file.ends_with(Model::Refcount.src_filename()) {
self.refcount_ir.as_mut()
} else if file.ends_with(Model::Unsafe.src_filename()) {
self.unsafe_ir.as_mut()
} else {
None
}
}
}
17 changes: 11 additions & 6 deletions rule-preprocessor/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,15 @@ fn main() {
let mut args = std::env::args().skip(1);
let out_dir = args
.next()
.expect("usage: rule-preprocessor <out-dir> [rules-crate-dir]");
let in_dir = args.next().unwrap_or_else(|| "../rules".to_string());
SemanticAnalysis::run(SyntacticAnalysis::run(
&std::fs::canonicalize(&in_dir).unwrap(),
))
.write_ir(&std::path::PathBuf::from(out_dir));
.expect("usage: rule-preprocessor <out-dir> <rule-dir>");
let in_dir = args
.next()
.expect("usage: rule-preprocessor <out-dir> <rule-dir>");

let out = std::path::PathBuf::from(out_dir);
SemanticAnalysis::run(
SyntacticAnalysis::run(&std::fs::canonicalize(&in_dir).unwrap()),
&out,
)
.write_ir(&out);
}
34 changes: 25 additions & 9 deletions rule-preprocessor/src/semantic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// Copyright (c) 2022-present INESC-ID.
// Distributed under the MIT license that can be found in the LICENSE file.

use crate::ir::RulesIR;
use crate::ir::{Access, FnIr, RuleIr};
use crate::ir::{Access, FnIr, RuleIr, RulesIR};
use std::collections::HashMap;
use std::path::{Path, PathBuf};

pub struct SemanticAnalysis;

impl SemanticAnalysis {
pub fn run(ir: RulesIR) -> RulesIR {
let args = build_rustc_args(&ir.crate_root);
pub fn run(ir: RulesIR, out_dir: &Path) -> RulesIR {
let crate_root = write_crate_root(&ir, out_dir);
let args = build_rustc_args(&crate_root);
let mut resolver = MethodResolver { ir };

if rustc_driver::catch_fatal_errors(|| {
Expand All @@ -26,14 +26,30 @@ impl SemanticAnalysis {
}
}

fn write_crate_root(ir: &RulesIR, out_dir: &Path) -> PathBuf {
let mut buf = String::from("#![allow(warnings)]\n");
for (model, _) in ir {
let filename = model.src_filename();
let stem = filename.strip_suffix(".rs").unwrap();
let source = ir.dir.join(filename);
buf.push_str(&format!(
"#[path = r#\"{}\"#]\npub mod rule_{stem};\n",
source.display()
));
}

let path = out_dir.join("crate_root.rs");
std::fs::write(&path, buf).unwrap();
path
}

fn build_rustc_args(crate_root: &Path) -> Vec<String> {
let sysroot = get_sysroot();
let lib_path = crate_root.join("src").join("lib.rs");
let build_dir = find_build_dir();

let mut args = vec![
"rustc".to_string(),
lib_path.to_string_lossy().to_string(),
crate_root.to_string_lossy().to_string(),
"--crate-name".to_string(),
"rules".to_string(),
"--crate-type".to_string(),
Expand Down Expand Up @@ -217,7 +233,7 @@ struct MethodResolver {

impl MethodResolver {
fn resolve_rule<'tcx>(&mut self, tcx: rustc_middle::ty::TyCtxt<'tcx>, f: &FnDecl<'tcx>) {
let Some(file_ir) = self.ir.all_ir.get_mut(&f.source_file) else {
let Some(file_ir) = self.ir.get_mut(&f.source_file) else {
return;
};
match file_ir.get_mut(&f.name) {
Expand All @@ -231,14 +247,14 @@ impl MethodResolver {
}

fn assert_no_unknowns(&self) {
for (source_file, file_ir) in &self.ir.all_ir {
for (model, file_ir) in &self.ir {
for (rule_name, rule) in file_ir {
let RuleIr::Fn(fn_ir) = rule else { continue };
assert!(
!fn_ir.has_unknowns(),
"unresolved access=\"unknown\" in {} ({})",
rule_name,
source_file
self.ir.dir.join(model.src_filename()).display()
);
}
}
Expand Down
51 changes: 14 additions & 37 deletions rule-preprocessor/src/syntactic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ use cfg_expr::Expression;
use cfg_expr::expr::{Predicate, TargetPredicate};
use ra_ap_syntax::ast::{HasAttrs, HasGenericParams, HasName, HasTypeBounds};
use ra_ap_syntax::{AstNode, SyntaxKind, ast, match_ast};
use std::collections::{BTreeMap, HashMap};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

use crate::ir::{
Access, BodyFragment, FileIr, FnIr, MethodCallInner, PlaceholderInner, RuleIr, RulesIR,
Access, BodyFragment, FileIr, FnIr, MethodCallInner, Model, PlaceholderInner, RuleIr, RulesIR,
TypeInfo, TypeIr,
};

Expand Down Expand Up @@ -87,46 +87,23 @@ fn cfg_matches_host(fn_item: &ast::Fn) -> bool {
pub struct SyntacticAnalysis;

impl SyntacticAnalysis {
pub fn run(crate_root: &Path) -> RulesIR {
let rule_files = Self::collect_rule_files(crate_root);
let mut all_ir = HashMap::new();

for rule_file in &rule_files {
let source = std::fs::read_to_string(rule_file).unwrap();
let file_ir = Self::parse_rule_file(&source, rule_file);

let canonical = rule_file
.canonicalize()
.unwrap_or_else(|_| rule_file.clone())
.to_string_lossy()
.to_string();
all_ir.insert(canonical, file_ir);
}
pub fn run(rule_dir: &Path) -> RulesIR {
let [refcount_ir, unsafe_ir] = Self::collect_rule_files(rule_dir).map(|opt| {
opt.map(|f| Self::parse_rule_file(&std::fs::read_to_string(&f).unwrap(), &f))
});

RulesIR {
all_ir,
crate_root: crate_root.to_path_buf(),
dir: rule_dir.to_path_buf(),
refcount_ir,
unsafe_ir,
}
}

fn collect_rule_files(dir: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
let Ok(entries) = std::fs::read_dir(dir) else {
return out;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
out.extend(Self::collect_rule_files(&path));
} else if let Some(name) = path.file_name().and_then(|s| s.to_str())
&& name.starts_with("tgt_")
&& name.ends_with(".rs")
{
out.push(path);
}
}
out.sort();
out
fn collect_rule_files(dir: &Path) -> [Option<PathBuf>; 2] {
Model::ALL.map(|m| {
let p = dir.join(m.src_filename());
p.exists().then_some(p)
})
}

fn parse_rule_file(source: &str, path: &Path) -> FileIr {
Expand Down
Loading
Loading