diff --git a/compiler/rustc_attr_parsing/src/attributes/rad_protected.rs b/compiler/rustc_attr_parsing/src/attributes/rad_protected.rs index 7e2d545d61faf..53dae45c36b79 100644 --- a/compiler/rustc_attr_parsing/src/attributes/rad_protected.rs +++ b/compiler/rustc_attr_parsing/src/attributes/rad_protected.rs @@ -8,7 +8,7 @@ use crate::target_checking::{ALL_TARGETS, AllowedTargets}; pub(crate) struct RadProtectedParser; impl NoArgsAttributeParser for RadProtectedParser { - const PATH: &[Symbol] = &[sym::rad_protected]; + const PATH: &[Symbol] = &[sym::rad_protected_mir]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Ignore; const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs index f671f59f983a3..36a40fe8fdeae 100644 --- a/compiler/rustc_builtin_macros/src/lib.rs +++ b/compiler/rustc_builtin_macros/src/lib.rs @@ -45,6 +45,7 @@ mod global_allocator; mod iter; mod log_syntax; mod pattern_type; +mod rad_protected; mod source_util; mod test; mod trace_macros; @@ -118,6 +119,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) { eii_declaration: eii::eii_declaration, eii_shared_macro: eii::eii_shared_macro, global_allocator: global_allocator::expand, + rad_protected: rad_protected::triplicate, test: test::expand_test, test_case: test::expand_test_case, unsafe_eii: eii::unsafe_eii, diff --git a/compiler/rustc_builtin_macros/src/rad_protected/mod.rs b/compiler/rustc_builtin_macros/src/rad_protected/mod.rs new file mode 100644 index 0000000000000..dcc313066936e --- /dev/null +++ b/compiler/rustc_builtin_macros/src/rad_protected/mod.rs @@ -0,0 +1,4 @@ +mod triplicate; +mod parse_attr_opts; + +pub(crate) use triplicate::triplicate; diff --git a/compiler/rustc_builtin_macros/src/rad_protected/parse_attr_opts.rs b/compiler/rustc_builtin_macros/src/rad_protected/parse_attr_opts.rs new file mode 100644 index 0000000000000..f90829d7f84b1 --- /dev/null +++ b/compiler/rustc_builtin_macros/src/rad_protected/parse_attr_opts.rs @@ -0,0 +1,43 @@ +use rustc_ast as ast; +use rustc_ast::MetaItemInner; +use rustc_expand::base::ExtCtxt; +use thin_vec::{thin_vec, ThinVec}; +use rustc_span::sym; + +pub(super) struct AttrOpts { + triplicate_unsafe: bool, +} + +impl AttrOpts { + pub(super) fn triplicate_unsafe(&self) -> bool { + self.triplicate_unsafe + } +} + +pub(super) fn parse_attr_opts(cx: &ExtCtxt<'_>, meta_item: &ast::MetaItem) -> Option { + + let attr_opts: ThinVec = match meta_item.kind { + ast::MetaItemKind::List(ref vec) => vec.clone(), + ast::MetaItemKind::Word => thin_vec![], + _ => { + cx.dcx().span_err(meta_item.span, "unsupported options kind in `#[rad_protected]`"); + return None; + } + }; + + let mut triplicate_unsafe = false; + + for opt in attr_opts { + match opt { + MetaItemInner::MetaItem(opt) if opt.has_name(sym::triplicate_unsafe) => { + triplicate_unsafe = true; + } + _ => { + cx.dcx().span_err(opt.span(), "unsupported option in `#[rad_protected]`"); + return None; + } + } + } + + Some(AttrOpts { triplicate_unsafe }) +} diff --git a/compiler/rustc_builtin_macros/src/rad_protected/triplicate.rs b/compiler/rustc_builtin_macros/src/rad_protected/triplicate.rs new file mode 100644 index 0000000000000..928f3f19ca14b --- /dev/null +++ b/compiler/rustc_builtin_macros/src/rad_protected/triplicate.rs @@ -0,0 +1,85 @@ +use rustc_ast as ast; +use rustc_expand::base::{Annotatable, ExtCtxt}; +use rustc_span::{Span, symbol::Ident, sym, DUMMY_SP}; +use thin_vec::thin_vec; +use super::parse_attr_opts::parse_attr_opts; + +pub(crate) fn triplicate( + cx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + mut item: Annotatable, +) -> Vec { + + let Some(opts) = parse_attr_opts(cx, meta_item) else { + return vec![item]; + }; + + if opts.triplicate_unsafe() { + let valid = match &mut item { + Annotatable::Expr(expr) + if matches!(&expr.kind, ast::ExprKind::Block(block, _) + if matches!(block.rules, ast::BlockCheckMode::Unsafe(_)) + ) => { + expr.attrs.push(cx.attr_nested_word( + sym::rad_protected_mir, + sym::triplicate_unsafe, + DUMMY_SP, + )); + true + } + _ => false, + }; + + if !valid { + cx.dcx().span_err( + span, + "`#[rad_protected(triplicate_unsafe)]` can only be applied to `unsafe` blocks", + ); + } + + return vec![item]; + } + + let Annotatable::Item(mut item) = item else { + cx.dcx().span_err(span, "`#[rad_protected]` can only be applied to functions"); + return vec![item]; + }; + + let ast::Item { + kind: ast::ItemKind::Fn(func), + .. + } = &mut *item + else { + cx.dcx().span_err(span, "`#[rad_protected]` can only be applied to functions"); + return vec![Annotatable::Item(item)]; + }; + + let func_body = match &mut func.body { + Some(b) => b, + None => { + cx.dcx().span_err(span, "`#[rad_protected]` can only be applied to functions with a body"); + return vec![Annotatable::Item(item)]; + } + }; + + func_body.stmts.insert(0, cx.stmt_let( + DUMMY_SP, + false, + Ident::new(sym::__guard, DUMMY_SP), + cx.expr_call_global( + DUMMY_SP, + vec![ + Ident::new(sym::std, DUMMY_SP), + Ident::new(sym::RadRustRuntime, DUMMY_SP), + Ident::new(sym::triplicate_process, DUMMY_SP), + ], + thin_vec![cx.expr_usize(DUMMY_SP, 0usize)] + ) + )); + + let mir_attr = cx.attr_word(sym::rad_protected_mir, DUMMY_SP); + item.attrs.push(mir_attr); + + vec![Annotatable::Item(item)] +} diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index 19a2d65762e86..845d4016fae6d 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -769,6 +769,21 @@ impl<'a> ExtCtxt<'a> { ) } + // Builds `#[unsafe(name = val)]`. + // + // Note: `span` is used for both the identifier and the value. + pub fn attr_name_value_str_unsafe(&self, name: Symbol, val: Symbol, span: Span) -> ast::Attribute { + let g = &self.sess.psess.attr_id_generator; + attr::mk_attr_name_value_str( + g, + ast::AttrStyle::Outer, + ast::Safety::Unsafe(span), + name, + val, + span, + ) + } + // Builds `#[outer(inner)]`. pub fn attr_nested_word(&self, outer: Symbol, inner: Symbol, span: Span) -> ast::Attribute { let g = &self.sess.psess.attr_id_generator; diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index 5914fee8315d4..4658ff3012acd 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -23,6 +23,8 @@ pub mod expand; pub mod module; pub mod proc_macro; +pub mod patch_unsafe; + pub fn provide(providers: &mut rustc_middle::query::Providers) { providers.derive_macro_expansion = proc_macro::provide_derive_macro_expansion; } diff --git a/compiler/rustc_expand/src/patch_unsafe.rs b/compiler/rustc_expand/src/patch_unsafe.rs new file mode 100644 index 0000000000000..3304fb197f6de --- /dev/null +++ b/compiler/rustc_expand/src/patch_unsafe.rs @@ -0,0 +1,115 @@ +use rustc_ast as ast; +use rustc_ast::mut_visit::{self, MutVisitor}; +use crate::base::ExtCtxt; +use rustc_span::{symbol::Ident, Symbol, sym, DUMMY_SP}; +use thin_vec::{ThinVec, thin_vec}; +use rustc_ast::MetaItemInner; + +pub fn patch_unsafe_blocks(cx: &mut ExtCtxt<'_>, krate: &mut ast::Crate) { + if cx.sess.opts.unstable_opts.force_unstable_if_unmarked { + return; + } + let mut visitor = UnsafeBlockRewriter { cx }; + visitor.visit_crate(krate); +} + +struct DummyIdAssigner<'a, 'cx> { + cx: &'a mut ExtCtxt<'cx>, +} + +impl MutVisitor for DummyIdAssigner<'_, '_> { + fn visit_id(&mut self, id: &mut ast::NodeId) { + if *id == ast::DUMMY_NODE_ID { + *id = self.cx.resolver.next_node_id(); + } + } +} + +struct UnsafeBlockRewriter<'a, 'cx> { + cx: &'a mut ExtCtxt<'cx>, +} + +impl MutVisitor for UnsafeBlockRewriter<'_, '_> { + fn visit_expr(&mut self, expr: &mut ast::Expr) { + + if let ast::ExprKind::Block(block, _) = &mut expr.kind { + if matches!(block.rules, ast::BlockCheckMode::Unsafe(_)) { + + if !skip_patch(&mut expr.attrs) { + patch_unsafe_block(self.cx, block); + } + return; + } + } + + mut_visit::walk_expr(self, expr); + } +} + +fn patch_unsafe_block(cx: &mut ExtCtxt<'_>, block: &mut ast::Block) { + let runtime_method_call = |cx: &mut ExtCtxt<'_>, name: Symbol| { + cx.expr_call_global( + DUMMY_SP, + vec![ + Ident::new(sym::std, DUMMY_SP), + Ident::new(sym::RadRustRuntime, DUMMY_SP), + Ident::new(name, DUMMY_SP), + ], + thin_vec![], + ) + }; + + let enter_call = runtime_method_call(cx, sym::enter_critical_section); + let exit_call = runtime_method_call(cx, sym::exit_critical_section); + + let inner_block = cx.block(block.span, block.stmts.clone()); + let if_stmt = cx.stmt_expr(cx.expr_if(DUMMY_SP, + enter_call, + cx.expr_block(inner_block), + None + )); + let exit_call_stmt = cx.stmt_expr(exit_call); + + let mut assigner = DummyIdAssigner { cx }; + let if_stmt = assign_stmt(&mut assigner, if_stmt); + let exit_call_stmt = assign_stmt(&mut assigner, exit_call_stmt); + + block.stmts = thin_vec![if_stmt, exit_call_stmt]; +} + +fn skip_patch(attrs: &mut ThinVec) -> bool { + let mut removed = false; + + attrs.retain(|attr| { + let keep = !attr.meta().is_some_and(is_skip_attr); + + if !keep { + removed = true; + } + keep + }); + + removed +} + +fn is_skip_attr(meta: ast::MetaItem) -> bool { + if !meta.has_name(sym::rad_protected_mir) { + return false; + } + + let ast::MetaItemKind::List(items) = &meta.kind else { + return false; + }; + + items.iter().any(|item| { + matches!(item, MetaItemInner::MetaItem(mi) if mi.has_name(sym::triplicate_unsafe)) + }) +} + +fn assign_stmt(assigner: &mut DummyIdAssigner<'_, '_>, stmt: ast::Stmt) -> ast::Stmt { + assigner + .flat_map_stmt(stmt) + .into_iter() + .next() + .expect("statement unexpectedly removed") +} diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index fe2642d508ddd..404ed11152d9a 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -903,9 +903,10 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ EncodeCrossCrate::Yes, pin_ergonomics, experimental!(pin_v2), ), - // Radshield protection attribute (ungated for easy testing) + // Radshield MIR pass protection attribute (ungated for easy testing) + // This attribute is used internally by rad_protected, and is not intended for use by the user ungated!( - rad_protected, Normal, template!(Word), WarnFollowing, + rad_protected_mir, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::No, ), diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs index 470d3816b9bae..3cce89b071950 100644 --- a/compiler/rustc_hir/src/attrs/data_structures.rs +++ b/compiler/rustc_hir/src/attrs/data_structures.rs @@ -1234,7 +1234,7 @@ pub enum AttributeKind { /// Represents `#[profiler_runtime]` ProfilerRuntime, - /// Represents `#[rad_protected]` + /// Represents `#[rad_protected_mir]` RadProtected(Span), /// Represents [`#[recursion_limit]`](https://doc.rust-lang.org/reference/attributes/limits.html#the-recursion_limit-attribute) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 15addd2407857..06dc7b61f959c 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -15,6 +15,7 @@ use rustc_data_structures::sync::{AppendOnlyIndexVec, FreezeLock, WorkerLocal, p use rustc_data_structures::thousands; use rustc_errors::timings::TimingSection; use rustc_expand::base::{ExtCtxt, LintStoreExpand}; +use rustc_expand::patch_unsafe::patch_unsafe_blocks; use rustc_feature::Features; use rustc_fs_util::try_canonicalize; use rustc_hir::attrs::AttributeKind; @@ -213,12 +214,14 @@ fn configure_and_expand( let mut ecx = ExtCtxt::new(sess, cfg, resolver, Some(&lint_store)); ecx.num_standard_library_imports = num_standard_library_imports; // Expand macros now! - let krate = sess.time("expand_crate", || ecx.monotonic_expander().expand_crate(krate)); + let mut krate = sess.time("expand_crate", || ecx.monotonic_expander().expand_crate(krate)); if ecx.nb_macro_errors > 0 { sess.dcx().abort_if_errors(); } + patch_unsafe_blocks(&mut ecx, &mut krate); + // The rest is error reporting and stats sess.psess.buffered_lints.with_lock(|buffered_lints: &mut Vec| { diff --git a/compiler/rustc_mir_build/src/builder/checkpoint.rs b/compiler/rustc_mir_build/src/builder/checkpoint.rs new file mode 100644 index 0000000000000..744ddcedaf06c --- /dev/null +++ b/compiler/rustc_mir_build/src/builder/checkpoint.rs @@ -0,0 +1,44 @@ +use rustc_hir::def_id::LocalDefId; +use rustc_middle::mir::{TerminatorKind, Operand, Place, UnwindAction, CallSource, BasicBlock}; +use rustc_middle::ty::TyCtxt; +use rustc_span::{sym, Span}; +use rustc_hir::find_attr; + +use super::Builder; + +impl<'a, 'tcx> Builder<'a, 'tcx> { + pub(super) fn inject_checkpoint_marker(&mut self, block: BasicBlock, span: Span) -> BasicBlock { + if !Self::is_checkpoint(self.tcx, self.def_id) { + return block; + } + + let Some(marker_def_id) = self.tcx.get_diagnostic_item(sym::__checkpoint) else { + return block; + }; + + let next = self.cfg.start_new_block(); + let source_info = self.source_info(span); + let func = Operand::function_handle(self.tcx, marker_def_id, [], span); + let destination = Place::from(self.temp(self.tcx.types.unit, span)); + + self.cfg.terminate( + block, + source_info, + TerminatorKind::Call { + func, + args: Box::new([]), + destination, + target: Some(next), + unwind: UnwindAction::Continue, + call_source: CallSource::Misc, + fn_span: span, + }, + ); + + next + } + + fn is_checkpoint(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { + find_attr!(tcx, def_id, RadProtected(_)) + } +} diff --git a/compiler/rustc_mir_build/src/builder/expr/into.rs b/compiler/rustc_mir_build/src/builder/expr/into.rs index 24d184121ffd0..52cc2f8d608f4 100644 --- a/compiler/rustc_mir_build/src/builder/expr/into.rs +++ b/compiler/rustc_mir_build/src/builder/expr/into.rs @@ -449,6 +449,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => { + if from_hir_call { + block = this.inject_checkpoint_marker(block, fn_span); + } + let fun = unpack!(block = this.as_local_operand(block, fun)); let args: Box<[_]> = args .into_iter() diff --git a/compiler/rustc_mir_build/src/builder/mod.rs b/compiler/rustc_mir_build/src/builder/mod.rs index c8ca7bfccc07c..ccea1bbb9493e 100644 --- a/compiler/rustc_mir_build/src/builder/mod.rs +++ b/compiler/rustc_mir_build/src/builder/mod.rs @@ -1276,5 +1276,6 @@ mod expr; mod matches; mod misc; mod scope; +mod checkpoint; pub(crate) use expr::category::Category as ExprCategory; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 125bc5decf119..850539412a8a8 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -51,6 +51,7 @@ mod lint; mod lint_tail_expr_drop_order; mod liveness; mod patch; +mod rad_protected_liveness_analysis; mod shim; mod ssa; mod trivial_const; diff --git a/compiler/rustc_mir_transform/src/rad_protected_analysis.rs b/compiler/rustc_mir_transform/src/rad_protected_analysis.rs index ebe33b2b1cc23..5da48963c3651 100644 --- a/compiler/rustc_mir_transform/src/rad_protected_analysis.rs +++ b/compiler/rustc_mir_transform/src/rad_protected_analysis.rs @@ -1,12 +1,19 @@ //! This pass performs an analysis to determine reference, raw pointer, and unsafe function call accesses that are protected by `#[rad_protected]` use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; -use rustc_hir::find_attr; +use rustc_hir::{find_attr, Mutability}; use rustc_middle::mir::{ Body, Local, LocalKind, Operand, Place, RETURN_PLACE, Rvalue, StatementKind, TerminatorKind, + BasicBlock, SourceInfo, LocalDecl, Statement, CastKind, AggregateKind, ProjectionElem, + RawPtrKind, CoercionSource, BorrowKind, }; +use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::print::with_no_trimmed_paths; -use rustc_middle::ty::{self, TyCtxt}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use super::rad_protected_liveness_analysis::{CheckpointAnalysis, LiveLocals}; +use rustc_span::{sym, Span, source_map::Spanned}; +use rustc_index::IndexVec; +use rustc_middle::mir::interpret::Scalar; pub(super) struct RadProtectedAnalysis; @@ -24,6 +31,31 @@ impl<'tcx> crate::MirPass<'tcx> for RadProtectedAnalysis { return; } + let checkpoint_analysis = CheckpointAnalysis::analyze(tcx, body); + + eprintln!("=== Liveness analysis for {:?} ===", def_id); + for (bb_idx, live) in &checkpoint_analysis.checkpoints { + eprintln!("Checkpoint {:?}", bb_idx); + eprintln!("\tSync: {:?}\n", live.locals()); + } + eprintln!("================================"); + + eprintln!("=== Injecting checkpoints ==="); + + let mut max_payload_size: u64 = 0; + + for (bb_idx, live) in checkpoint_analysis.checkpoints { + let payload_size = inject_checkpoint_call(tcx, body, live, bb_idx); + max_payload_size = max_payload_size.max(payload_size); + eprintln!("Successfully injected checkpoint call at {:?}", bb_idx); + } + + inject_payload_size_arg(tcx, body, max_payload_size) + .expect("Failed to find a triplicate_process call to inject payload size"); + eprintln!("Checkpoint injection complete with payload size of {} bytes", max_payload_size); + + eprintln!("================================"); + let sources = build_pointer_sources(body); with_no_trimmed_paths!({ @@ -422,3 +454,159 @@ fn resolve_pointer_source<'tcx>( } } } + +fn inject_checkpoint_call<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, live: LiveLocals, next: BasicBlock) -> u64 { + let checkpoint_def_id = tcx.get_diagnostic_item(sym::checkpoint).unwrap(); + let source_info = SourceInfo::outermost(body.span); + let span = body.span; + let num_locals = live.locals().len() as u64; + + let mut payload_size: u64 = 0; + + let mut stmts: Vec> = Vec::new(); + let mut push_stmt = |kind: StatementKind<'tcx>| { + stmts.push(Statement::new(source_info, kind)); + }; + + let push_local = |body: &mut Body<'tcx>, ty| { + body.local_decls.push(LocalDecl::new(ty, span)) + }; + + let mut push_assign = |place, rvalue| { + push_stmt(StatementKind::Assign(Box::new((place, rvalue)))); + }; + + let u8_ptr_ty = Ty::new_ptr(tcx, tcx.types.u8, Mutability::Mut); + let slot_ty = Ty::new_tup(tcx, &[u8_ptr_ty, tcx.types.usize]); + + // let array: [(*mut u8, usize); usize]; + let array_ty = Ty::new_array(tcx, slot_ty, num_locals); + let array_local = push_local(body, array_ty); + + let typing_env = body.typing_env(tcx); + + for (i, &local) in live.locals().iter().enumerate() { + let local_ty = body.local_decls[local].ty; + + // _1 = &raw mut local; + let raw_ptr_ty = Ty::new_ptr(tcx, local_ty, Mutability::Mut); + let raw_ptr = push_local(body, raw_ptr_ty); + push_assign( + Place::from(raw_ptr), + Rvalue::RawPtr(RawPtrKind::Mut, Place::from(local)), + ); + + // _2 = _1 as *mut u8 (PtrToPtr); + let u8_ptr = push_local(body, u8_ptr_ty); + push_assign( + Place::from(u8_ptr), + Rvalue::Cast( + CastKind::PtrToPtr, + Operand::Move(Place::from(raw_ptr)), + u8_ptr_ty + ), + ); + + let local_size = tcx + .layout_of(typing_env.as_query_input(local_ty)) + .unwrap() + .size + .bytes(); + payload_size += local_size; + + let size_operand = Operand::const_from_scalar( + tcx, + tcx.types.usize, + Scalar::from_target_usize(local_size.try_into().unwrap(), &tcx), + span, + ); + + // _3 = (_2, local_size); + let slot_local = push_local(body, slot_ty); + push_assign( + Place::from(slot_local), + Rvalue::Aggregate( + Box::new(AggregateKind::Tuple), + IndexVec::from_raw(vec![Operand::Move(Place::from(u8_ptr)), size_operand]), + ), + ); + + // array[i] = move _3; + let elem_place = Place::from(array_local).project_deeper( + &[ProjectionElem::ConstantIndex { + offset: i as u64, + min_length: num_locals, + from_end: false + }], + tcx, + ); + push_assign( + elem_place, + Rvalue::Use(Operand::Move(Place::from(slot_local))), + ); + } + + // _1 = &array; + let array_ref_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, array_ty); + let array_ref = push_local(body, array_ref_ty); + push_assign( + Place::from(array_ref), + Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, Place::from(array_local)), + ); + + // _2 = move _1 as &[(*mut u8, usize)] (PointerCoercion(Unsize, Implicit)); + let slice_ty = Ty::new_slice(tcx, slot_ty); + let slice_ref_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, slice_ty); + let slice_ref = push_local(body, slice_ref_ty); + push_assign( + Place::from(slice_ref), + Rvalue::Cast( + CastKind::PointerCoercion( + PointerCoercion::Unsize, + CoercionSource::Implicit + ), + Operand::Move(Place::from(array_ref)), + slice_ref_ty, + ), + ); + + let func = Operand::function_handle(tcx, checkpoint_def_id, [], span); + let block_data = &mut body.basic_blocks_mut()[next]; + block_data.statements.extend(stmts); + + let (callee, args, term_fn_span) = call_terminator_parts_mut(&mut block_data.terminator_mut().kind).unwrap(); + *callee = func; + *args = Box::new([Spanned { node: Operand::Move(Place::from(slice_ref)), span }]); + *term_fn_span = span; + + payload_size +} + +fn inject_payload_size_arg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, max_payload_size: u64) -> Option<()> { + + for bb_data in body.basic_blocks.as_mut().iter_mut() { + if CheckpointAnalysis::is_call_to(tcx, bb_data, sym::triplicate_process) { + let span = bb_data.terminator().source_info.span; + let size_operand = Operand::const_from_scalar( + tcx, + tcx.types.usize, + Scalar::from_target_usize(max_payload_size, &tcx), + span, + ); + + let (_, args, _) = call_terminator_parts_mut(&mut bb_data.terminator_mut().kind).unwrap(); + *args = Box::new([Spanned { node: size_operand, span }]); + return Some(()); + } + } + + None +} + +fn call_terminator_parts_mut<'a, 'tcx>(kind: &'a mut TerminatorKind<'tcx>) -> Option<(&'a mut Operand<'tcx>, &'a mut Box<[Spanned>]>, &'a mut Span)> { + match kind { + TerminatorKind::Call { func, args, fn_span, .. } + | TerminatorKind::TailCall { func, args, fn_span, .. } => Some((func, args, fn_span)), + _ => None, + } +} diff --git a/compiler/rustc_mir_transform/src/rad_protected_liveness_analysis.rs b/compiler/rustc_mir_transform/src/rad_protected_liveness_analysis.rs new file mode 100644 index 0000000000000..9c07b304d551a --- /dev/null +++ b/compiler/rustc_mir_transform/src/rad_protected_liveness_analysis.rs @@ -0,0 +1,244 @@ +use rustc_middle::mir::{BasicBlock, Location, visit::{PlaceContext, Visitor}}; +use rustc_middle::mir::{ + Body, BasicBlocks, Local, Place, Rvalue, TerminatorKind, BasicBlockData +}; +use std::collections::VecDeque; +use rustc_data_structures::{fx::FxHashSet, graph::Successors}; +use rustc_index::{bit_set::DenseBitSet, IndexVec}; +use rustc_span::def_id::DefId; +use rustc_span::{sym, Symbol}; +use rustc_middle::ty::TyCtxt; + +pub(super) struct CheckpointAnalysis { + pub checkpoints: Vec<(BasicBlock, LiveLocals)>, +} + +impl CheckpointAnalysis { + pub(super) fn analyze<'tcx>(tcx: TyCtxt<'_>, body: &Body<'tcx>) -> Self { + Self { + checkpoints: + LivenessAnalysis::analyze(tcx, body) + .liveness + .into_iter_enumerated() + .filter(|(bb_idx, _)| Self::is_checkpoint(tcx, &body.basic_blocks[*bb_idx])) + .map(|(bb_idx, liveness)| (bb_idx, LiveLocals::new(liveness.out))) + .collect() + } + } + + fn postprocess_gen_kill<'tcx>(tcx: TyCtxt<'_>, body: &Body<'tcx>, bb_data: &BasicBlockData<'_>, gen_kill: &mut GenKill) { + if Self::is_checkpoint(tcx, &bb_data) { + // Order independent since target is unordered (HashSet) + #[allow(rustc::potential_query_instability)] + gen_kill.kill.extend(body.local_decls.indices()); + } + } + + pub(super) fn is_checkpoint(tcx: TyCtxt<'_>, bb_data: &BasicBlockData<'_>) -> bool { + Self::is_call_to(tcx, bb_data, sym::__checkpoint) + } + + pub(super) fn is_call_to(tcx: TyCtxt<'_>, bb_data: &BasicBlockData<'_>, symbol: Symbol) -> bool { + Self::callee_def_id(bb_data).is_some_and(|def_id| tcx.is_diagnostic_item(symbol, def_id)) + } + + fn callee_def_id<'tcx>(bb_data: &BasicBlockData<'tcx>) -> Option { + let func = match &bb_data.terminator().kind { + TerminatorKind::Call { func, .. } + | TerminatorKind::TailCall { func, .. } => func, + _ => return None, + }; + func.const_fn_def().map(|(def_id, _)| def_id) + } +} + +pub(super) struct LiveLocals { + locals: Vec, +} + +impl LiveLocals { + fn new(locals: FxHashSet) -> Self { + Self { + locals: Self::sort_locals(locals) + } + } + + fn sort_locals(set: FxHashSet) -> Vec { + // Values are sorted after being collected into a Vec + #[allow(rustc::potential_query_instability)] + let mut locals: Vec = set.into_iter().collect(); + + locals.sort(); + locals + } + + pub(super) fn locals(&self) -> &Vec { + &self.locals + } +} + +struct LivenessAnalysis { + liveness: IndexVec, +} + +// Liveness Analysis Pass (based on: https://en.wikipedia.org/wiki/Live-variable_analysis) +impl LivenessAnalysis { + fn analyze<'tcx>(tcx: TyCtxt<'_>, body: &Body<'tcx>) -> Self { + let gk_analysis = Self::generate_gk_analysis(tcx, &body); + Self::calculate_liveness(gk_analysis, &body.basic_blocks) + } + + fn calculate_liveness<'tcx>(gk_analysis: GenKillAnalysis, basic_blocks: &BasicBlocks<'tcx>) -> Self { + let mut liveness = IndexVec::from_fn_n( + |_| Liveness::new(), + basic_blocks.len() + ); + + let mut work_queue: VecDeque = basic_blocks.indices().rev().collect(); + let mut in_queue = DenseBitSet::new_filled(basic_blocks.len()); + + while let Some(bb_idx) = work_queue.pop_front() { + in_queue.remove(bb_idx); + + let old_in = liveness[bb_idx]._in.clone(); + liveness[bb_idx].out.clear(); + + for s in basic_blocks.successors(bb_idx) { + let successor_in = liveness[s]._in.clone(); + // Order independent since target is unordered (HashSet) + #[allow(rustc::potential_query_instability)] + liveness[bb_idx].out.extend(successor_in); + } + + let mut live_in = gk_analysis.gen_kill[bb_idx]._gen.clone(); + // Order independent since target is unordered (HashSet) + #[allow(rustc::potential_query_instability)] + live_in.extend(liveness[bb_idx].out.difference(&gk_analysis.gen_kill[bb_idx].kill)); + liveness[bb_idx]._in = live_in; + + if liveness[bb_idx]._in != old_in { + for &p in basic_blocks.predecessors()[bb_idx].iter() { + if in_queue.insert(p) { + work_queue.push_back(p); + } + } + } + } + + Self { liveness } + } + + fn generate_gk_analysis<'tcx>(tcx: TyCtxt<'_>, body: &Body<'tcx>) -> GenKillAnalysis { + let mut gen_kill = IndexVec::::from_fn_n( + |_| GenKill::new(), + body.basic_blocks.len() + ); + + for (bb_idx, bb_data) in body.basic_blocks.iter_enumerated() { + let mut collector = GenKillCollector::new(); + + for (stmt_idx, stmt) in bb_data.statements.iter().enumerate() { + let location = Location { + block: bb_idx, + statement_index: stmt_idx, + }; + + collector.visit_statement(stmt, location); + } + + let terminator_location = Location { + block: bb_idx, + statement_index: bb_data.statements.len(), + }; + + collector.visit_terminator(bb_data.terminator(), terminator_location); + + gen_kill[bb_idx] = collector.take(); + + // Special GenKill postprocessing step to calculate liveness for checkpoints + // Not found in typical liveness analysis + CheckpointAnalysis::postprocess_gen_kill(tcx, body, &bb_data, &mut gen_kill[bb_idx]); + } + + GenKillAnalysis { gen_kill } + } +} + +pub(super) struct Liveness { + _in: FxHashSet, + out: FxHashSet, +} + +impl Liveness { + fn new() -> Self { + Self { + _in: FxHashSet::default(), + out: FxHashSet::default(), + } + } +} + +struct GenKillAnalysis { + gen_kill: IndexVec, +} + + +struct GenKillCollector { + gen_kill: GenKill, +} + +impl GenKillCollector { + fn new() -> Self { + Self { gen_kill: GenKill::new() } + } + + fn take(self) -> GenKill { + self.gen_kill + } +} + +impl<'tcx> Visitor<'tcx> for GenKillCollector { + fn visit_local(&mut self, local: Local, context: PlaceContext, _location: Location) { + + if context.is_place_assignment() { + self.gen_kill.push_kill(local); + } + else if context.is_mutating_use() { + self.gen_kill.push_gen(local); + // The push_kill is not strictly necessary here, but it follows the convention + // of kill marking all writes (not just writes before a use) + self.gen_kill.push_kill(local); + + } + else if context.is_use() { + self.gen_kill.push_gen(local); + } + } + + fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + self.visit_rvalue(rvalue, location); + self.gen_kill.push_kill(place.local); + } +} + +struct GenKill { + _gen: FxHashSet, + kill: FxHashSet, +} + +impl GenKill { + fn push_kill(&mut self, local: Local) { + self.kill.insert(local); + } + fn push_gen(&mut self, local: Local) { + if !self.kill.contains(&local) { + self._gen.insert(local); + } + } + fn new() -> Self { + Self { + _gen: FxHashSet::default(), + kill: FxHashSet::default(), + } + } +} diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 28b3bb2824456..d50595092cc45 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -283,6 +283,7 @@ symbols! { Pointer, Poll, ProcMacro, + RadRustRuntime, Range, RangeCopy, RangeFrom, @@ -347,6 +348,8 @@ symbols! { Vec, Wrapper, _DECLS, + __checkpoint, + __guard, __H, __S, __awaitee, @@ -611,6 +614,7 @@ symbols! { cfi, cfi_encoding, char, + checkpoint, client, clippy, clobber_abi, @@ -878,6 +882,7 @@ symbols! { emscripten_wasm_eh, enable, end, + enter_critical_section, entry_nops, env, env_CFG_RELEASE: env!("CFG_RELEASE"), @@ -891,6 +896,7 @@ symbols! { exhaustive_integer_patterns, exhaustive_patterns, existential_type, + exit_critical_section, exp2f16, exp2f32, exp2f64, @@ -1591,6 +1597,7 @@ symbols! { question_mark, quote, rad_protected, + rad_protected_mir, range_inclusive_new, raw_dash_dylib: "raw-dylib", raw_dylib, @@ -2040,6 +2047,8 @@ symbols! { transparent, transparent_enums, transparent_unions, + triplicate_unsafe, + triplicate_process, trivial_bounds, trivial_clone, truncf16, diff --git a/library/Cargo.lock b/library/Cargo.lock index 4801f92c63e5a..87c47bebcaa24 100644 --- a/library/Cargo.lock +++ b/library/Cargo.lock @@ -248,6 +248,16 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "rad_protected" +version = "0.0.0" +dependencies = [ + "alloc", + "compiler_builtins", + "core", + "libc", +] + [[package]] name = "rand" version = "0.9.2" @@ -338,6 +348,7 @@ dependencies = [ "panic_unwind", "r-efi", "r-efi-alloc", + "rad_protected", "rand", "rand_xorshift", "rustc-demangle", diff --git a/library/core/src/macros/mod.rs b/library/core/src/macros/mod.rs index c2d714d2b7877..bbbf8d842131e 100644 --- a/library/core/src/macros/mod.rs +++ b/library/core/src/macros/mod.rs @@ -1789,6 +1789,14 @@ pub(crate) mod builtin { /* compiler built-in */ } + + /// Attribute macro that implements radiation hardening through triplication + #[stable(feature = "rad_protected", since = "1.95.0")] + #[rustc_builtin_macro] + pub macro rad_protected($item:item) { + /* compiler built-in */ + } + /// Attribute macro applied to a function to give it a post-condition. /// /// The attribute carries an argument token-tree which is diff --git a/library/core/src/prelude/v1.rs b/library/core/src/prelude/v1.rs index f2eb047d342bc..1f8eda726b7fb 100644 --- a/library/core/src/prelude/v1.rs +++ b/library/core/src/prelude/v1.rs @@ -166,3 +166,6 @@ pub use crate::macros::builtin::{eii, unsafe_eii}; #[unstable(feature = "eii_internals", issue = "none")] pub use crate::macros::builtin::eii_declaration; + +#[stable(feature = "rad_protected_builtin_macro", since = "1.95.0")] +pub use crate::macros::builtin::rad_protected; diff --git a/library/rad_protected/Cargo.toml b/library/rad_protected/Cargo.toml new file mode 100644 index 0000000000000..9a3956e5e7baf --- /dev/null +++ b/library/rad_protected/Cargo.toml @@ -0,0 +1,24 @@ +cargo-features = ["public-dependency"] + +[package] +name = "rad_protected" +version = "0.0.0" +edition = "2024" +repository = "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/rad-rust/rust.git" +description = "Runtime support for Rad-Rust Software TMR" + +[lib] +crate-type = ["rlib"] + +[dependencies] +core = { path = "../core", public = true } +alloc = { path = "../alloc", public = true } +compiler_builtins = { path = "../compiler-builtins/compiler-builtins", features = ["rustc-dep-of-std"] } + +[target.'cfg(not(all(windows, target_env = "msvc")))'.dependencies] +libc = { version = "0.2.178", default-features = false, features = [ + 'rustc-dep-of-std', +], public = true } + +[lints.rust.unexpected_cfgs] +level = "warn" diff --git a/library/rad_protected/src/fork.rs b/library/rad_protected/src/fork.rs new file mode 100644 index 0000000000000..c03b924d8ad22 --- /dev/null +++ b/library/rad_protected/src/fork.rs @@ -0,0 +1,73 @@ +use super::mini_std::{fs::File, io::BufReader}; +use core::ptr; +use super::libc_helpers::{fork, sysconf_sc_pagesize}; +use super::role::ChildLink; + +pub(super) fn fork_copy() -> Option { + + match unsafe { fork() }.ok()? { + 0 => { + force_copy_pages(); + Some(ForkOutcome::Child) + }, + child_pid => { + Some(ForkOutcome::Parent(ChildLink::new(child_pid))) + }, + } +} + +pub(super) enum ForkOutcome { + Parent(ChildLink), + Child, +} + +fn force_copy_pages() { + let page_size = sysconf_sc_pagesize().unwrap(); + + let file = File::open("/proc/self/maps").unwrap(); + let reader = BufReader::new(file); + + for line in reader.lines().map_while(Result::ok) { + let mut parts = line.split_whitespace(); + + let (Some(range), Some(perms)) = (parts.next(), parts.next()) else { + continue; + }; + + if !matches!(perms.as_bytes(), [_, b'w', _, b'p', ..]) { + continue; + } + + if line.contains("[vsyscall]") + || line.contains("[vvar]") + || line.contains("[vdso]") + { + continue; + } + + let Some((start, end)) = range.split_once('-') else { + continue; + }; + + let (Ok(start), Ok(end)) = ( + usize::from_str_radix(start, 16), + usize::from_str_radix(end, 16), + ) else { + continue; + }; + + unsafe { + let mut addr = start; + + while addr < end { + let p = ptr::without_provenance_mut::(addr); + + // Force a write so the kernel faults in a private copy of the COW page. + let v = ptr::read_volatile(p); + ptr::write_volatile(p, v); + + addr += page_size; + } + } + } +} diff --git a/library/rad_protected/src/lib.rs b/library/rad_protected/src/lib.rs new file mode 100644 index 0000000000000..23c90f2cfff65 --- /dev/null +++ b/library/rad_protected/src/lib.rs @@ -0,0 +1,23 @@ +//! The Rad-Rust `rad_protected` Runtime +//! +//! This library contains a static runtime for running software radiation-hardened programs through TMR. +//! +//! The runtime is intended to be used with a rustc fork to generate runtime calls, +//! and thus only meant to be referenced directly by the compiler itself. +//! +//! For porting this library to a new target, see mini_std/mod.rs + +#![feature(rustc_attrs)] +#![feature(staged_api)] +#![allow(internal_features)] +#![no_std] + +/// `rad_protected` runtime module +#[stable(feature = "rad_protected", since = "1.95.0")] +pub mod runtime; + +mod fork; +mod libc_helpers; +mod role; +mod mini_std; +mod shared_memory; diff --git a/library/rad_protected/src/libc_helpers.rs b/library/rad_protected/src/libc_helpers.rs new file mode 100644 index 0000000000000..feb09ba249670 --- /dev/null +++ b/library/rad_protected/src/libc_helpers.rs @@ -0,0 +1,103 @@ +use super::mini_std::io; +use core::{ptr, sync::atomic::AtomicU32}; +use libc; + +pub(super) type Pid = libc::pid_t; + +pub(super) unsafe fn fork() -> io::Result { + let pid = unsafe { libc::fork() }; + + if pid == -1 { + return Err(io::Error::last_os_error()); + } + Ok(pid) +} + +pub(super) fn kill(pid: libc::pid_t) -> io::Result<()> { + if unsafe { libc::kill(pid, libc::SIGKILL) } == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +pub(super) fn waitpid(pid: libc::pid_t) -> io::Result { + let pid = unsafe { libc::waitpid(pid, ptr::null_mut(), 0) }; + + if pid == -1 { + return Err(io::Error::last_os_error()); + } + Ok(pid) +} + +pub(super) fn sysconf_sc_pagesize() -> io::Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + + if page_size == -1 { + return Err(io::Error::last_os_error()); + } + Ok(page_size as usize) +} + +pub(super) fn shared_mmap(size: usize) -> io::Result<*mut libc::c_void> { + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED | libc::MAP_ANON, + -1, + 0, + ) + }; + + if ptr == libc::MAP_FAILED { + return Err(io::Error::last_os_error()); + } + Ok(ptr) +} + +pub(super) fn munmap(ptr: *mut libc::c_void, size: usize) { + unsafe { libc::munmap(ptr, size); } +} + +pub(super) fn futex_wait(addr: &AtomicU32, expected: u32) -> io::Result<()> { + let addr = addr as *const _ as *const u32; + + let res = unsafe { + libc::syscall( + libc::SYS_futex, + addr, + libc::FUTEX_WAIT, + expected, + ptr::null::(), + ptr::null::(), + 0 + ) + }; + + if res == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +pub(super) fn futex_wake_all(addr: &AtomicU32) -> io::Result { + let addr = addr as *const _ as *const u32; + + let res = unsafe { + libc::syscall( + libc::SYS_futex, + addr, + libc::FUTEX_WAKE, + libc::INT_MAX, + ptr::null::(), + ptr::null::(), + 0 + ) + } as i32; + + if res == -1 { + return Err(io::Error::last_os_error()); + } + Ok(res) +} diff --git a/library/rad_protected/src/mini_std/fs.rs b/library/rad_protected/src/mini_std/fs.rs new file mode 100644 index 0000000000000..c082e964f061f --- /dev/null +++ b/library/rad_protected/src/mini_std/fs.rs @@ -0,0 +1,36 @@ +use alloc::ffi::CString; +use super::io; + +pub struct File { + fd: i32, +} + +impl File { + pub fn open(path: &str) -> io::Result { + let path = CString::new(path).unwrap(); + let fd = unsafe { libc::open(path.as_ptr(), libc::O_RDONLY) }; + + if fd == -1 { + return Err(io::Error::last_os_error()); + } + Ok(Self { fd }) + } + + pub fn read(&mut self, buf: &mut [u8]) -> io::Result { + let len = buf.len(); + let buf = buf.as_mut_ptr() as *mut libc::c_void; + + let bytes_read = unsafe { libc::read(self.fd, buf, len) }; + + if bytes_read == -1 { + return Err(io::Error::last_os_error()); + } + Ok(bytes_read) + } +} + +impl Drop for File { + fn drop(&mut self) { + unsafe { libc::close(self.fd); } + } +} diff --git a/library/rad_protected/src/mini_std/io.rs b/library/rad_protected/src/mini_std/io.rs new file mode 100644 index 0000000000000..1eca92002ed64 --- /dev/null +++ b/library/rad_protected/src/mini_std/io.rs @@ -0,0 +1,92 @@ +use super::fs; +use alloc::string::String; +use alloc::vec::Vec; +use alloc::str::Utf8Error; + +pub type Result = core::result::Result; + +#[derive(Debug)] +pub enum Error { + Os(i32), + Utf8(Utf8Error), +} + +impl Error { + #[cfg(target_os = "linux")] + pub fn last_os_error() -> Self { + Self::Os(unsafe { *libc::__errno_location() }) + } + + #[allow(unused)] + pub fn raw_os_error(&self) -> Option { + match self { + Error::Os(errno) => Some(*errno), + _ => None + } + } + + #[allow(unused)] + pub fn raw_utf8_error(&self) -> Option { + match self { + Error::Utf8(err) => Some(*err), + _ => None + } + } +} + +pub struct BufReader { + file: fs::File, +} + +impl BufReader { + pub fn new(file: fs::File) -> Self { + Self { file } + } + + fn read_line(&mut self) -> Option> { + let mut bytes = Vec::new(); + + loop { + let mut buf = [0u8; 1]; + + match self.file.read(&mut buf) { + Ok(0) => { + if bytes.is_empty() { + return None; + } + break; + } + Ok(_) => { + if buf[0] == b'\n' { + break; + } + bytes.push(buf[0]); + } + Err(err) => { + return Some(Err(err)); + } + } + } + + match String::from_utf8(bytes) { + Ok(str) => Some(Ok(str)), + Err(err) => Some(Err(Error::Utf8(err.utf8_error()))) + } + } + + pub fn lines(self) -> Lines { + Lines { reader: self } + } +} + +pub struct Lines { + reader: BufReader, +} + +impl Iterator for Lines { + type Item = Result; + + fn next(&mut self) -> Option { + self.reader.read_line() + } +} diff --git a/library/rad_protected/src/mini_std/ipc.rs b/library/rad_protected/src/mini_std/ipc.rs new file mode 100644 index 0000000000000..cdd4637e16061 --- /dev/null +++ b/library/rad_protected/src/mini_std/ipc.rs @@ -0,0 +1,59 @@ +use core::sync::atomic::{AtomicU32, Ordering}; +use super::super::libc_helpers::{futex_wait, futex_wake_all}; + +pub struct Barrier { + threshold: u32, + count: AtomicU32, + _gen: AtomicU32, +} + +impl Barrier { + pub fn new(threshold: u32) -> Self { + Self { + threshold, + count: AtomicU32::new(0), + _gen: AtomicU32::new(0) + } + } + + pub fn wait(&self) -> BarrierWaitResult { + let _gen = self._gen.load(Ordering::Acquire); + let arrived = self.count.fetch_add(1, Ordering::AcqRel) + 1; + + if arrived == self.threshold { + self.count.store(0, Ordering::Relaxed); + self._gen.fetch_add(1, Ordering::Release); + + let _ = futex_wake_all(&self._gen); + + return BarrierWaitResult(true); + } + + loop { + if self._gen.load(Ordering::Acquire) != _gen { + break; + } + + let res = futex_wait(&self._gen, _gen); + + if let Err(err) = res { + let err = err.raw_os_error().unwrap(); + + match err { + libc::EAGAIN | libc::EINTR => continue, + _ => panic!("Failed to acquire futex"), + } + } + } + + BarrierWaitResult(false) + } +} + +pub struct BarrierWaitResult(bool); + +impl BarrierWaitResult { + pub fn is_leader(&self) -> bool { + self.0 + } +} diff --git a/library/rad_protected/src/mini_std/mod.rs b/library/rad_protected/src/mini_std/mod.rs new file mode 100644 index 0000000000000..82937204db5aa --- /dev/null +++ b/library/rad_protected/src/mini_std/mod.rs @@ -0,0 +1,20 @@ +//! A near-faithful recreation and extension of the minimal subset of `std` required by `rad_protected` +//! for `no_std` environments. +//! +//! This module provides a `std`-like API to isolate platform-specific functionality. +//! +//! The current implementation targets Linux, but it is intended to be reimplemented for +//! other platforms as needed. +//! +//! Because this module mirrors the `std` API, it can be removed entirely on platforms +//! where the standard library is available, other than the IPC module. +//! +//! Since some `rad_protected` functions are guaranteed to invoke certain `mini_std` +//! functions from a single thread, synchronization is not uniformly implemented throughout this module. +//! +//! Be careful when using these APIs, as they may not be thread-safe outside of their intended usage. + +pub mod io; +pub mod sync; +pub mod fs; +pub mod ipc; diff --git a/library/rad_protected/src/mini_std/sync.rs b/library/rad_protected/src/mini_std/sync.rs new file mode 100644 index 0000000000000..a7f44dc552a26 --- /dev/null +++ b/library/rad_protected/src/mini_std/sync.rs @@ -0,0 +1,62 @@ +use libc; +use core::{cell::UnsafeCell, ops::{Deref, DerefMut}}; + +pub struct Mutex { + lock: libc::pthread_mutex_t, + data: UnsafeCell, +} + +pub struct MutexGuard<'a, T> { + mutex: &'a Mutex, +} + +unsafe impl Send for Mutex {} +unsafe impl Sync for Mutex {} + +type LockResult = Result; + +impl Mutex { + pub const fn new(data: T) -> Self { + Self { + lock: libc::PTHREAD_MUTEX_INITIALIZER, + data: UnsafeCell::new(data), + } + } + + pub fn lock(&self) -> LockResult> { + let res = unsafe { libc::pthread_mutex_lock(&self.lock as *const _ as *mut _) }; + + if res != 0 { + return Err(res); + } + Ok(MutexGuard { mutex: self }) + } +} + +impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.mutex.data.get() } + } +} + +impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mutex.data.get() } + } +} + +impl Drop for MutexGuard<'_, T> { + fn drop(&mut self) { + unsafe { libc::pthread_mutex_unlock(&self.mutex.lock as *const _ as *mut _); } + } +} + +impl Drop for Mutex { + fn drop(&mut self) { + unsafe { + libc::pthread_mutex_destroy(&mut self.lock); + } + } +} diff --git a/library/rad_protected/src/role.rs b/library/rad_protected/src/role.rs new file mode 100644 index 0000000000000..e5c5c7caeffd9 --- /dev/null +++ b/library/rad_protected/src/role.rs @@ -0,0 +1,121 @@ +use super::mini_std::sync::Mutex; +use super::libc_helpers::{kill, waitpid, Pid}; +use super::shared_memory::SharedMemory; + +pub(super) static ROLE: Mutex> = Mutex::new(None); + +#[derive(Debug)] +pub(super) enum Role { + Parent(Parent), + Child(Child), +} + +#[derive(Debug)] +pub struct Parent { + shared_mem_ctx: SharedMemoryContext, + child1: ChildLink, + child2: ChildLink, +} + +impl Parent { + pub(super) fn new(shared_memory: SharedMemory, child1: ChildLink, child2: ChildLink) -> Self { + Self { + shared_mem_ctx: SharedMemoryContext::new(shared_memory), + child1, + child2 + } + } + + pub(super) fn kill_children(&self) { + self.child1.kill_child(); + self.child2.kill_child(); + } + + pub(super) fn close_shared_mem(&self) { + self.shared_mem_ctx.shared_memory.close(); + } +} + +#[derive(Debug)] +pub struct Child { + shared_mem_ctx: SharedMemoryContext, +} + +impl Child { + pub(super) fn new(shared_memory: SharedMemory) -> Self { + Self { + shared_mem_ctx: SharedMemoryContext::new(shared_memory), + } + } +} + +#[derive(Debug)] +pub(super) struct SharedMemoryContext { + shared_memory: SharedMemory, + leader_depth: u32, +} + +impl SharedMemoryContext { + pub(super) fn new(shared_memory: SharedMemory) -> Self { + Self { shared_memory, leader_depth: 0 } + } + + pub(super) fn sync(&self) -> bool { + self.shared_memory.sync() + } + + pub(super) fn enter_critical_section(&mut self) -> bool { + let leader = self.is_leader() || self.shared_memory.sync(); + + if leader { + self.leader_depth += 1; + } + + leader + } + + pub(super) fn exit_critical_section(&mut self) { + if self.is_leader() { + self.leader_depth -= 1; + } + + if !self.is_leader() { + self.shared_memory.sync(); + } + } + + fn is_leader(&self) -> bool { + self.leader_depth > 0 + } +} + +impl Role { + pub(super) fn ctx(&self) -> &SharedMemoryContext { + match self { + Role::Parent(parent) => &parent.shared_mem_ctx, + Role::Child(child) => &child.shared_mem_ctx, + } + } + pub(super) fn ctx_mut(&mut self) -> &mut SharedMemoryContext { + match self { + Role::Parent(parent) => &mut parent.shared_mem_ctx, + Role::Child(child) => &mut child.shared_mem_ctx, + } + } +} + +#[derive(Debug)] +pub(super) struct ChildLink { + pid: Pid, +} + +impl ChildLink { + pub(super) fn new(pid: Pid) -> Self { + Self { pid } + } + + pub(super) fn kill_child(&self) { + let _ = kill(self.pid); + let _ = waitpid(self.pid); + } +} diff --git a/library/rad_protected/src/runtime.rs b/library/rad_protected/src/runtime.rs new file mode 100644 index 0000000000000..4a32a96a35bde --- /dev/null +++ b/library/rad_protected/src/runtime.rs @@ -0,0 +1,142 @@ +use super::fork::{fork_copy, ForkOutcome}; +use super::role::{ROLE, Role, Parent, Child}; +use super::shared_memory::SharedMemory; + +/// Runtime for rad_protected +#[stable(feature = "rad_protected", since = "1.95.0")] +#[derive(Debug)] +pub struct Runtime; + +impl Runtime { + + /// Triplicate the running process over the current rad_protected function + /// Fork the running process and copy its memory to create 3 identical processes + #[stable(feature = "rad_protected", since = "1.95.0")] + #[rustc_diagnostic_item = "triplicate_process"] + pub fn triplicate_process(payload_size: usize) -> Result { + if ROLE.lock().unwrap().as_ref().is_some() { + return Err(()); + } + + let Ok(shared_memory) = SharedMemory::open(payload_size) else { + return Err(()); + }; + + let link1 = match fork_copy() { + Some(ForkOutcome::Parent(link1)) => link1, + Some(ForkOutcome::Child) => { + ROLE.lock().unwrap().replace(Role::Child(Child::new(shared_memory))); + return Ok(ProcessGuard{}); + } + None => { + shared_memory.close(); + return Err(()); + } + }; + + let link2 = match fork_copy() { + Some(ForkOutcome::Parent(link2)) => link2, + Some(ForkOutcome::Child) => { + ROLE.lock().unwrap().replace(Role::Child(Child::new(shared_memory))); + return Ok(ProcessGuard{}); + } + None => { + shared_memory.close(); + link1.kill_child(); + return Err(()); + } + }; + + ROLE.lock().unwrap().replace(Role::Parent(Parent::new( + shared_memory, + link1, + link2, + ))); + + Ok(ProcessGuard{}) + } + + /// Enter a critical (unsafe) section of code, allowing only a single process through + /// Syncs the three processes. Returns `true` for the one leader (parent) process + #[stable(feature = "rad_protected", since = "1.95.0")] + pub fn enter_critical_section() -> bool { + let mut guard = ROLE.lock().unwrap(); + if let Some(role) = guard.as_mut() { + return role.ctx_mut().enter_critical_section(); + } + true + } + + /// Exit a critical (unsafe) section of code + /// Syncs the three processes + #[stable(feature = "rad_protected", since = "1.95.0")] + pub fn exit_critical_section() { + let mut guard = ROLE.lock().unwrap(); + if let Some(role) = guard.as_mut() { + return role.ctx_mut().exit_critical_section(); + } + } + + /// Close and clean up the triplicated processes at the end of rad_protected execution + #[stable(feature = "rad_protected", since = "1.95.0")] + pub fn close() { + let mut guard = ROLE.lock().unwrap(); + if let Some(role) = guard.as_ref() { + role.ctx().sync(); + if let Role::Parent(parent) = role { + parent.kill_children(); + parent.close_shared_mem(); + } else { + unsafe { libc::pause(); } + } + } + guard.take(); + } + + // Checkpoint given locals via a majority vote over the triplicated threads + #[stable(feature = "rad_protected", since = "1.95.0")] + #[rustc_diagnostic_item = "checkpoint"] + pub fn checkpoint(locals: &[(*mut u8, usize)]) { + if locals.is_empty() { + return; + } + + let hex_fmt = b"%02x \0".as_ptr() as *const libc::c_char; + let nl_fmt = b"\n\0".as_ptr() as *const libc::c_char; + + for i in 0..locals.len() { + let (buf_ptr, buf_len) = locals[i]; + + if buf_ptr.is_null() || buf_len == 0 { + continue; + } + + for j in 0..buf_len { + let byte = unsafe { *buf_ptr.add(j) }; + unsafe { libc::printf(hex_fmt, byte as libc::c_int); } + } + unsafe { libc::printf(nl_fmt); } + } + } + + + // Internal checkpoint marker inserted during MIR building + // Indicates the MIR pass should rewrite the terminator to a `checkpoint` call + #[stable(feature = "rad_protected", since = "1.95.0")] + #[rustc_diagnostic_item = "__checkpoint"] + pub fn __checkpoint() { + } +} + +/// Guard to properly drop processes when done with the rad_protected function +#[stable(feature = "rad_protected", since = "1.95.0")] +#[derive(Debug)] +pub struct ProcessGuard; + +/// Drop method for `ProcessGuard`, close the child processes +#[stable(feature = "rad_protected", since = "1.95.0")] +impl Drop for ProcessGuard { + fn drop(&mut self) { + Runtime::close(); + } +} diff --git a/library/rad_protected/src/shared_memory.rs b/library/rad_protected/src/shared_memory.rs new file mode 100644 index 0000000000000..240f6080d7035 --- /dev/null +++ b/library/rad_protected/src/shared_memory.rs @@ -0,0 +1,64 @@ +use super::libc_helpers::{shared_mmap, munmap}; +use super::mini_std::{io, ipc::Barrier}; +use core::{mem::size_of, ptr, ops::{Deref, DerefMut}}; + +pub(super) struct SharedMemoryHeader { + barrier: Barrier, +} + +impl SharedMemoryHeader { + pub(super) fn sync(&self) -> bool { + self.barrier.wait().is_leader() + } +} + +#[derive(Debug)] +pub(super) struct SharedMemory { + header: *mut SharedMemoryHeader, + _payload: *mut u8, + size: usize, +} + +impl SharedMemory { + pub(super) fn open(payload_size: usize) -> io::Result { + let header_size = size_of::(); + let size = header_size + (payload_size * 3); + + let ptr = shared_mmap(size)? as *mut u8; + + let header = ptr.cast::(); + let payload = unsafe { ptr.add(header_size) }; + + unsafe { + ptr::write(header, + SharedMemoryHeader { + barrier: Barrier::new(3), + }, + ); + } + + Ok(Self { header, _payload: payload, size }) + } + + pub(super) fn close(&self) { + unsafe { ptr::drop_in_place(self.header); } + munmap(self.header as *mut _, self.size); + } +} + +unsafe impl Send for SharedMemory {} +unsafe impl Sync for SharedMemory {} + +impl Deref for SharedMemory { + type Target = SharedMemoryHeader; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.header } + } +} + +impl DerefMut for SharedMemory { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.header } + } +} diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index 1b7a41d697367..8296ef9e78459 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -14,6 +14,7 @@ crate-type = ["dylib", "rlib"] [dependencies] alloc = { path = "../alloc", public = true } +rad_protected = { path = "../rad_protected", public = true } # std no longer uses cfg-if directly, but the included copy of backtrace does. cfg-if = { version = "1.0", features = ['rustc-dep-of-std'] } panic_unwind = { path = "../panic_unwind", optional = true } diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index b3425e4969ac0..a5a2d903fadcb 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -611,6 +611,10 @@ pub mod random; pub mod sync; pub mod time; +/// Re-export the runtime for easier usage +#[stable(feature = "rad_protected", since = "1.95.0")] +pub use rad_protected::runtime::Runtime as RadRustRuntime; + // Pull in `std_float` crate into std. The contents of // `std_float` are in a different repository: rust-lang/portable-simd. #[path = "../../portable-simd/crates/std_float/src/lib.rs"]