[MSAN when using rustc compiled with llvm at HEAD] Unexpected memory sanitizer error in core::hint::black_box

[krasimirgg]

We're building the rust toolchain with LLVM at HEAD (https://buildkite.com/llvm-project/rust-llvm-integrate-prototype/builds?branch=master). I'm looking into using sanitizers there. I noticed that building rust + llvm @ head produced a toolchain that is generally useable, but when using it to compile a sample binary with msan, running the binary fails with an msan error at core::hint::black_box (dummy=()):

#0  __msan_warning_noreturn () at /example/rust/src/llvm-project/compiler-rt/lib/msan/msan.cpp:394
#1  0x00005642583215c4 in core::hint::black_box (dummy=()) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/core/src/hint.rs:226
#2  0x0000564258310e4b in std::sys_common::backtrace::__rust_begin_short_backtrace (f=0x5642583106b0 <foo::main>) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:124
#3  0x00005642583109a9 in std::rt::lang_start::{{closure}} () at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/rt.rs:166
#4  0x00005642584225a5 in core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once (self=..., args=()) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/core/src/ops/function.rs:286
#5  0x00005642585799da in std::panicking::try::do_call (data=0x7ffcb0931f80 "\310#\223\260\374\177") at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/panicking.rs:483
#6  0x00005642585844aa in __rust_try ()
#7  0x00005642585790b7 in std::panicking::try (f=...) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/panicking.rs:447
#8  0x0000564258419f9e in std::panic::catch_unwind (f=...) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/panic.rs:137
#9  0x00005642584d5b5f in std::rt::lang_start_internal::{{closure}} () at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/rt.rs:148
#10 0x0000564258579d0f in std::panicking::try::do_call (data=0x7ffcb09321b0 "\310#\223\260\374\177") at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/panicking.rs:483
#11 0x00005642585844aa in __rust_try ()
#12 0x0000564258578957 in std::panicking::try (f=...) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/panicking.rs:447
#13 0x000056425841a17e in std::panic::catch_unwind (f=...) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/panic.rs:137
#14 0x00005642584d55af in std::rt::lang_start_internal (main=..., argc=1, argv=0x7ffcb0932528, sigpipe=2) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/rt.rs:148
#15 0x00005642583108c6 in std::rt::lang_start (main=0x5642583106b0 <foo::main>, argc=1, argv=0x7ffcb0932528, sigpipe=2) at /example/rust/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/library/std/src/rt.rs:165
#16 0x000056425831077a in main ()
#17 0x00007f77d10ba083 in __libc_start_main (main=0x564258310720 <main>, argc=1, argv=0x7ffcb0932528, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffcb0932518) at ../csu/libc-start.c:308
#18 0x00005642582a721e in _start ()

Since there are quite a few steps to reproduce this (we need to build rustc with llvm at head, then we need to make cargo pick this up etc), I set up https://github.com/krasimirgg/example-rust-llvm-head-16-msan-error/tree/main with a docker reproducer.

I checked that using the initial llvm-init-16 revision the problem doesn't occur. I'm currently bisecting over llvm revisions.

[saethlin]

I have seen similar errors when running criterion's tests under ASan: https://asan.saethlin.dev/logs/criterion/0.4.0.html

running 1 test
test test_measurement_time ... 
--- STDERR:              criterion::criterion_tests test_measurement_time ---
Benchmarking test_meas_time_1
Benchmarking test_meas_time_1: Warming up for 250.00 ms
AddressSanitizer:DEADLYSIGNAL
=================================================================
==16137==ERROR: AddressSanitizer: SEGV on unknown address 0x10009b671c3f (pc 0x5598a8f92815 bp 0x7fffdb3ce230 sp 0x7fffdb3ce130 T0)
==16137==The signal is caused by a READ memory access.
    #0 0x5598a8f92815 in core::ptr::read_volatile::h1e9c02ea583b8e0d (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x61c815) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #1 0x5598a90fc4dd in criterion::black_box::hae404db5835d6cdd (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x7864dd) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #2 0x5598a90d7555 in criterion::bencher::Bencher$LT$M$GT$::iter::h93fee6ddc6cc6ce0 (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x761555) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #3 0x5598a8f4740f in criterion_tests::test_measurement_time::_$u7b$$u7b$closure$u7d$$u7d$::hea2cd86757806f7a (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x5d140f) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #4 0x5598a8faf6cf in criterion::benchmark_group::BenchmarkGroup$LT$M$GT$::bench_function::_$u7b$$u7b$closure$u7d$$u7d$::hd042ce055f6fd19b (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x6396cf) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #5 0x5598a8df5d6b in _$LT$criterion..routine..Function$LT$M$C$F$C$T$GT$$u20$as$u20$criterion..routine..Routine$LT$M$C$T$GT$$GT$::warm_up::hf7b6b8ea8839b272 (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x47fd6b) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #6 0x5598a8e34587 in criterion::routine::Routine::sample::h833d2d40f718aa34 (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x4be587) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #7 0x5598a8f7d438 in criterion::analysis::common::h54916339714b3685 (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x607438) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #8 0x5598a8fd9d93 in criterion::benchmark_group::BenchmarkGroup$LT$M$GT$::run_bench::h6ffad90df37824ec (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x663d93) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #9 0x5598a8facdf2 in criterion::benchmark_group::BenchmarkGroup$LT$M$GT$::bench_function::hc758120eb1b398de (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x636df2) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #10 0x5598a90f5658 in criterion::Criterion$LT$M$GT$::bench_function::h43c1b741f0e071ea (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x77f658) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #11 0x5598a8f46b2b in criterion_tests::test_measurement_time::h87cadba36f25acc1 (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x5d0b2b) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)
    #12 0x5598a8f46122 in criterion_tests::test_measurement_time::_$u7b$$u7b$closure$u7d$$u7d$::h1150f3d38a166b6d (/root/build/target/x86_64-unknown-linux-gnu/debug/deps/criterion_tests-94e045d31360a092+0x5d0122) (BuildId: 46d3bcc4e5b56eee8af8b7d71a998ef2f69c7df4)

As well as another benchmarking library which does the same read_volatile trick.

Similarly, I do not understand why this happens, but I have not reported that anywhere because I have been completely unable to reproduce these ASan issues outside of my somewhat-complicated test harness: https://github.com/saethlin/miri-tools/blob/9799fd71de5d33f3dc360d94d3ffdb76b28ee841/src/main.rs#L218-L234

@rustbot label +A-LLVM

[krasimirgg]

I digged a bit more. This started failing after LLVM commit llvm/llvm-project@e7bac3b [msan] Convert Msan to ModulePass.
There's something wrong with the core::hint::black_box intrinsic:

rust/library/core/src/hint.rs

Lines 225 to 227 in 822f8c2

	pub const fn black_box<T>(dummy: T) -> T {
	crate::intrinsics::black_box(dummy)
	}

That gets lowered into assembly here:

rust/compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines 341 to 366 in 822f8c2

	sym::black_box => {
	args[0].val.store(self, result);
	// We need to "use" the argument in some way LLVM can't introspect, and on
	// targets that support it we can typically leverage inline assembly to do
	// this. LLVM's interpretation of inline assembly is that it's, well, a black
	// box. This isn't the greatest implementation since it probably deoptimizes
	// more than we want, but it's so far good enough.
	crate::asm::inline_asm_call(
	self,
	"",
	"r,~{memory}",
	&[result.llval],
	self.type_void(),
	true,
	false,
	llvm::AsmDialect::Att,
	&[span],
	false,
	None,
	)
	.unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
	// We have copied the value to `result` already.
	return;
	}

I set up an example here: https://github.com/krasimirgg/example-rust-llvm-head-16-msan-error, reproducible via run_in_docker.sh penguin. Consider this code:

#![feature(core_intrinsics)]
#[no_mangle]
extern "C" fn penguin() -> i64 {
    let dummy = ();
    core::intrinsics::black_box(dummy);
    0
}

That produces this LLVM IR:

*** IR Dump Before MemorySanitizerPass on [module] ***
; Function Attrs: nonlazybind sanitize_memory uwtable
define i64 @penguin() unnamed_addr #0 !dbg !6 {
start:
  %dummy.dbg.spill = alloca {}, align 1
  call void @llvm.dbg.declare(metadata ptr %dummy.dbg.spill, metadata !13, metadata !DIExpression()), !dbg !17
  call void asm sideeffect "", "r,~{memory}"(ptr undef), !dbg !18, !srcloc !19
  br label %bb1, !dbg !18

bb1:                                              ; preds = %start
  ret i64 0, !dbg !20
}
*** IR Dump Before AnnotationRemarksPass on penguin ***
; Function Attrs: nonlazybind sanitize_memory uwtable
define i64 @penguin() unnamed_addr #0 !dbg !6 {
start:
  call void @llvm.donothing()
  %dummy.dbg.spill = alloca {}, align 1
  %0 = ptrtoint ptr %dummy.dbg.spill to i64
  %1 = xor i64 %0, 87960930222080
  %2 = inttoptr i64 %1 to ptr
  call void @llvm.memset.p0.i64(ptr align 1 %2, i8 -1, i64 0, i1 false)
  call void @llvm.dbg.declare(metadata ptr %dummy.dbg.spill, metadata !13, metadata !DIExpression()), !dbg !17
  call void @__msan_warning_noreturn() #5, !dbg !18
  call void asm sideeffect "", "r,~{memory}"(ptr undef), !dbg !18, !srcloc !19
  br label %bb1, !dbg !18

bb1:                                              ; preds = %start
  store i64 0, ptr @__msan_retval_tls, align 8, !dbg !20
  ret i64 0, !dbg !20
}

I asked around and was told MSAN is supposed to warn on stuff like this (ptr undef) and warns after that LLVM commit via the unconditional call to __msan_warning_noreturn after the MSAN pass. I suspect before that LLVM commit something else in the pipeline was interfering with producing this warning.