diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index ae675d770..7b6258179 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -370,12 +370,30 @@ jobs: with: name: prover-tests + - name: Cache compiled recursion guest ELF artifacts + id: cache-recursion-elfs + uses: actions/cache@v4 + with: + path: executor/program_artifacts/recursion + key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }} + restore-keys: | + recursion-elf-artifacts- + + - name: Setup Rust Environment (recursion ELFs) + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' + uses: ./.github/actions/setup-rust + + - name: Compile recursion guest ELFs + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' + run: | + make compile-recursion-elfs + - name: Run comprehensive prover tests run: | cargo nextest run \ --archive-file prover-tests.tar.zst \ --test-threads=1 \ - -E 'test(test_prove_elfs_all_instructions_64_full)' \ + -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion_execute)' \ --run-ignored ignored-only # Seed ELF caches on refs/heads/main so merge-queue runs can restore them. @@ -424,3 +442,20 @@ jobs: - name: Compile Rust programs to ELF if: steps.cache-rust-elfs.outputs.cache-hit != 'true' run: make compile-programs-rust + + - name: Cache compiled recursion guest ELF artifacts + id: cache-recursion-elfs + uses: actions/cache@v4 + with: + path: executor/program_artifacts/recursion + key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }} + restore-keys: | + recursion-elf-artifacts- + + - name: Setup Rust Environment (recursion ELFs) + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' + uses: ./.github/actions/setup-rust + + - name: Compile recursion guest ELFs + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' + run: make compile-recursion-elfs diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index a4f9fb7b0..f072eec53 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -4,17 +4,13 @@ //! 1. Proves an inner program on the host. //! 2. Serializes `(VmProof, inner_elf, opts)` with postcard. //! 3. Hands that as private input to the recursion guest. -//! 4. Either **proves** the recursion guest's execution and verifies the outer -//! proof (`OuterMode::Prove`), or merely **executes** the guest in-VM and -//! reads the committed marker off the trace (`OuterMode::ExecuteOnly`) — a -//! cheaper tier that skips the LDE/FRI that dominate the full pipeline. +//! 4. Either **proves** the recursion guest's execution (memory-bounded via +//! continuations) and verifies the outer proof (`OuterMode::Prove`), or +//! merely **executes** the guest in-VM and reads the committed marker +//! straight off the executor's memory (`OuterMode::ExecuteOnly`) — a cheaper +//! tier that skips the LDE/FRI that dominate the full pipeline. //! -//! The guest ELFs are built by `make compile-recursion-elfs` (which the -//! `test-prover-all` make target depends on) and read from -//! `executor/program_artifacts/recursion/`, like every other program test. -//! -//! Tests are `#[ignore]`d because the outer proof runs the full STARK verifier -//! inside the VM (minutes per run, large memory footprint). +//! The guest ELFs are assumed built by `make compile-recursion-elfs`. use std::path::PathBuf; @@ -83,64 +79,77 @@ fn prove_inner_and_encode_blob( /// whether we also prove the guest's own execution. #[derive(Clone, Copy, Debug)] enum OuterMode { - /// Execute the guest in-VM and read the committed marker off the trace. - /// Skips the LDE blowup + FRI commit that dominate the full pipeline's - /// footprint, so it needs materially less RAM than `Prove`. - /// - /// "Less" is not "little": `Executor::run` retains a per-instruction log - /// and `Traces` materializes the full execution trace, so verifying even a - /// 1-query inner proof still needs tens of GB — it OOMs on a 36 GB box. + /// Execute the guest in-VM and read the committed marker straight off the + /// executor's memory. Streams logs via `Executor::resume()` and never + /// builds a `Traces`, so footprint stays bounded to the VM's touched + /// memory + instruction cache. Skips the LDE/FRI of the full pipeline entirely. ExecuteOnly, - /// Prove the guest's execution and verify the outer proof on the host. The - /// full STARK verifier inside the VM — minutes per run, ~125 GB. + /// Prove the guest's execution via continuations, then verify the outer + /// proof on the host. `prove_and_verify_continuation` retains every epoch's + /// STARK proof in the bundle before verifying, so peak RAM grows with epoch + /// count. Heavy — excluded from CI, run manually. A future verify-one-and- + /// discard API extension would make this memory-friendlier. Prove, } /// Execute the recursion guest in-VM on `blob` and return the bytes it /// committed (the success marker the in-VM verifier emits). +/// +/// Streams execution via `Executor::resume()`. The committed marker is +/// read directly off the executor's memory. This avoids OOMs. fn execute_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { use executor::elf::Elf; use executor::vm::execution::Executor; - eprintln!("[{label}] executing outer (recursion guest, in-VM verify) ..."); + eprintln!("[{label}] executing outer (recursion guest, in-VM verify, streaming) ..."); let program = Elf::load(recursion_elf_bytes).expect("load recursion elf"); - let result = Executor::new(&program, blob.to_vec()) - .expect("executor new") - .run() - .expect("recursion guest execution failed (verify panicked in-VM?)"); - - let traces = crate::tables::trace_builder::Traces::from_elf_and_logs( - &program, - &result.logs, - &crate::MaxRowsConfig::default(), - blob, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let mut executor = Executor::new(&program, blob.to_vec()).expect("executor new"); + + // Drain chunks to completion without retaining logs or building a trace. + while executor + .resume() + .expect("recursion guest execution failed (verify panicked in-VM?)") + .is_some() + {} + + let committed = executor + .finish() + .expect("read committed output after execution") + .memory_values; eprintln!( "[{label}] committed {} bytes: {:?} (as str: {:?})", - traces.public_output_bytes.len(), - traces.public_output_bytes, - String::from_utf8_lossy(&traces.public_output_bytes), + committed.len(), + committed, + String::from_utf8_lossy(&committed), ); - traces.public_output_bytes + committed } -/// Prove the recursion guest's execution on `blob`, verify the outer proof on -/// the host, and return the bytes the guest committed. -fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { - eprintln!("[{label}] proving outer (recursion guest) ..."); - let outer_proof = - crate::prove_with_inputs(recursion_elf_bytes, blob).expect("outer prove should succeed"); - eprintln!("[{label}] outer proof generated"); +/// Epoch size for the outer prove: 2^16 ≈ 65K cycles. Small so one epoch's +/// trace+LDE stays under the ~16GiB CI runners. +const OUTER_EPOCH_SIZE_LOG2: u32 = 16; - assert!( - crate::verify(&outer_proof, recursion_elf_bytes).expect("outer verify errored"), - "outer proof must verify on host" +/// Prove the recursion guest's execution on `blob` memory-bounded via +/// continuations and verify the bundle on the host, returning the bytes the +/// guest committed. +fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { + let opts = + crate::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); + eprintln!( + "[{label}] proving outer (recursion guest) via continuations \ + (epoch=2^{OUTER_EPOCH_SIZE_LOG2} cycles) ..." ); - outer_proof.public_output + let committed = crate::continuation::prove_and_verify_continuation( + recursion_elf_bytes, + blob, + OUTER_EPOCH_SIZE_LOG2, + &opts, + ) + .expect("outer continuation prove/verify errored") + .expect("outer continuation proof must verify on host"); + eprintln!("[{label}] outer continuation proof generated and verified"); + committed } /// Core pipeline: prove an inner program with the given options, hand the @@ -214,10 +223,7 @@ fn run_recursion_pipeline( /// Reproduce the recursion guest's EXACT path on the host — decode the postcard /// blob into `(VmProof, Vec, ProofOptions)` and call `verify_with_options`. -/// The cheapest regression guard in this file: no VM execution, just the -/// encode/decode contract plus a host verify, so it catches drift in the proof -/// format or the blob layout in seconds. Unlike the guest, a failure here -/// surfaces the actual error instead of an infinite abort loop. +/// Cheap regression guard. #[test] #[ignore = "needs prebuilt guest ELF (make compile-recursion-elfs)"] fn test_recursion_blob_decodes_and_verifies_on_host() { @@ -249,15 +255,11 @@ fn test_recursion_blob_decodes_and_verifies_on_host() { } // === Execute-only tier ======================================================== -// Mirrors the proving tests below, but stops at `OuterMode::ExecuteOnly`: the -// guest runs in-VM and we read the committed marker off the trace, skipping the -// outer STARK prove. Needs tens of GB (execution trace), not the ~125 GB the -// full outer prove wants — but still OOMs on a 36 GB box. /// Execute-only mirror of `test_recursion_prove_empty`: verify a `blowup=8` /// proof of the empty program in-VM. #[test] -#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"] +#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_execute_empty() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -272,7 +274,7 @@ fn test_recursion_execute_empty() { /// Execute-only mirror of `test_recursion_prove_1query`: smallest possible /// inner proof (blowup=2, 1 query) → least guest work. #[test] -#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"] +#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_execute_1query() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -288,7 +290,7 @@ fn test_recursion_execute_1query() { /// Execute-only mirror of `test_recursion_prove`: verify a `blowup=8` proof of /// fibonacci(10) in-VM. #[test] -#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"] +#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_execute() { let root = workspace_root(); let fib_elf_bytes = read_guest_elf(&root, "fibonacci"); @@ -307,10 +309,9 @@ fn test_recursion_execute() { // === Full-prove tier ========================================================== /// Inner program: empty (halt immediately). Useful for measuring the -/// lambda-vm verifier's intrinsic recursion overhead — i.e. what it costs -/// to verify the smallest possible lambda-vm proof, with no inner workload. +/// verifier's intrinsic recursion overhead. #[test] -#[ignore = "slow: runs the full STARK verifier inside the VM"] +#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove_empty() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -323,11 +324,9 @@ fn test_recursion_prove_empty() { } /// Inner program: empty, but with the absolute-minimum FRI parameters -/// (blowup=2, **fri_number_of_queries=1**). This is a "can the pipeline even -/// run end-to-end on a 125 GB box" experiment — security is intentionally -/// terrible. Use only for capacity probing. +/// (blowup=2, **fri_number_of_queries=1**). For quick profiling only. #[test] -#[ignore = "slow: runs the full STARK verifier inside the VM"] +#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove_1query() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -343,7 +342,7 @@ fn test_recursion_prove_1query() { /// Inner program: fibonacci(10). #[test] -#[ignore = "slow: runs the full STARK verifier inside the VM"] +#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove() { let root = workspace_root(); let fib_elf_bytes = read_guest_elf(&root, "fibonacci");