tooling(recursion): count keccak hashes to verify a proof (excl. grin…

crypto/crypto/Cargo.toml

@@ -44,4 +44,7 @@ serde = ["dep:serde"]
 parallel = ["dep:rayon"]
 disk-spill = ["std", "dep:memmap2", "dep:tempfile", "dep:libc"]
 alloc = []
-rkyv = ["dep:rkyv", "math/rkyv"]
+rkyv = ["dep:rkyv", "math/rkyv"]
+# Host-only diagnostic: count keccak finalizes during verify (see `hash_metrics`).
+# Off by default → `PlatformKeccak256 = sha3::Keccak256`, provably unchanged.
+hash-metrics = []

crypto/crypto/src/hash/platform_keccak.rs

@@ -58,7 +58,69 @@ mod imp {
     }
 }
 
-#[cfg(not(target_arch = "riscv64"))]
+// Host, `hash-metrics` feature ON: `sha3::Keccak256` plus a finalize counter for
+// [`crate::hash_metrics`]. The counter is a PURE SIDE EFFECT — every method
+// forwards to the inner hasher (byte-identical digest) and is `#[inline(always)]`,
+// so no cross-crate call is added over the bare alias.
+#[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))]
+mod imp {
+    use digest::{
+        FixedOutput, FixedOutputReset, HashMarker, Output, OutputSizeUser, Reset, Update,
+    };
+
+    /// The `usize` accumulates bytes absorbed for the CURRENT hash, so
+    /// `finalize` can report the permutation count (`bytes / 136 + 1`). It resets
+    /// to zero on `reset` / `finalize_into_reset`, and `finalize_into` consumes
+    /// `self`.
+    #[derive(Clone, Default)]
+    pub struct PlatformKeccak256(sha3::Keccak256, usize);
+
+    impl HashMarker for PlatformKeccak256 {}
+
+    impl OutputSizeUser for PlatformKeccak256 {
+        type OutputSize = digest::typenum::U32;
+    }
+
+    impl Update for PlatformKeccak256 {
+        #[inline(always)]
+        fn update(&mut self, data: &[u8]) {
+            // Track absorbed bytes (for the finalize permutation count) and the
+            // host-side absorption stats.
+            self.1 += data.len();
+            crate::hash_metrics::count_absorb(data.len());
+            Update::update(&mut self.0, data);
+        }
+    }
+
+    impl FixedOutput for PlatformKeccak256 {
+        #[inline(always)]
+        fn finalize_into(self, out: &mut Output<Self>) {
+            crate::hash_metrics::count_finalize(self.1);
+            FixedOutput::finalize_into(self.0, out);
+        }
+    }
+
+    impl Reset for PlatformKeccak256 {
+        #[inline(always)]
+        fn reset(&mut self) {
+            self.1 = 0;
+            Reset::reset(&mut self.0);
+        }
+    }
+
+    impl FixedOutputReset for PlatformKeccak256 {
+        #[inline(always)]
+        fn finalize_into_reset(&mut self, out: &mut Output<Self>) {
+            crate::hash_metrics::count_finalize(self.1);
+            self.1 = 0;
+            FixedOutputReset::finalize_into_reset(&mut self.0, out);
+        }
+    }
+}
+
+// Default host build (no `hash-metrics` feature): the plain alias, provably
+// unchanged from upstream.
+#[cfg(all(not(target_arch = "riscv64"), not(feature = "hash-metrics")))]
 mod imp {
     pub type PlatformKeccak256 = sha3::Keccak256;
 }

crypto/crypto/src/hash_metrics.rs

@@ -0,0 +1,160 @@
+//! Host-only keccak-hash counters for measuring the cost of VERIFYING a proof
+//! (a proxy for the recursion guest's dominant work: keccak hashing).
+//!
+//! Behind the `hash-metrics` cargo feature: a normal build keeps
+//! `PlatformKeccak256 = sha3::Keccak256` and every counter call compiles to
+//! nothing, so the prover is provably unchanged. With the feature on (host only),
+//! the host `PlatformKeccak256` wrapper counts, per keccak op:
+//! * `perms` — keccak-f permutations, THE unit the guest pays (one `keccak_permute`
+//!   syscall each): `sum over hashes of (absorbed_bytes / 136 + 1)`. Guest-faithful
+//!   regardless of host call shape — a 64-byte Merkle parent is 1 perm here (2
+//!   updates + finalize) and 1 on the guest (`keccak256_pair`) — so it is
+//!   self-validatable against a measured `keccak_permute` census;
+//! * `total` — every finalize (leaf / node / transcript squeeze / program-id fold).
+//!   `merkle` and `grinding` are disjoint keccak-only subsets of it (the remainder
+//!   is the transcript / program-id fold); `merkle_nodes ⊆ merkle`, so
+//!   leaves = `merkle − merkle_nodes`;
+//! * `absorb_calls` / `absorb_bytes` — every `Update::update`. HOST-side absorption:
+//!   the guest takes `keccak256_pair` for Merkle parents (0 updates), so its absorb
+//!   calls ≈ `absorb_calls − 2·merkle_nodes`. Useful for spotting a block-absorption
+//!   change (fewer, larger updates — same `perms`, fewer `absorb_calls`).
+//!
+//! No enable/disable toggle and nothing in the verifier: counting is always on
+//! under the feature, and a measuring caller just [`reset`]s before the verify
+//! and reads [`snapshot`] after. Grinding is separated by counter, not excluded
+//! at a call site, so there is no cross-thread race under a parallel verify.
+
+/// Snapshot of the verify-hash counters (all zero without the `hash-metrics`
+/// feature / on the guest).
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Counts {
+    /// keccak-f permutations — the unit the guest pays (one `keccak_permute` each).
+    pub perms: u64,
+    /// Every keccak-256 finalize.
+    pub total: u64,
+    /// Merkle finalizes (keccak-guarded subset of `total`).
+    pub merkle: u64,
+    /// Merkle auth-path (parent) compressions (subset of `merkle`).
+    pub merkle_nodes: u64,
+    /// Grinding proof-of-work finalizes (subset of `total`).
+    pub grinding: u64,
+    /// Keccak absorb (`Update::update`) invocations (host-side).
+    pub absorb_calls: u64,
+    /// Bytes fed through absorb (sum of `data.len()`).
+    pub absorb_bytes: u64,
+}
+
+#[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))]
+mod imp {
+    use super::Counts;
+    use core::sync::atomic::{AtomicU64, Ordering};
+
+    /// keccak-256 rate in bytes (1088 bits): bytes absorbed per permutation.
+    const RATE: usize = 136;
+
+    static PERMS: AtomicU64 = AtomicU64::new(0);
+    static TOTAL: AtomicU64 = AtomicU64::new(0);
+    static MERKLE: AtomicU64 = AtomicU64::new(0);
+    static MERKLE_NODES: AtomicU64 = AtomicU64::new(0);
+    static GRINDING: AtomicU64 = AtomicU64::new(0);
+    static ABSORB_CALLS: AtomicU64 = AtomicU64::new(0);
+    static ABSORB_BYTES: AtomicU64 = AtomicU64::new(0);
+
+    /// A keccak-256 finalize of a hash that absorbed `nbytes`. Bumps the finalize
+    /// count and the permutation count (`nbytes / RATE + 1` — the full blocks
+    /// absorbed plus the padded final block), the unit the guest pays.
+    #[inline(always)]
+    pub fn count_finalize(nbytes: usize) {
+        TOTAL.fetch_add(1, Ordering::Relaxed);
+        PERMS.fetch_add((nbytes / RATE + 1) as u64, Ordering::Relaxed);
+    }
+
+    /// A Merkle finalize (leaf or node), counted ONLY when the backend digest is
+    /// the platform keccak wrapper — the one whose `finalize` also bumps
+    /// [`count_finalize`]. This keeps `merkle` a strict subset of `total` for ANY
+    /// `D` (a non-keccak backend, as in the crypto tests, does not go through the
+    /// counted wrapper, so counting it here would let `merkle` exceed `total`).
+    #[inline(always)]
+    pub fn count_merkle<D: 'static>() {
+        if core::any::TypeId::of::<D>()
+            == core::any::TypeId::of::<crate::hash::platform_keccak::PlatformKeccak256>()
+        {
+            MERKLE.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+
+    /// A Merkle parent (auth-path) compression. Subset of [`count_merkle`];
+    /// same keccak-only guard. Every parent must ALSO call [`count_merkle`] so
+    /// `merkle_nodes ⊆ merkle`.
+    #[inline(always)]
+    pub fn count_merkle_node<D: 'static>() {
+        if core::any::TypeId::of::<D>()
+            == core::any::TypeId::of::<crate::hash::platform_keccak::PlatformKeccak256>()
+        {
+            MERKLE_NODES.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+
+    /// A grinding (proof-of-work) finalize. Subset of [`count_finalize`]; a caller
+    /// reports `total - grinding` to exclude the PoW check.
+    #[inline(always)]
+    pub fn count_grinding() {
+        GRINDING.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// A keccak absorb (`Update::update`) of `nbytes` (host-side). Bumps the call
+    /// count and the byte total.
+    #[inline(always)]
+    pub fn count_absorb(nbytes: usize) {
+        ABSORB_CALLS.fetch_add(1, Ordering::Relaxed);
+        ABSORB_BYTES.fetch_add(nbytes as u64, Ordering::Relaxed);
+    }
+
+    /// Zero all counters.
+    pub fn reset() {
+        PERMS.store(0, Ordering::Relaxed);
+        TOTAL.store(0, Ordering::Relaxed);
+        MERKLE.store(0, Ordering::Relaxed);
+        MERKLE_NODES.store(0, Ordering::Relaxed);
+        GRINDING.store(0, Ordering::Relaxed);
+        ABSORB_CALLS.store(0, Ordering::Relaxed);
+        ABSORB_BYTES.store(0, Ordering::Relaxed);
+    }
+
+    pub fn snapshot() -> Counts {
+        Counts {
+            perms: PERMS.load(Ordering::Relaxed),
+            total: TOTAL.load(Ordering::Relaxed),
+            merkle: MERKLE.load(Ordering::Relaxed),
+            merkle_nodes: MERKLE_NODES.load(Ordering::Relaxed),
+            grinding: GRINDING.load(Ordering::Relaxed),
+            absorb_calls: ABSORB_CALLS.load(Ordering::Relaxed),
+            absorb_bytes: ABSORB_BYTES.load(Ordering::Relaxed),
+        }
+    }
+}
+
+// Feature off, or the riscv64 guest: every entry compiles to nothing.
+#[cfg(any(target_arch = "riscv64", not(feature = "hash-metrics")))]
+mod imp {
+    use super::Counts;
+
+    #[inline(always)]
+    pub fn count_finalize(_nbytes: usize) {}
+    #[inline(always)]
+    pub fn count_merkle<D: 'static>() {}
+    #[inline(always)]
+    pub fn count_merkle_node<D: 'static>() {}
+    #[inline(always)]
+    pub fn count_grinding() {}
+    #[inline(always)]
+    pub fn count_absorb(_nbytes: usize) {}
+    pub fn reset() {}
+    pub fn snapshot() -> Counts {
+        Counts::default()
+    }
+}
+
+pub use imp::{
+    count_absorb, count_finalize, count_grinding, count_merkle, count_merkle_node, reset, snapshot,
+};

crypto/crypto/src/lib.rs

@@ -9,6 +9,7 @@ extern crate alloc;
 
 pub mod fiat_shamir;
 pub mod hash;
+pub mod hash_metrics;
 pub mod merkle_tree;
 #[cfg(feature = "disk-spill")]
 pub mod mmap_util;

crypto/crypto/src/merkle_tree/backends/field_element.rs

@@ -22,7 +22,7 @@ impl<F, D: Digest, const NUM_BYTES: usize> Default for FieldElementBackend<F, D,
     }
 }
 
-impl<F, D: Digest, const NUM_BYTES: usize> IsMerkleTreeBackend
+impl<F, D: Digest + 'static, const NUM_BYTES: usize> IsMerkleTreeBackend
     for FieldElementBackend<F, D, NUM_BYTES>
 where
     F: IsField,
@@ -33,12 +33,20 @@ where
     type Data = FieldElement<F>;
 
     fn hash_data(input: &FieldElement<F>) -> [u8; NUM_BYTES] {
+        // Merkle leaf finalize (see `crate::hash_metrics`); counts only when `D`
+        // is the platform keccak (so `merkle ⊆ total`), no-op without the feature.
+        crate::hash_metrics::count_merkle::<D>();
         let mut hasher = D::new();
         input.stream_bytes(&mut |b| hasher.update(b));
         hasher.finalize().into()
     }
 
     fn hash_new_parent(left: &[u8; NUM_BYTES], right: &[u8; NUM_BYTES]) -> [u8; NUM_BYTES] {
+        // Merkle auth-path (node) compression; keccak-only guard, no-op without
+        // the feature. Unlike `field_element_vector`, this backend does not route
+        // through `hash_streamed`, so count BOTH here to keep `nodes ⊆ merkle`.
+        crate::hash_metrics::count_merkle::<D>();
+        crate::hash_metrics::count_merkle_node::<D>();
         let mut hasher = D::new();
         hasher.update(left);
         hasher.update(right);

crypto/crypto/src/merkle_tree/backends/field_element_vector.rs

@@ -43,6 +43,9 @@ use lambda_vm_syscalls::keccak::Keccak256 as SyscallKeccak256;
 fn hash_streamed<D: Digest + 'static, const NUM_BYTES: usize>(
     feed: impl Fn(&mut dyn FnMut(&[u8])),
 ) -> [u8; NUM_BYTES] {
+    // Metric: a Merkle finalize (leaf or node). Counts only when `D` is the
+    // platform keccak (so `merkle ⊆ total`); no-op on guest / without the feature.
+    crate::hash_metrics::count_merkle::<D>();
     #[cfg(target_arch = "riscv64")]
     if NUM_BYTES == 32 && TypeId::of::<D>() == TypeId::of::<PlatformKeccak256>() {
         let mut hasher = SyscallKeccak256::new();
@@ -75,6 +78,10 @@ fn hash_new_parent_bytes<D: Digest + 'static, const NUM_BYTES: usize>(
     left: &[u8; NUM_BYTES],
     right: &[u8; NUM_BYTES],
 ) -> [u8; NUM_BYTES] {
+    // Metric: a Merkle parent (auth-path) compression. On the host this also
+    // flows through `hash_streamed` (one `count_merkle`), so merkle − nodes =
+    // leaves. Keccak-only guard; no-op on guest / without the feature.
+    crate::hash_metrics::count_merkle_node::<D>();
     #[cfg(target_arch = "riscv64")]
     if NUM_BYTES == 32 && TypeId::of::<D>() == TypeId::of::<PlatformKeccak256>() {
         let l: &[u8; 32] = left[..].try_into().unwrap();

crypto/stark/src/grinding.rs

@@ -65,6 +65,9 @@ pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option<u64> {
 /// when interpreted as `u64`.
 #[inline(always)]
 fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool {
+    // Tag this finalize as grinding so a verify-hash metric can report it apart
+    // (see `crypto::hash_metrics`); no-op unless the `hash-metrics` feature is on.
+    crypto::hash_metrics::count_grinding();
     let mut data = [0; 40];
     data[..32].copy_from_slice(inner_hash);
     data[32..].copy_from_slice(&candidate_nonce.to_be_bytes());
@@ -79,6 +82,8 @@ fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, li
 /// Hash(prefix || seed || grinding_factor)
 /// `prefix` is the bit-string `0x123456789abcded`
 fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] {
+    // Grinding finalize (see `crypto::hash_metrics`); no-op unless enabled.
+    crypto::hash_metrics::count_grinding();
     let mut inner_data = [0u8; 41];
     inner_data[0..8].copy_from_slice(&PREFIX);
     inner_data[8..40].copy_from_slice(seed);

prover/Cargo.toml

@@ -15,6 +15,8 @@ instruments = ["stark/instruments"]
 nvtx = ["cuda", "instruments", "stark/nvtx"]
 profile-markers = ["stark/profile-markers"]
 disk-spill = ["stark/disk-spill"]
+# Host-only verify-hash counter for `test_count_recursion_hashes` (see `hash_metrics`).
+hash-metrics = ["crypto/hash-metrics"]
 
 [dependencies]
 stark = { path = "../crypto/stark" }