feat: guest-side step-profiling markers and per-step function histograms for the recursion verifier

.github/scripts/aggregate_recursion_histogram.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Format the recursion-guest per-function profile as a Markdown PR comment.
+
+`test_recursion_profile_1query`/`_multiquery` print a global top-25 functions
+table (folded over all verifier steps, % of total run cycles), followed by
+one top-25 table per verifier step (% of that step's own cycles, so the
+table shows what dominates *within* the step) — e.g. how much of
+`step4:openings` is `keccak`. We parse all of those tables and render them
+as Markdown.
+
+    Top 25 functions by cycle count (aggregated over their PCs, all steps; % of total cycles):
+      rank          cycles        %    cum %    PCs  function
+         1         5335072   24.95%   24.95%     72  <...>::visit_seq::<...>
+
+    Top 25 functions by cycle count — step airs_bus_balance (% of this step's 5129138364 cycles):
+      rank          cycles        %    cum %    PCs  function
+         1         5335072   24.95%   24.95%     72  <...>::visit_seq::<...>
+
+Reads the test's captured output from argv[1]; writes the Markdown body to
+argv[2] (or stdout).
+"""
+
+import re
+import sys
+from collections import OrderedDict
+
+# A per-function summary row: rank, cycles, pct%, cum%, pcs, function.
+FN_ROW = re.compile(
+    r"^\s*\d+\s+(\d+)\s+([\d.]+)%\s+([\d.]+)%\s+(\d+)\s+(.*\S)\s*$"
+)
+HEADER_ROW = re.compile(r"^\s*rank\s+cycles")
+GLOBAL_TABLE_START = re.compile(
+    r"Top \d+ functions by cycle count \(aggregated over their PCs, all steps"
+)
+STEP_TABLE_START = re.compile(
+    r"Top \d+ functions by cycle count — step (\S+) \(% of this step's (\d+) cycles\):"
+)
+TOTAL_CYCLES = re.compile(r"Total cycles\s*:\s*(\d+)")
+UNIQUE_PCS = re.compile(r"Unique PCs\s*:\s*(\d+)")
+EXEC_TIME = re.compile(r"Exec time\s*:\s*(\S+)")
+
+GLOBAL_KEY = "__global__"
+
+
+def parse(text):
+    total_cycles = unique_pcs = exec_time = None
+    # GLOBAL_KEY -> {"denom": int|None, "rows": [...]}, then one entry per
+    # step tag in first-seen order.
+    tables = OrderedDict()
+    current = None
+    skip_header = False
+    for line in text.splitlines():
+        if total_cycles is None and (m := TOTAL_CYCLES.search(line)):
+            total_cycles = int(m.group(1))
+        if unique_pcs is None and (m := UNIQUE_PCS.search(line)):
+            unique_pcs = int(m.group(1))
+        if exec_time is None and (m := EXEC_TIME.search(line)):
+            exec_time = m.group(1)
+
+        if GLOBAL_TABLE_START.search(line):
+            current = GLOBAL_KEY
+            tables[current] = {"denom": total_cycles, "rows": []}
+            skip_header = True
+            continue
+        if m := STEP_TABLE_START.search(line):
+            current = m.group(1)
+            tables[current] = {"denom": int(m.group(2)), "rows": []}
+            skip_header = True
+            continue
+
+        if current is None:
+            continue
+        if skip_header:
+            # The header row right after a table-start line; anything else
+            # (e.g. a stray blank line) just ends the table early, which is
+            # fine — an empty table renders as "no rows".
+            skip_header = False
+            if HEADER_ROW.match(line):
+                continue
+        if m := FN_ROW.match(line):
+            tables[current]["rows"].append(
+                {
+                    "cycles": int(m.group(1)),
+                    "pct": m.group(2),
+                    "cum": m.group(3),
+                    "pcs": int(m.group(4)),
+                    "fn": m.group(5),
+                }
+            )
+        else:
+            current = None
+
+    return total_cycles, unique_pcs, exec_time, tables
+
+
+def short(name, width=90):
+    return name if len(name) <= width else name[: width - 1] + "…"
+
+
+def render_table(rows, denom_label):
+    if not rows:
+        return "> _no rows_\n"
+    body = "| Rank | Cycles | % | Cum % | PCs | Function |\n"
+    body += "|-----:|-------:|--:|------:|----:|----------|\n"
+    for i, r in enumerate(rows, 1):
+        body += (
+            f"| {i} | {r['cycles']:,} | {r['pct']}% | {r['cum']}% | "
+            f"{r['pcs']} | `{short(r['fn'])}` |\n"
+        )
+    last_cum = rows[-1]["cum"]
+    body += (
+        f"\n<sub>Each function's cycles are summed over all its program counters "
+        f"in this table's scope; the top {len(rows)} cover {last_cum}% of "
+        f"{denom_label}.</sub>\n"
+    )
+    return body
+
+
+def render(total_cycles, unique_pcs, exec_time, tables, title="Recursion guest profile"):
+    if not tables.get(GLOBAL_KEY, {}).get("rows"):
+        return (
+            f"### {title}\n\n"
+            "> ⚠️ No per-function rows found in the test output — the run may "
+            "have failed before printing the table. Check the workflow logs.\n"
+        )
+
+    body = f"### {title}\n\n"
+    if total_cycles is not None:
+        body += f"**Total cycles:** {total_cycles:,}"
+        if unique_pcs is not None:
+            body += f" · **Unique PCs:** {unique_pcs:,}"
+        if exec_time:
+            body += f" · **Exec time:** {exec_time}"
+        body += "\n\n"
+
+    global_rows = tables[GLOBAL_KEY]["rows"]
+    body += f"#### Top {len(global_rows)} functions by cycles (all steps)\n\n"
+    body += render_table(global_rows, "total cycles")
+
+    for step, table in tables.items():
+        if step == GLOBAL_KEY:
+            continue
+        rows, denom = table["rows"], table["denom"]
+        denom_note = f" of {denom:,} step cycles" if denom is not None else ""
+        body += (
+            f"\n<details><summary>Step <code>{step}</code>{denom_note} — "
+            f"top {len(rows)} functions</summary>\n\n"
+        )
+        body += render_table(rows, "this step's cycles")
+        body += "\n</details>\n"
+
+    return body
+
+
+def main():
+    import argparse
+
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("log", help="captured test output to parse")
+    ap.add_argument("-o", "--out", help="write Markdown here instead of stdout")
+    ap.add_argument(
+        "-t",
+        "--title",
+        default="Recursion guest profile",
+        help="section heading (e.g. the test/config name)",
+    )
+    args = ap.parse_args()
+
+    with open(args.log, "r", errors="replace") as f:
+        text = f.read()
+    body = render(*parse(text), title=args.title)
+    if args.out:
+        with open(args.out, "w") as f:
+            f.write(body)
+    else:
+        sys.stdout.write(body)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/profile-recursion.yml

@@ -0,0 +1,178 @@
+name: Profile Recursion (PR)
+
+# Runs the recursion-guest PC histogram diagnostics (single-query and
+# multi-query, in parallel via a matrix) and posts a combined per-function
+# profile as a PR comment. Triggered by a `/profile_recursion` comment from a
+# repo member, or manually via workflow_dispatch.
+
+on:
+  workflow_dispatch:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: profile-recursion-${{ github.event.issue.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # One job per configuration; they run in parallel and each uploads a Markdown
+  # fragment artifact. The `comment` job stitches them into one PR comment.
+  profile:
+    # Skip unless: workflow_dispatch, or "/profile_recursion" comment on a PR by a member.
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'issue_comment' &&
+       github.event.issue.pull_request &&
+       startsWith(github.event.comment.body, '/profile_recursion') &&
+       contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association))
+    runs-on: [self-hosted, bench]
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: single-query
+            test: single
+            title: "Single query (blowup=2, 1 query)"
+          - name: multi-query
+            test: multi
+            title: "Multi query (blowup=8, 128-bit)"
+    steps:
+      - name: React to comment
+        if: github.event_name == 'issue_comment' && matrix.name == 'single-query'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.reactions.createForIssueComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: context.payload.comment.id,
+              content: 'eyes'
+            });
+
+      - name: Get PR head ref
+        id: pr-ref
+        if: github.event_name == 'issue_comment'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUM: ${{ github.event.issue.number }}
+        run: |
+          SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
+          echo "sha=$SHA" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.pr-ref.outputs.sha || github.sha }}
+
+      - name: Setup Rust Environment
+        uses: ./.github/actions/setup-rust
+
+      - name: Add cargo to PATH
+        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+
+      - name: Run recursion PC histogram (${{ matrix.name }})
+        env:
+          TEST: ${{ matrix.test }}
+        run: |
+          # Self-provision the RISC-V sysroot in a user-writable dir (the default
+          # /opt path on the bench runner is root-owned); the guest ELF build the
+          # test triggers picks this up via the Makefile's `SYSROOT_DIR ?=`.
+          export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
+          set -o pipefail
+          make test-profile-recursion-$TEST 2>&1 | tee /tmp/hist.log
+
+      - name: Aggregate into a per-function fragment
+        if: always()
+        env:
+          TITLE: ${{ matrix.title }}
+        run: |
+          python3 .github/scripts/aggregate_recursion_histogram.py \
+            /tmp/hist.log --title "$TITLE" --out "/tmp/fragment-${{ matrix.name }}.md"
+          cat "/tmp/fragment-${{ matrix.name }}.md" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload fragment
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: profile-fragment-${{ matrix.name }}
+          path: /tmp/fragment-${{ matrix.name }}.md
+          retention-days: 7
+
+  # Stitch the matrix fragments into a single PR comment.
+  comment:
+    needs: profile
+    # always() so partial-matrix failures still post; skip when `profile` was
+    # skipped (non-/profile_recursion or non-member comment) so this job — and
+    # the self-hosted bench runner it spins up — doesn't fire on every comment.
+    if: always() && github.event_name == 'issue_comment' && needs.profile.result != 'skipped'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR head ref
+        id: pr-ref
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUM: ${{ github.event.issue.number }}
+        run: |
+          SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
+          echo "sha=$SHA" >> "$GITHUB_OUTPUT"
+
+      - name: Download fragments
+        uses: actions/download-artifact@v4
+        with:
+          path: fragments
+          pattern: profile-fragment-*
+          merge-multiple: true
+
+      - name: Assemble comment body
+        env:
+          COMMIT_SHA: ${{ steps.pr-ref.outputs.sha }}
+        run: |
+          {
+            echo "## Recursion guest profile"
+            echo
+            # Single-query first, then multi-query, then any others.
+            for frag in fragments/fragment-single-query.md \
+                        fragments/fragment-multi-query.md; do
+              [ -f "$frag" ] && { cat "$frag"; echo; }
+            done
+            echo "<sub>Commit: ${COMMIT_SHA:0:8} · Runner: self-hosted bench</sub>"
+          } > /tmp/profile_comment.md
+          cat /tmp/profile_comment.md
+
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('/tmp/profile_comment.md', 'utf8');
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            // Reuse our own marker comment so repeated /profile_recursion runs update in place.
+            const existing = comments.find(c =>
+              c.user.type === 'Bot' &&
+              c.body.includes('Recursion guest profile')
+            );
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+            }

Makefile

@@ -1,7 +1,7 @@
 .PHONY: deps deps-linux deps-macos compile-programs-asm compile-programs-rust compile-bench \
 compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-shared \
 clean-recursion-elfs clean test test-asm \
-test-rust test-executor test-flamegraph flamegraph-prover \
+test-rust test-executor test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \
 test-fast test-prover test-prover-all test-disk-spill test-math-cuda test-cuda-integration \
 bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \
 update-ethrex-fixture-checksums check-ethrex-fixture-checksums
@@ -232,6 +232,14 @@ test-rust: compile-programs-rust
 test-flamegraph:
 	cargo test -p executor --test flamegraph
 
+test-profile-recursion: test-profile-recursion-single test-profile-recursion-multi
+
+test-profile-recursion-single: compile-recursion-elfs
+	cargo test --package lambda-vm-prover --lib test_recursion_profile_1query -- --ignored --nocapture
+
+test-profile-recursion-multi: compile-recursion-elfs
+	cargo test --package lambda-vm-prover --lib test_recursion_profile_multiquery -- --ignored --nocapture
+
 # Regenerate the committed ethrex block fixtures (see tooling/ethrex-fixtures).
 # Run after bumping the ethrex rev; README checksums are refreshed automatically.
 regen-ethrex-fixtures:

bench_vs/lambda/recursion/Cargo.toml

@@ -6,6 +6,8 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
-lambda-vm-prover = { path = "../../../prover", default-features = false }
+lambda-vm-prover = { path = "../../../prover", default-features = false, features = [
+    "profile-markers",
+] }
 lambda-vm-syscalls = { path = "../../../syscalls" }
 postcard = { version = "1.0", features = ["alloc"] }