From 62abc09454f901d9f63049efb843cdd3fdf8ba41 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 3 Sep 2026 08:08:36 +0000 Subject: [PATCH] fix(relearn): pin proven CUDA scoring eval image digest Pin eval_image_digest to sha256:4806db4b from CortexLM/relearn@8ffbe8a publish-eval-image run 33730126042, which pulled that digest and proved harvest-PATH plus import vllm, torchvision. Not the contract slim image and not cbc4bbb8. --- config/relearn-pin.toml | 19 ++++++------ crates/relearn-eval/tests/committed_pin.rs | 36 ++++++++++------------ docs/COMPLETENESS.md | 8 ++--- docs/RELEARN.md | 21 +++++++------ docs/external-miner/troubleshoot.md | 2 +- 5 files changed, 42 insertions(+), 44 deletions(-) diff --git a/config/relearn-pin.toml b/config/relearn-pin.toml index 6a26e3f88..4af09c71c 100644 --- a/config/relearn-pin.toml +++ b/config/relearn-pin.toml @@ -41,13 +41,13 @@ eval_image = "ghcr.io/cortexlm/relearn-eval" # deploy/env/relearn-challenge.env.example). Check GET /v1/status -> # can_score, live_harvest_wired, champion_baseline_recorded. # -# EMPTY ON PURPOSE. Every digest built so far fails the harvest, and an empty -# digest is the fail-closed state: submissions answer 503 "eval image digest -# not pinned" instead of renting a B200 that cannot score. Do not guess a -# digest here to make the 503 go away. +# Proven CUDA scoring image from CortexLM/relearn publish-eval-image on +# 8ffbe8a0481cc62c1e2d6ed80a2211befa98dd3f (run 33730126042). The runtime job +# pulled this digest, ran the harvest-PATH check, and proved +# `import vllm, torchvision` plus `relearn-eval selftest` on these bytes. +# Not the slim contract image. A live host still needs harvest + champion. # -# The next pin needs a CortexLM/relearn image whose CUDA scoring layer ships -# **vLLM and torchvision**. Do not re-pin or re-harvest any of these: +# Do not re-pin or re-harvest any of these: # # sha256:cbc4bbb8… (f3cfa69) exit 1, no EVAL_OK: no vLLM on the rented GPU # image, transformers fallback, then Qwen3VLVideoProcessor @@ -56,11 +56,10 @@ eval_image = "ghcr.io/cortexlm/relearn-eval" # sha256:303c6357… printed no RELEARN_EVAL_OK # sha256:00839671… exit 127, no /usr/bin/relearn-eval # sha256:86240d76… pre-CUDA -eval_image_digest = "" +eval_image_digest = "sha256:4806db4bd6650415c3705290117fd0190cd2a6d387cef871dd3d9ff1a8bb6d7e" relearn_git = "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/CortexLM/relearn" -# CortexLM/relearn commit that built the digest above. Empty while unpinned: -# f3cfa69 built cbc4bbb8, which never printed RELEARN_EVAL_OK. -relearn_git_sha = "" +# CortexLM/relearn commit that built the digest above. +relearn_git_sha = "8ffbe8a0481cc62c1e2d6ed80a2211befa98dd3f" # Public split ids (miners may train on these). Holdout selection must exclude # every one of them. The scored holdout records stay off git. diff --git a/crates/relearn-eval/tests/committed_pin.rs b/crates/relearn-eval/tests/committed_pin.rs index 30e48cf71..c3631eb7e 100644 --- a/crates/relearn-eval/tests/committed_pin.rs +++ b/crates/relearn-eval/tests/committed_pin.rs @@ -75,36 +75,34 @@ fn pin_says_its_commitment_is_not_the_live_seal() { ); } -/// Every digest built so far failed the live harvest, so the committed pin is -/// empty and live scoring is refused. An empty digest is a 503 on submit — the -/// fail-closed state — not a fallback to the sim harness. +/// The committed pin is the CUDA scoring digest publish-eval-image proved +/// on CortexLM/relearn@8ffbe8a (run 33730126042): harvest-PATH, import +/// vllm/torchvision, and selftest against the pulled bytes. Not the slim +/// contract image, and not any digest that already failed a live harvest. #[test] -fn committed_pin_is_fail_closed_until_a_working_image_ships() { +fn committed_pin_is_the_proven_runtime_eval_image() { let p = pin(); - assert!( - p.eval_image_digest.is_empty(), - "no CortexLM/relearn image has printed RELEARN_EVAL_OK on a rented pod; \ - an empty digest 503s, a guessed one rents a B200 that cannot score" + assert_eq!( + p.eval_image_digest, + "sha256:4806db4bd6650415c3705290117fd0190cd2a6d387cef871dd3d9ff1a8bb6d7e" ); - assert!(!p.can_rent(), "an unpinned host must not rent"); - assert!( - p.relearn_git_sha.is_empty(), - "no digest, no commit that built it" + assert!(p.can_rent(), "the proven CUDA scoring digest is rentable"); + assert_eq!( + p.relearn_git_sha, + "8ffbe8a0481cc62c1e2d6ed80a2211befa98dd3f" ); } -/// What the committed pin actually does to a live host, not just what it says. -/// The digest is the root cause, so it has to be the reported one. +/// Pinning the image is not the whole job: a live host still needs a wired +/// harvest. Sim stays opt-in. #[test] -fn the_committed_pin_refuses_live_scoring_and_says_why() { +fn the_committed_pin_still_needs_a_wired_harvest() { let err = relearn_eval::scoring_readiness(&pin(), relearn_eval::EvalBackend::Lium, None) - .expect_err("an unpinned host must not be ready to score"); + .expect_err("a pinned host without a harvest must not be ready to score"); assert!( - matches!(err, relearn_eval::EvalError::EvalImageUnpinned), + matches!(err, relearn_eval::EvalError::LiveHarvestUnavailable), "{err}" ); - assert!(err.to_string().contains("eval image digest not pinned")); - // Sim is still opt-in only, so this is a 503 rather than a sim verdict. assert!(relearn_eval::scoring_readiness(&pin(), relearn_eval::EvalBackend::Sim, None).is_ok()); assert!(!relearn_eval::force_sim(), "sim is never implicit"); } diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 65066bd3c..54f5a2bb7 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -56,7 +56,7 @@ Removed as **live products**. Shared rails (`prism-lium*`, `prism-competition` p | Crates (`crates/relearn-*`) | **done** | task (holdout commitment + contamination fingerprints), score (public–holdout gap, contamination evidence, vision shuffle, off-path general-bench canary), store, eval, http, challenge. | | Binary (`bins/relearn-challenge`) | **done** | HTTP API on `:8095`. | | Compose / images | **done** | Default compose + `images.yml` target `relearn-challenge`. | -| Eval pin | **blocked** | `config/relearn-pin.toml` — `eval_image_digest` empty on purpose, so submissions 503 `eval image digest not pinned`. Needs a `CortexLM/relearn` image whose CUDA scoring layer ships **vLLM + torchvision**. Every digest tried so far failed on a rented pod and is named in the pin: `cbc4bbb8…` (exit 1, transformers fallback then `Qwen3VLVideoProcessor` wanted torchvision), `201cc5d2…`, `303c6357…`, `00839671…`, `86240d76…`. Do not re-pin or re-harvest any of them, and do not guess a digest. | +| Eval pin | **done** | `config/relearn-pin.toml` — `eval_image` `ghcr.io/cortexlm/relearn-eval`, digest `sha256:4806db4b…`, `relearn_git_sha` `8ffbe8a0…` ([`CortexLM/relearn`](https://github.com/CortexLM/relearn) `8ffbe8a`, [publish-eval-image run 33730126042](https://github.com/CortexLM/relearn/actions/runs/33730126042)). CUDA scoring image: harvest-PATH + `import vllm, torchvision` + selftest proved on the pulled digest. Not the slim contract image. Do not re-pin `cbc4bbb8…` / `201cc5d2…` / `303c6357…` / `00839671…` / `86240d76…`. A live host still 503s until harvest + champion baseline are recorded. | | Holdout | **done** | Commitment in git, records operator-side (`RELEARN_HOLDOUT_FILE`) and verified at boot. Mismatch → submissions 503. | | Teacher | **v0** | Weights `incoai/GLM-5.3-NVFP4` served from `RELEARN_TEACHER_LOCAL_DIR` (never pass the Hugging Face repo id to vLLM). HTTP wire `glm-5.3`. Missing `RELEARN_TEACHER_API_URL` → `can_score: false` and 503 before rent. Judge-only. | | Emission | **4000 bps** | Default share (sum across all four challenges is `10000`). | @@ -175,17 +175,17 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] |-----|--------| | DCAP verify holds the attest mutex | A cold Intel PCS fetch (up to 20 s) serialises attestation submissions. | | DCAP error classification | Matches on `anyhow` message text; re-run `cargo test -p attest-policy --features dcap` after any `dcap-qvl` bump. | -| Relearn eval image digests | `relearn-image-eval` `sha256:81c40dc6…` and `relearn-agent-eval` `sha256:4db52b13…` are pinned (PR #3, `54d3537`). **`relearn-eval` is not pinned**: the last candidate, `sha256:cbc4bbb8…` (`f3cfa69`), exited 1 on a rented pod without `RELEARN_EVAL_OK` — no vLLM on the CUDA scoring image, transformers fallback, then `Qwen3VLVideoProcessor` crashed for want of torchvision. `relearn` submissions 503 until [`CortexLM/relearn`](https://github.com/CortexLM/relearn) ships an image with vLLM + torchvision. A live host still 503s after that until harvest + champion baseline are recorded. | +| Relearn eval image digests | `relearn-eval` `sha256:4806db4b…` (`8ffbe8a`, [run 33730126042](https://github.com/CortexLM/relearn/actions/runs/33730126042)), `relearn-image-eval` `sha256:81c40dc6…` and `relearn-agent-eval` `sha256:4db52b13…` (PR #3, `54d3537`). Digest-only, no floating tag. Do not re-pin `cbc4bbb8…`. A live host still 503s until harvest + champion baseline are recorded. | | Relearn Image / Agent holdout salts | Both committed commitments use documented **dev** salts so local and staging work out of the box. Production must rotate to a private salt **and** a private catalogue for each, replace the commitments, and re-sign. | | Relearn holdout salt | The committed `holdout_commitment` is the CI / local one — a documented dev salt over a synthetic catalog so the stack boots without operator secrets. It is **not** the live seal. Production must rotate to a private salt **and** a private catalog, replace the commitment in `config/relearn-pin.toml`, and re-sign the trust root ([`../config/CEREMONY.md`](../config/CEREMONY.md)). | -| Relearn live scoring | Blocked on the eval image (no working digest). Behind that, the remaining blockers are operator state: the harvest (`live_harvest_wired`) and the champion baseline (`champion_baseline_recorded`). Each has its own **503** and its own boot-log line. Sim is opt-in (`RELEARN_FORCE_SIM=1`, CI / local only), reported as `eval_backend` on `/v1/status` and on the submit row — never a fallback. Refusals persist no row. | +| Relearn live scoring | Eval image is pinned. Remaining blockers are operator state: the harvest (`live_harvest_wired`) and the champion baseline (`champion_baseline_recorded`). Each has its own **503** and its own boot-log line. Sim is opt-in (`RELEARN_FORCE_SIM=1`, CI / local only), reported as `eval_backend` on `/v1/status` and on the submit row — never a fallback. Refusals persist no row. | | Relearn live harvest | Control-plane client is **done** (`crates/relearn-lium-harvest`): boot the digest-pinned image, deliver the request, read `RELEARN_METRICS=`, verify against pin + run identity, terminate with verification. Wired on the Lium path from `LIUM_API_KEY` + `LIUM_SSH_PUBLIC_KEY_FILE`; `/v1/status` reports `live_harvest_wired`. The **scoring code** ships in [`CortexLM/relearn`](https://github.com/CortexLM/relearn) and must implement the contract in [`RELEARN.md`](RELEARN.md) § Eval image contract — an image that does not print a bound `RELEARN_METRICS=` document is a 503, never a sim score. | | Relearn holdout on rented pods | The harvest request carries the holdout items, so a rented pod sees the private split for the run. Mitigated by the digest-pinned image, `/tmp/relearn_eval` delivery, post-run scrub, and verified termination — not eliminated. Rotate salt + catalog and re-sign on suspicion. An in-enclave design would be needed to remove the exposure. | | Relearn teacher / judge keys on rented pods | `RELEARN_TEACHER_API_KEY` and `RELEARN_T2I_JUDGE_API_KEY` are forwarded into the pod when set, because a Lium `InstanceSpec` carries no env and the image judges over HTTP. Delivered over stdin (never a command line) and scrubbed after the run. A missing URL refuses **before** rent. A miner-controlled pod could still spend the operator's quota — scope and rate-limit those credentials, or leave them unset when the pod reaches the API without auth. | | Relearn champion baseline | Live hosts need an operator-recorded measurement (`RELEARN_BASE_CHAMPION_FILE`, verified against the pin's `eval_image_digest` + `holdout_commitment`). Unset means `champion_baseline_recorded: false` and every submission 503s before the gates. Sim hosts seed the sim baseline. | | Bounty severity on the backend feed | Scoring credits a `valid` row only when the backend publishes a `severity`. Until CortexLM/backend emits it, valid rows land as `valid_unpriced`, no miner can be crowned, and the share burns. Fail-closed by design: an unpriced bug cannot be paid for. | | Bounty scoring backend | With neither `BOUNTY_BACKEND_PUBLIC_URL` nor `BOUNTY_FORCE_SIM=1`, `POST /v1/reports` answers **503** rather than collecting bug-hunting work the host could never pay for. | -| Relearn public repo | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) exists; `relearn_git_sha` is empty in `config/relearn-pin.toml` because no image from it has scored on a rented pod. Bump the SHA and the digest together, from a commit reachable on that repo's default branch. Seed mirror: `docs/external-miner/relearn-seed/`. | +| Relearn public repo | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) exists; `relearn_git_sha` `8ffbe8a0…` is pinned with `eval_image_digest` `sha256:4806db4b…`. Seed mirror: `docs/external-miner/relearn-seed/`. | | Mainnet (netuid 100) | Owner wallet not yet on this machine, so prod runs with `BASE_GATEWAY_REQUIRE_OWNER=0`. | | Prod pin placeholders | `deploy/pins/prod.json` still ships zero-digests until the first successful promote; registry mode rejects placeholders. | | Spaces backup secrets | First prod promote is fail-closed without `BASE_BACKUP_ENDPOINT` + `SPACES_ACCESS_KEY_ID` / `SPACES_SECRET_ACCESS_KEY` (or AWS_* fallbacks) in GitHub. | diff --git a/docs/RELEARN.md b/docs/RELEARN.md index 028b80537..eda0f9936 100644 --- a/docs/RELEARN.md +++ b/docs/RELEARN.md @@ -42,16 +42,17 @@ one of these holds: `live_harvest_wired`, and `champion_baseline_recorded`, and every submit row echoes `eval_backend`, so a sim run is never mistaken for a real verdict. -`eval_image_digest` is **empty on purpose**. Every image built so far failed on -a rented pod, most recently `sha256:cbc4bbb8…` (`f3cfa69`), which exited 1 -without printing `RELEARN_EVAL_OK`: the CUDA scoring layer shipped no vLLM, fell -back to transformers, and then crashed because `Qwen3VLVideoProcessor` needs -torchvision. The next pin waits on a `CortexLM/relearn` image that carries -**vLLM and torchvision**. Do not guess a digest to clear the 503 — an unpinned -host refuses for free, a wrongly pinned one rents a B200 that cannot score. - -Even once the image is pinned, a live host must still wire the harvest and -record a champion baseline before `can_score` turns true; each has its own 503. +`eval_image_digest` is pinned to the CUDA scoring image +`sha256:4806db4bd6650415c3705290117fd0190cd2a6d387cef871dd3d9ff1a8bb6d7e` +(`CortexLM/relearn` `8ffbe8a`, +[publish-eval-image run 33730126042](https://github.com/CortexLM/relearn/actions/runs/33730126042)). +The runtime job pulled that digest, ran the harvest-PATH check, and proved +`import vllm, torchvision` plus `relearn-eval selftest` on those bytes. It is +**not** the slim contract image. Do not re-pin `sha256:cbc4bbb8…` (`f3cfa69`) +or the other named dead digests in `config/relearn-pin.toml`. + +A live host must still wire the harvest and record a champion baseline before +`can_score` turns true; each has its own 503. A refusal is not a submission: nothing is persisted unless scoring produced a verdict, so a 503 leaves no row behind. diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index 8cb5b2c87..07b92d47e 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -20,7 +20,7 @@ | `rejected` with `PerturbationEvidenceMissing` | No perturbed rerun in the eval document | Fail-closed, like an empty `manifest`. The brittleness floor is not skipped by omitting the series | | `rejected` with `BaseCanaryEvidenceMissing` | No known-answer canaries in the eval document | Fail-closed. The base-competence floor is not skipped by omitting the series | | `503` on submit | Holdout file missing or mismatched | Operator: `RELEARN_HOLDOUT_FILE` must match the pin commitment | -| `503 eval image digest not pinned` | Host has no `sha256:` eval image and did not opt into sim | Expected on `relearn` right now: no `relearn-eval` image has scored on a rented pod yet, so the digest is deliberately empty and the subnet refuses rather than rent hardware that cannot score. Image and Agent digests are pinned (`relearn` PR #3), so a 503 there is an operator pin/file mismatch. `GET /v1/status` shows `can_score: false` | +| `503 eval image digest not pinned` | Host has no `sha256:` eval image and did not opt into sim | Operator pin/file mismatch: `relearn-eval` is pinned (`sha256:4806db4b…`, `8ffbe8a`). Image and Agent digests are also pinned (`relearn` PR #3). `GET /v1/status` shows `can_score: false` | | `503 … no in-process sim` | Digest is pinned but the live harvest is not wired on that host | Operator issue; `/v1/status` shows `live_harvest_wired: false`. The control plane refuses to substitute sim numbers | | `503 backend: lium …` | The eval pod could not be rented, reached, or torn down | Transient. Retry; the run is not banked and no verdict was recorded | | `503 backend: RELEARN_TEACHER_API_URL not set …` | Operator has not configured the judge the eval image needs | Operator issue; `/v1/status` shows `can_score: false`. No pod was rented |