From c49bb2875a3df46ae1b3a81bc4c61d67038a9f12 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:21:20 +0000 Subject: [PATCH 01/15] feat(proof): admin probe for the topic-vm orchestrator wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /v1/admin/proof/vm-orchestrator (operator bearer, read-only, no VM, no spend) reports what proof-challenge resolved for the topic-VM orchestrator and whether the KVM-host agent answers, through the very client the runner drives: `ready()` (bearer file present, RLM image pinned — re-read now), the locked template, one agent health call, and the host's own gates (live_harvest_wired, registered_custom). A broken wire is data, not an error: bearer refused, agent unreachable, digest unpinned, and URL unset each show up by name so a staging operator can prove the wire without cargo on the droplet. Never the bearer value. proof-challenge resolves the orchestrator once per process and shares the Arc between the runner registry and the probe; the registry is still built only over a wired harvest, so log lines are unchanged. Tests: route is 401 / 503 auth_unconfigured / 200; unwired hosts name the env vars; against the in-process fake agent the report shows ready + fake hypervisor, then a bearer rotated on one side, a stopped agent, and an emptied bearer file, each as data. Co-authored-by: Mathis --- Cargo.lock | 2 + bins/proof-challenge/Cargo.toml | 2 + bins/proof-challenge/src/main.rs | 209 +++++++++++++++++++--- crates/proof-challenge/src/lib.rs | 5 +- crates/proof-http/Cargo.toml | 2 +- crates/proof-http/src/lib.rs | 283 ++++++++++++++++++++++++++++++ 6 files changed, 477 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eee09722b..a2d96a516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3487,6 +3487,7 @@ dependencies = [ name = "proof-challenge-bin" version = "0.1.0" dependencies = [ + "async-trait", "axum", "challenge-keys", "clap", @@ -3502,6 +3503,7 @@ dependencies = [ "proof-rlm-scorer", "proof-rlm-store", "proof-task", + "proof-vm-agent", "proof-vm-fc", "reqwest 0.12.28", "serde_json", diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index b66d44c5c..ec0887afe 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -13,6 +13,7 @@ name = "proof-challenge" path = "src/main.rs" [dependencies] +async-trait = "0.1" axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } challenge-keys = { path = "../../crates/challenge-keys" } clap = { version = "4", features = ["derive", "env"] } @@ -36,6 +37,7 @@ tracing = "0.1" crypto = { path = "../../crates/crypto" } hex = "0.4" proof-rlm = { path = "../../crates/proof-rlm", features = ["test-fixtures"] } +proof-vm-agent = { path = "../../crates/proof-vm-agent", features = ["test-fixtures"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } sha2 = "0.10" tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "process", "time"] } diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 82e58c3eb..7c4d442f9 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -17,13 +17,15 @@ use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::sync::Arc; +use async_trait::async_trait; use challenge_keys::load_challenge_secret; use clap::Parser; use prism_lium::LiumClient; use proof_challenge::{ executor_slot, hash_admin_token, parse_holdout_file, proof_router, AppState, BaselineMeasurement, EvalBackend, EvalExecutorOffer, HarvestOverrides, InferenceOffer, - LiveScorer, MemoryStore, ProofPin, TopicDocument, CHALLENGE_ID, SCORING_VERSION, + LiveScorer, MemoryStore, ProofPin, TopicDocument, VmAgentHealth, VmOrchestratorProbe, + VmOrchestratorReport, CHALLENGE_ID, SCORING_VERSION, }; use proof_eval::{custom_ids_ref, registered_custom, FamilyMux}; use proof_harvest::{HarvestLimits, LiumProofHarvest}; @@ -183,7 +185,10 @@ fn run(cli: &Cli) -> Result<(), String> { cli.proxy_model_dir.clone(), cli.holdout_store.clone(), ); - let live_scorer = live_scorer(backend, harvest, rlm_store, &cli.artefact_root); + // One topic-VM orchestrator per process: the runner registry drives it + // and the admin probe reports on the same client (bearer file, pin, CA). + let vm = Arc::new(topic_vm_orchestrator()); + let live_scorer = live_scorer(backend, harvest, rlm_store, &cli.artefact_root, &vm); log_live_wiring(backend, live_scorer.as_deref(), &cli.artefact_root); let store = MemoryStore::new(); @@ -210,6 +215,7 @@ fn run(cli: &Cli) -> Result<(), String> { executor: executor_slot(executor), judge_api_key, admin_hashes: Arc::new(load_admin_hashes(cli.admin_tokens_file.as_deref())), + vm_probe: Some(vm), epoch: 0, }; rt.block_on(serve(cli.bind, state)) @@ -312,11 +318,12 @@ fn live_scorer( harvest: Option>, rlm_store: Arc, artefact_root: &Path, + vm: &TopicVm, ) -> Option> { if backend != EvalBackend::Lium { return None; } - let custom = custom_family(rlm_store, artefact_root); + let custom = custom_family(rlm_store, artefact_root, vm); match harvest { Some(harvest) => Some(Arc::new( FamilyMux::new(harvest).with_custom_family(custom.scorer), @@ -343,14 +350,14 @@ struct CustomFamily { /// /// No benchmark, model, or repository is compiled in: the registry holds only /// the generic `VmBackedRunner`, under the custom ids the operator lists in -/// `PROOF_VM_RUNNER_CUSTOM_IDS`, over the topic-VM orchestrator -/// [`topic_vm_orchestrator`] resolved. With no ids the registry is empty and -/// every custom topic answers 503 (`RunnerUnwired`); with ids but an unwired -/// or unpinned orchestrator, 503 naming the missing env var. It never falls -/// back to the digest-pinned harvest and never spends. -fn custom_family(rlm_store: Arc, artefact_root: &Path) -> CustomFamily { - let vm = topic_vm_orchestrator(); - let registry = runner_registry(&vm); +/// `PROOF_VM_RUNNER_CUSTOM_IDS`, over the topic-VM orchestrator `vm` +/// ([`topic_vm_orchestrator`] resolved once per process). With no ids the +/// registry is empty and every custom topic answers 503 (`RunnerUnwired`); +/// with ids but an unwired or unpinned orchestrator, 503 naming the missing +/// env var. It never falls back to the digest-pinned harvest and never +/// spends. +fn custom_family(rlm_store: Arc, artefact_root: &Path, vm: &TopicVm) -> CustomFamily { + let registry = runner_registry(vm); let standalone = vm.live && !registry.is_empty(); let scorer = RlmScorer::new(Arc::new(registry), rlm_store) .with_artefacts(Some(ArtefactStore::new(artefact_root))); @@ -360,7 +367,10 @@ fn custom_family(rlm_store: Arc, artefact_root: &Path) -> CustomFa } } -/// What the topic-VM orchestrator env resolved to. +/// What the topic-VM orchestrator env resolved to. Resolved once per process +/// and shared: the runner registry drives `orchestrator`, and +/// `GET /v1/admin/proof/vm-orchestrator` reports through the same client +/// ([`VmOrchestratorProbe`]) — same bearer file, same pin, same TLS roots. struct TopicVm { orchestrator: Arc, /// RLM VM template the runner boots for topics without a VM. @@ -369,6 +379,11 @@ struct TopicVm { /// file env, https). False = `UnwiredVmOrchestrator`, which refuses /// every call. live: bool, + /// The live client, concretely, for the probe's agent health call. The + /// same allocation as `orchestrator`; `None` when unwired. + fc: Option>, + /// Why nothing is wired (names the env vars). Empty when live. + unwired_reason: String, } /// The topic-VM orchestrator this host talks to, plus the RLM VM template. @@ -397,35 +412,79 @@ fn topic_vm_orchestrator() -> TopicVm { topics answer 503 until fixed" ), } + let fc = Arc::new(fc); TopicVm { - orchestrator: Arc::new(fc), + orchestrator: fc.clone(), template, live: true, + fc: Some(fc), + unwired_reason: String::new(), } } Ok(None) => { - tracing::warn!( + let reason = format!( "no topic-vm orchestrator ({VM_ORCHESTRATOR_URL_ENV} / \ - {VM_ORCHESTRATOR_TOKEN_FILE_ENV} / {RLM_VM_IMAGE_DIGEST_ENV} unset); every custom \ - topic answers 503 and nothing runs on this host" + {VM_ORCHESTRATOR_TOKEN_FILE_ENV} / {RLM_VM_IMAGE_DIGEST_ENV} unset)" ); - TopicVm::unwired() - } - Err(e) => { tracing::warn!( - "topic-vm orchestrator refused ({e}); staying unwired, custom topics 503" + "{reason}; every custom topic answers 503 and nothing runs on this host" ); - TopicVm::unwired() + TopicVm::unwired(reason) + } + Err(e) => { + let reason = format!("topic-vm orchestrator refused ({e})"); + tracing::warn!("{reason}; staying unwired, custom topics 503"); + TopicVm::unwired(reason) } } } impl TopicVm { - fn unwired() -> Self { + fn unwired(reason: String) -> Self { Self { orchestrator: Arc::new(UnwiredVmOrchestrator), template: VmTemplate::from_env(), live: false, + fc: None, + unwired_reason: reason, + } + } +} + +#[async_trait] +impl VmOrchestratorProbe for TopicVm { + /// `ready()` (bearer file + pin, re-read now) and one agent health call + /// through the very client the runner uses — the same TLS roots, the same + /// bearer file — so a green answer here is the wire the runner will use. + /// Unwired hosts report the boot-time reason. Never the bearer. + async fn probe(&self) -> VmOrchestratorReport { + let Some(fc) = &self.fc else { + return VmOrchestratorReport::unwired( + &self.unwired_reason, + &self.template.image_digest, + ); + }; + let template = fc.template(); + let ready = fc.ready(); + let health = fc.health().await; + VmOrchestratorReport { + orchestrator: "firecracker".into(), + ready: ready.is_ok(), + reason: ready.err().map(|e| e.to_string()).unwrap_or_default(), + image_digest: template.image_digest.clone(), + vcpus: template.vcpus, + mem_mib: template.mem_mib, + agent: health.as_ref().ok().map(|h| VmAgentHealth { + api_version: h.api_version, + ready: h.ready, + reason: h.reason.clone(), + hypervisor: h.hypervisor.clone(), + vms: h.vms, + }), + agent_error: health.err().map(|e| e.to_string()), + live_harvest_wired: false, + custom_family_wired: false, + registered_custom: Vec::new(), } } } @@ -779,6 +838,7 @@ mod tests { Some(harvest), Arc::new(MemoryRlmStore::new()), &root, + &topic_vm_orchestrator(), ) .expect("a wired harvest is the live scorer"); assert!(mux.harvest_wired(), "the Lium harvest is the default route"); @@ -971,6 +1031,98 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// The admin probe reports through the very client the runner drives: an + /// unwired host names the env vars; a wired one shows `ready()` next to + /// one agent health call, and a bad bearer, a dead agent, or an emptied + /// bearer file each show up as data — never as a fallback, never as the + /// bearer itself. + #[test] + fn vm_orchestrator_probe_reports_ready_agent_bearer_and_outage_as_data() { + const TOKEN: &str = "probe-bearer-not-a-real-secret"; + let _guard = LIUM_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + clear_vm_env(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + + let unwired = topic_vm_orchestrator(); + let report = rt.block_on(unwired.probe()); + assert_eq!(report.orchestrator, "unwired"); + assert!(!report.ready); + assert!( + report.reason.contains(VM_ORCHESTRATOR_URL_ENV), + "{report:?}" + ); + assert!(report.agent.is_none() && report.agent_error.is_none()); + + // Two copies of the bearer, as on a real deployment: the agent's + // /etc/proof-vm/token and the CP's PROOF_VM_ORCHESTRATOR_TOKEN_FILE. + let agent_token = proof_vm_agent::fixtures::token_file("probe-agent", TOKEN); + let token = proof_vm_agent::fixtures::token_file("probe-cp", TOKEN); + let agent = rt.block_on(proof_vm_agent::fixtures::FakeAgent::serve( + proof_vm_agent::fixtures::FakeHypervisor::new(0.8), + &agent_token, + )); + std::env::set_var(VM_ORCHESTRATOR_URL_ENV, agent.url()); + std::env::set_var(VM_ORCHESTRATOR_TOKEN_FILE_ENV, &token); + std::env::set_var( + RLM_VM_IMAGE_DIGEST_ENV, + format!("sha256:{}", "ab".repeat(32)), + ); + let wired = topic_vm_orchestrator(); + assert!(wired.live && wired.fc.is_some()); + let report = rt.block_on(wired.probe()); + assert_eq!(report.orchestrator, "firecracker"); + assert!(report.ready, "{report:?}"); + assert_eq!((report.vcpus, report.mem_mib), (4, 8_192)); + let health = report.agent.as_ref().expect("agent answered"); + assert!(health.ready && health.hypervisor == "fake" && health.vms == 0); + assert_eq!(report.agent_error, None); + let dump = serde_json::to_string(&report).expect("json"); + assert!(!dump.contains(TOKEN), "bearer leaked: {dump}"); + + std::fs::write(&token, "another-bearer-not-a-real-secret\n").expect("rotate one side"); + let report = rt.block_on(wired.probe()); + assert!( + report.ready, + "a non-empty bearer file is ready on the client" + ); + assert!(report.agent.is_none()); + assert!( + report + .agent_error + .as_deref() + .is_some_and(|e| e.contains("refused the bearer")), + "{report:?}" + ); + + agent.stop(); + std::fs::write(&token, format!("{TOKEN}\n")).expect("restore"); + let report = rt.block_on(wired.probe()); + assert!( + report + .agent_error + .as_deref() + .is_some_and(|e| e.contains("unreachable")), + "{report:?}" + ); + + std::fs::write(&token, "\n").expect("empty"); + let report = rt.block_on(wired.probe()); + assert!(!report.ready); + assert!( + report.reason.contains(VM_ORCHESTRATOR_TOKEN_FILE_ENV), + "{report:?}" + ); + assert!(report.agent.is_none(), "no call without a bearer"); + clear_vm_env(); + let _ = std::fs::remove_file(&token); + let _ = std::fs::remove_file(&agent_token); + } + /// Full topic-VM env (https URL, bearer file, image pin, one custom id), /// **no** Lium credentials: the custom family stands on its own. The /// registry is non-empty, the mux passes the host-wide gate and the @@ -1007,6 +1159,7 @@ mod tests { harvest, Arc::new(MemoryRlmStore::new()), &root, + &topic_vm_orchestrator(), ) .expect("the custom family is wired from the topic-vm env alone"); assert!( @@ -1072,7 +1225,8 @@ mod tests { EvalBackend::Sim, None, Arc::new(MemoryRlmStore::new()), - &root + &root, + &topic_vm_orchestrator(), ) .is_none()); clear_vm_env(); @@ -1098,7 +1252,14 @@ mod tests { let store: Arc = Arc::new(MemoryRlmStore::new()); let none = |label: &str| { assert!( - live_scorer(EvalBackend::Lium, None, store.clone(), &root).is_none(), + live_scorer( + EvalBackend::Lium, + None, + store.clone(), + &root, + &topic_vm_orchestrator() + ) + .is_none(), "{label}: nothing may be wired" ); }; diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index 82a722dba..007c4b57d 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -23,7 +23,10 @@ pub use proof_executor::{ EvalExecutorOffer, ExecutorOfferError, HarvestOverrides, OfferStatus, EVAL_EXECUTOR_OFFER_FILE_ENV, }; -pub use proof_http::{executor_slot, hash_admin_token, proof_router, AppState, ExecutorSlot}; +pub use proof_http::{ + executor_slot, hash_admin_token, proof_router, AppState, ExecutorSlot, VmAgentHealth, + VmOrchestratorProbe, VmOrchestratorReport, +}; pub use proof_store::{ArtifactManifest, MemoryStore}; pub use proof_task::{ HoldoutRecord, InferenceOffer, OfferError, ProofPin, TopicDocument, BASE_MODEL_FAMILY, diff --git a/crates/proof-http/Cargo.toml b/crates/proof-http/Cargo.toml index cefd57a83..3749dd654 100644 --- a/crates/proof-http/Cargo.toml +++ b/crates/proof-http/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true publish = false [dependencies] +async-trait = "0.1" axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } hex = "0.4" proof-eval = { path = "../proof-eval" } @@ -22,7 +23,6 @@ sha2 = "0.10" tokio = { version = "1", features = ["macros", "rt", "sync"] } [dev-dependencies] -async-trait = "0.1" crypto = { path = "../crypto" } http-body-util = "0.1" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 24242d2d9..75b0fc0f3 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -11,6 +11,7 @@ //! GET /v1/submissions/{id} //! POST /v1/admin/proof/topics operator publish (signed document) //! POST /v1/admin/proof/executor operator rotate the live executor offer +//! GET /v1/admin/proof/vm-orchestrator operator probe: topic-VM orchestrator readiness + agent health //! ``` #![forbid(unsafe_code)] @@ -24,6 +25,7 @@ use std::sync::{Arc, PoisonError, RwLock}; +use async_trait::async_trait; use axum::extract::{Path, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; @@ -59,6 +61,92 @@ pub fn executor_slot(offer: Option) -> ExecutorSlot { Arc::new(RwLock::new(offer)) } +/// The KVM-host agent's health as the control plane saw it on one +/// `GET /v1/health` (mirrors `proof_vm_proto::AgentHealth` field for field). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmAgentHealth { + /// Wire version the agent speaks. + pub api_version: u32, + /// Whether the hypervisor could boot a VM right now. + pub ready: bool, + /// Why not (empty when ready). Never a secret. + pub reason: String, + /// Backend name (`firecracker`; `fake` only in tests). + pub hypervisor: String, + /// VMs currently bound on the host. + pub vms: usize, +} + +/// `GET /v1/admin/proof/vm-orchestrator` body: what this host resolved for +/// the topic-VM orchestrator and whether its agent answers. Operator data +/// behind the admin bearer — it may name env vars and container paths, never +/// the bearer, a key, or an origin the RLM could reach. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmOrchestratorReport { + /// `firecracker` (live client resolved at boot) or `unwired`. + pub orchestrator: String, + /// The client's own `ready()`: bearer file present and non-empty, RLM + /// image digest pinned. Checked per request, so a fix needs no restart. + pub ready: bool, + /// Why not ready (empty when ready). Names the env var to fix. + pub reason: String, + /// `sha256:` pin of the RLM VM image the client asks the agent to boot + /// (empty = unpinned = nothing ever boots). + pub image_digest: String, + /// RLM VM vCPUs (locked default 4). + pub vcpus: u32, + /// RLM VM memory in MiB (locked default 8192). + pub mem_mib: u32, + /// The agent's answer to one health call, when it answered. + pub agent: Option, + /// Why the agent did not answer: unreachable, bearer refused, not wired. + pub agent_error: Option, + /// Filled by the host: the digest-pinned Lium harvest (`nll` / + /// `throughput`) is wired. Lium only — informational for the custom + /// family, which is wired from the topic-VM env on its own. + #[serde(default)] + pub live_harvest_wired: bool, + /// Filled by the host: at least one custom id has a registered runner + /// (the custom family is routed, harvest or not). + #[serde(default)] + pub custom_family_wired: bool, + /// Filled by the host: custom ids with a registered runner. + #[serde(default)] + pub registered_custom: Vec, +} + +impl VmOrchestratorReport { + /// Report for a host that keeps `UnwiredVmOrchestrator`; `reason` names + /// the env vars a live one reads. `image_digest` is whatever pin the env + /// carries so "pinned but URL unset" is visible. + pub fn unwired(reason: &str, image_digest: &str) -> Self { + Self { + orchestrator: "unwired".into(), + ready: false, + reason: reason.trim().to_owned(), + image_digest: image_digest.trim().to_owned(), + vcpus: 0, + mem_mib: 0, + agent: None, + agent_error: None, + live_harvest_wired: false, + custom_family_wired: false, + registered_custom: Vec::new(), + } + } +} + +/// Operator diagnostic over the topic-VM orchestrator this host resolved at +/// boot. The binary implements it over the live `FirecrackerOrchestrator` +/// (its `ready()` plus one agent health call) or the unwired stand-in; the +/// route only adds what the host knows (harvest wired, registered ids). It +/// changes nothing and spends nothing. +#[async_trait] +pub trait VmOrchestratorProbe: Send + Sync { + /// Snapshot as of now (bearer file and pin re-read; one agent round trip). + async fn probe(&self) -> VmOrchestratorReport; +} + /// Shared HTTP state. #[derive(Clone)] pub struct AppState { @@ -82,6 +170,9 @@ pub struct AppState { pub judge_api_key: Option, /// Operator bearer hashes (sha256 hex). Empty → admin 503. pub admin_hashes: Arc>, + /// Topic-VM orchestrator diagnostic for `GET /v1/admin/proof/vm-orchestrator`. + /// `None` = the host resolved none (the route then reports `none`). + pub vm_probe: Option>, /// Chain epoch used for topic windows. v0 hosts pass 0. pub epoch: u64, } @@ -197,6 +288,10 @@ pub fn proof_router(state: AppState) -> Router { .route("/v1/submissions/{id}", get(get_sub)) .route("/v1/admin/proof/topics", post(publish_topic)) .route("/v1/admin/proof/executor", post(rotate_executor)) + .route( + "/v1/admin/proof/vm-orchestrator", + get(vm_orchestrator_probe), + ) .with_state(state) } @@ -758,6 +853,30 @@ async fn rotate_executor( )) } +/// Operator probe: is the topic-VM orchestrator wired, is its bearer file +/// and RLM image pin in place, and does the KVM-host agent answer? Same +/// bearer gate as the other admin routes; always 200 once authorised (a +/// broken wire is data, not an error). Read-only, no VM, no spend. +async fn vm_orchestrator_probe( + State(st): State, + headers: HeaderMap, +) -> Result)> { + if st.admin_hashes.is_empty() { + return Err(err(StatusCode::SERVICE_UNAVAILABLE, "auth_unconfigured")); + } + if !admin_ok(&headers, &st.admin_hashes) { + return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); + } + let mut report = match &st.vm_probe { + Some(probe) => probe.probe().await, + None => VmOrchestratorReport::unwired("no topic-vm orchestrator resolved on this host", ""), + }; + report.live_harvest_wired = st.live_harvest_wired(); + report.registered_custom = st.registered_custom(); + report.custom_family_wired = !report.registered_custom.is_empty(); + Ok(Json(report)) +} + fn admin_ok(headers: &HeaderMap, hashes: &[String]) -> bool { let Some(raw) = headers .get(axum::http::header::AUTHORIZATION) @@ -1074,6 +1193,7 @@ mod tests { // Testeur blocker. Sim does not call the judge, so it stays None. judge_api_key, admin_hashes: Arc::new(vec![hash_admin_token(token)]), + vm_probe: None, epoch: 0, }) } @@ -1118,6 +1238,7 @@ mod tests { executor: executor_slot(executor), judge_api_key: Some("test-judge-key".into()), admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }) } @@ -1424,6 +1545,162 @@ mod tests { assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); } + /// What the binary hands the route on a wired host: a canned snapshot. + struct StubProbe(VmOrchestratorReport); + + #[async_trait] + impl VmOrchestratorProbe for StubProbe { + async fn probe(&self) -> VmOrchestratorReport { + self.0.clone() + } + } + + fn app_with_probe(probe: Option>, admin: bool) -> Router { + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + proof_router(AppState { + store: MemoryStore::new(), + pin: p, + backend: EvalBackend::Lium, + live_scorer: Some(Arc::new(StubScorer::win())), + offer: Some(offer()), + executor: executor_slot(None), + judge_api_key: Some("test-judge-key".into()), + admin_hashes: Arc::new(if admin { + vec![hash_admin_token("op")] + } else { + Vec::new() + }), + vm_probe: probe, + epoch: 0, + }) + } + + /// The operator probe sits behind the admin bearer, is always 200 once + /// authorised (a broken wire is data), reports the host's own gates next + /// to the client's snapshot, and never carries a bearer value. + #[tokio::test] + async fn admin_vm_orchestrator_probe_is_bearer_gated_and_reports_the_wire() { + let none = app_with_probe(None, true); + let (st, body) = json_req( + none.clone(), + "GET", + "/v1/admin/proof/vm-orchestrator", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED, "{body}"); + let (st, body) = json_req( + none.clone(), + "GET", + "/v1/admin/proof/vm-orchestrator", + serde_json::json!({}), + Some("wrong"), + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED, "{body}"); + let (st, body) = json_req( + app_with_probe(None, false), + "GET", + "/v1/admin/proof/vm-orchestrator", + serde_json::json!({}), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!(body["error"], "auth_unconfigured"); + + let (st, body) = json_req( + none, + "GET", + "/v1/admin/proof/vm-orchestrator", + serde_json::json!({}), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["orchestrator"], "unwired", "{body}"); + assert_eq!(body["ready"], false); + assert_eq!(body["live_harvest_wired"], true, "{body}"); + assert_eq!(body["registered_custom"], serde_json::json!([]), "{body}"); + + let mut wired = VmOrchestratorReport::unwired("", &format!("sha256:{}", "cd".repeat(32))); + wired.orchestrator = "firecracker".into(); + wired.ready = true; + wired.vcpus = 4; + wired.mem_mib = 8_192; + wired.agent = Some(VmAgentHealth { + api_version: 1, + ready: true, + reason: String::new(), + hypervisor: "firecracker".into(), + vms: 2, + }); + // The probe's own view of the host gates is overwritten by the route. + wired.live_harvest_wired = false; + wired.custom_family_wired = true; + wired.registered_custom = vec!["stale".into()]; + let (st, body) = json_req( + app_with_probe(Some(Arc::new(StubProbe(wired))), true), + "GET", + "/v1/admin/proof/vm-orchestrator", + serde_json::json!({}), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + let report: VmOrchestratorReport = serde_json::from_value(body.clone()).expect("typed"); + assert_eq!(report.orchestrator, "firecracker"); + assert!(report.ready); + assert_eq!((report.vcpus, report.mem_mib), (4, 8_192)); + assert_eq!( + report.agent.as_ref().map(|a| a.hypervisor.as_str()), + Some("firecracker") + ); + assert_eq!(report.agent.as_ref().map(|a| a.vms), Some(2)); + assert!( + report.live_harvest_wired, + "Lium-only host gate, not the probe's copy" + ); + assert!( + report.registered_custom.is_empty(), + "host registry, not the probe's copy" + ); + assert!( + !report.custom_family_wired, + "follows the host registry, not the probe's copy" + ); + let dump = body.to_string(); + for forbidden in ["Bearer ", "\"token\"", "api_key"] { + assert!(!dump.contains(forbidden), "{forbidden} in {dump}"); + } + + let (st, body) = json_req( + app_with_probe( + Some(Arc::new(StubProbe(VmOrchestratorReport::unwired( + "PROOF_VM_ORCHESTRATOR_TOKEN_FILE (/run/base/proof/vm_orchestrator_token) missing or empty", + "", + )))), + true, + ), + "GET", + "/v1/admin/proof/vm-orchestrator", + serde_json::json!({}), + Some("op"), + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["ready"], false); + assert!( + body["reason"] + .as_str() + .unwrap_or_default() + .contains("PROOF_VM_ORCHESTRATOR_TOKEN_FILE"), + "{body}" + ); + assert_eq!(body["image_digest"], "", "unpinned stays visibly empty"); + } + #[tokio::test] async fn topic_pinning_another_executor_commitment_is_503() { let p = pin(&format!("sha256:{}", "ab".repeat(32))); @@ -1449,6 +1726,7 @@ mod tests { executor: executor_slot(Some(test_executor(&p))), judge_api_key: Some("test-judge-key".into()), admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }); let (st, body) = json_req( @@ -2057,6 +2335,7 @@ mod tests { executor: executor_slot(Some(executor)), judge_api_key: Some("test-judge-key".into()), admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }) } @@ -2347,6 +2626,7 @@ mod tests { executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }) } @@ -2436,6 +2716,7 @@ mod tests { executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }); let (st, body) = json_req( @@ -2477,6 +2758,7 @@ mod tests { executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }) } @@ -2685,6 +2967,7 @@ mod tests { executor: executor_slot(None), judge_api_key: None, admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }) } From 87ffb961a32291681d3d374c79e62d355ba62856 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:38:15 +0000 Subject: [PATCH 02/15] feat(deploy): proof-vm-wire-check.sh staging harness + fake-agent test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator harness for the Proof topic-VM wire on the staging master droplet: bash + curl + python3, no cargo, bearer only ever in a 0600 curl config, production hosts refused. env CP env file: https URL (loopback http = WARN), bearer file non-empty via --path-map (compose bind mount), RLM image pin is sha256:<64 hex> (empty = FAIL, never invented), optional CA is PEM, custom ids well-formed, locked 4/8192 shape, PROOF_FORCE_SIM off agent GET /v1/health with the bearer (ready / reason / hypervisor / vms); no bearer and wrong bearer → 401 cp /v1/status gates (lium, live_harvest_wired, registered_custom ⊇ ids), no URL / token / path leak, /v1/proof/topics holdout leak, executor readiness, and the admin vm-orchestrator probe (the CP's own rustls client: ready, agent health, agent_error) boot-probe create → attach → 409 → 409 topic_mismatch → destroy → 404 for ONE RLM VM (no job, no spend); Ctrl-C tears the VM down submit-probe POST /v1/submissions on a custom topic asserting the fail-closed code + reason; 2xx refused without --allow-live-run; a scored row must carry the sister-measured flops_used matrix the fail-closed flips with the expected 503 reasons Integration test (proof-vm-fc, fake agent on loopback, skipped without bash/curl/python3): env / agent / boot-probe speak the router's JSON, the bearer never appears in the output, one boot + one Destroy, a dead agent fails the check; unpinned digest and an emptied bearer file fail closed. Co-authored-by: Mathis --- crates/proof-vm-fc/tests/wire_check_script.rs | 205 ++++++ deploy/scripts/proof-vm-wire-check.sh | 617 ++++++++++++++++++ 2 files changed, 822 insertions(+) create mode 100644 crates/proof-vm-fc/tests/wire_check_script.rs create mode 100755 deploy/scripts/proof-vm-wire-check.sh diff --git a/crates/proof-vm-fc/tests/wire_check_script.rs b/crates/proof-vm-fc/tests/wire_check_script.rs new file mode 100644 index 000000000..873a3775e --- /dev/null +++ b/crates/proof-vm-fc/tests/wire_check_script.rs @@ -0,0 +1,205 @@ +//! The operator harness `deploy/scripts/proof-vm-wire-check.sh` against the +//! in-process fake agent: the `agent` and `boot-probe` subcommands must speak +//! the JSON the agent router speaks (create / attach / 409 / teardown / 404) +//! and never leak the bearer. No Firecracker, no VM, loopback only — what CI +//! runs. Skipped where bash, curl, or python3 are missing. + +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use proof_rlm::fixtures::pinned_template; +use proof_rlm::RetainPolicy; +use proof_vm_agent::fixtures::{token_file, FakeAgent, FakeHypervisor}; + +const TOKEN: &str = "wire-check-script-bearer-not-a-real-secret"; + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .expect("workspace root") +} + +fn tools_present() -> bool { + ["bash", "curl", "python3"].iter().all(|tool| { + Command::new("sh") + .args(["-c", &format!("command -v {tool}")]) + .output() + .is_ok_and(|o| o.status.success()) + }) +} + +/// A CP env file for the script plus the "container" secrets dir it maps to. +fn cp_env(tag: &str, agent_url: &str, digest: &str) -> (PathBuf, PathBuf) { + let dir = + std::env::temp_dir().join(format!("proof-vm-wire-script-{}-{tag}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let secrets = dir.join("secrets"); + std::fs::create_dir_all(&secrets).expect("secrets dir"); + std::fs::write(secrets.join("vm_orchestrator_token"), format!("{TOKEN}\n")).expect("token"); + let env = dir.join("proof-challenge.env"); + std::fs::write( + &env, + format!( + "# test env\nPROOF_VM_ORCHESTRATOR_URL={agent_url}\n\ + PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/base/proof/vm_orchestrator_token\n\ + PROOF_RLM_VM_IMAGE_DIGEST={digest}\n\ + PROOF_VM_RUNNER_CUSTOM_IDS=wire_metric\n" + ), + ) + .expect("env"); + (env, secrets) +} + +fn run_script(args: &[&str]) -> (bool, String) { + let out = Command::new("bash") + .arg(repo_root().join("deploy/scripts/proof-vm-wire-check.sh")) + .args(args) + .current_dir(repo_root()) + .output() + .expect("run script"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out.status.success(), text) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn agent_and_boot_probe_speak_the_router_json_and_never_print_the_bearer() { + if !tools_present() { + eprintln!("skipping: bash / curl / python3 not all present"); + return; + } + let agent_token = token_file("wire-script", TOKEN); + let agent = FakeAgent::serve(FakeHypervisor::new(0.8), &agent_token).await; + let digest = pinned_template().image_digest; + let (env, secrets) = cp_env("ok", &agent.url(), &digest); + let env_s = env.to_string_lossy().into_owned(); + let map = format!("/run/base/proof={}", secrets.display()); + + let (ok, text) = tokio::task::spawn_blocking({ + let env_s = env_s.clone(); + let map = map.clone(); + move || run_script(&["env", "--env-file", &env_s, "--path-map", &map]) + }) + .await + .expect("join"); + assert!(ok, "env check failed:\n{text}"); + assert!(text.contains("bearer file present and non-empty"), "{text}"); + assert!(text.contains("sha256 pin"), "{text}"); + assert!(!text.contains(TOKEN), "bearer printed:\n{text}"); + + let (ok, text) = tokio::task::spawn_blocking({ + let env_s = env_s.clone(); + let map = map.clone(); + move || run_script(&["agent", "--env-file", &env_s, "--path-map", &map]) + }) + .await + .expect("join"); + assert!(ok, "agent check failed:\n{text}"); + assert!( + text.contains("agent ready (hypervisor=fake vms=0)"), + "{text}" + ); + assert!(text.contains("no bearer → 401"), "{text}"); + assert!(text.contains("wrong bearer → 401"), "{text}"); + assert!(!text.contains(TOKEN), "bearer printed:\n{text}"); + + let (ok, text) = tokio::task::spawn_blocking({ + let env_s = env_s.clone(); + let map = map.clone(); + move || { + run_script(&[ + "boot-probe", + "--env-file", + &env_s, + "--path-map", + &map, + "--probe-topic", + "wire-probe-script", + ]) + } + }) + .await + .expect("join"); + assert!(ok, "boot-probe failed:\n{text}"); + for line in [ + "create bound vm wire-probe-script-", + "agent booted the pinned image", + "attach returns the same vm", + "second create for the topic → 409 already_exists", + "teardown naming another topic → 409 topic_mismatch", + "teardown destroyed wire-probe-script-", + "attach after destroy → 404", + ] { + assert!(text.contains(line), "missing {line:?} in:\n{text}"); + } + assert!(!text.contains(TOKEN), "bearer printed:\n{text}"); + let hv = &agent.hypervisor; + assert_eq!( + hv.boots().len(), + 1, + "one VM booted, the 409 create booted none" + ); + assert_eq!(hv.boots()[0].topic_id, "wire-probe-script"); + assert_eq!(hv.boots()[0].image_digest, digest); + assert_eq!(hv.teardowns().len(), 1); + assert_eq!(hv.teardowns()[0].1, RetainPolicy::Destroy); + assert!( + agent.state.running().await.is_empty(), + "nothing outlives the probe" + ); + + // A stopped agent is reported, not swallowed. + agent.stop(); + let (ok, text) = tokio::task::spawn_blocking(move || { + run_script(&["agent", "--env-file", &env_s, "--path-map", &map]) + }) + .await + .expect("join"); + assert!(!ok, "a dead agent must fail the check:\n{text}"); + assert!(text.contains("agent health → HTTP 000"), "{text}"); + let _ = std::fs::remove_dir_all(env.parent().expect("dir")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn env_check_fails_closed_on_unpinned_digest_and_empty_bearer() { + if !tools_present() { + eprintln!("skipping: bash / curl / python3 not all present"); + return; + } + let (env, secrets) = cp_env("unpinned", "https://kvm.example.invalid:8200", ""); + let env_s = env.to_string_lossy().into_owned(); + let map = format!("/run/base/proof={}", secrets.display()); + let (ok, text) = tokio::task::spawn_blocking({ + let env_s = env_s.clone(); + let map = map.clone(); + move || run_script(&["env", "--env-file", &env_s, "--path-map", &map]) + }) + .await + .expect("join"); + assert!(!ok, "unpinned digest must fail:\n{text}"); + assert!(text.contains("PROOF_RLM_VM_IMAGE_DIGEST unset"), "{text}"); + assert!(text.contains("DO NOT INVENT ONE"), "{text}"); + assert!( + text.contains("PROOF_VM_ORCHESTRATOR_URL is https"), + "{text}" + ); + + std::fs::write(secrets.join("vm_orchestrator_token"), "\n").expect("empty token"); + let (ok, text) = tokio::task::spawn_blocking(move || { + run_script(&["env", "--env-file", &env_s, "--path-map", &map]) + }) + .await + .expect("join"); + assert!(!ok, "{text}"); + assert!( + text.contains("is empty → ready() 503 naming PROOF_VM_ORCHESTRATOR_TOKEN_FILE"), + "{text}" + ); + let _ = std::fs::remove_dir_all(env.parent().expect("dir")); +} diff --git a/deploy/scripts/proof-vm-wire-check.sh b/deploy/scripts/proof-vm-wire-check.sh new file mode 100755 index 000000000..ab933dded --- /dev/null +++ b/deploy/scripts/proof-vm-wire-check.sh @@ -0,0 +1,617 @@ +#!/usr/bin/env bash +# Proof topic-VM orchestrator — staging wire check + end-to-end probes. +# +# Runs on the staging MASTER droplet (/opt/base) or any box that reaches the +# Proof control plane and the KVM-host agent. bash + curl + python3 only, no +# cargo. Never prints the bearer (it travels in a 0600 curl config, never in +# argv). Refuses production hosts. Runbook: +# docs/runbooks/proof-vm-orchestrator.md § DigitalOcean staging. +# +# Usage: +# proof-vm-wire-check.sh env # CP env file: https URL, non-empty bearer file, sha256 pin, CA, ids +# proof-vm-wire-check.sh agent # KVM-host agent: health with the bearer; no / wrong bearer → 401 +# proof-vm-wire-check.sh cp # control plane: /v1/status gates, no leaks, admin vm-orchestrator probe +# proof-vm-wire-check.sh all # env + agent + cp +# proof-vm-wire-check.sh boot-probe # create → attach → destroy ONE RLM VM on the KVM host (no job, no spend) +# proof-vm-wire-check.sh submit-probe --topic ID --expect 503 [--reason SUBSTR] +# # POST /v1/submissions on a custom topic; assert the fail-closed answer +# proof-vm-wire-check.sh submit-probe --topic ID --expect 201 --allow-live-run --artifact-uri URI +# # happy path: real RLM job + sister guest; prints flops_used from the row +# proof-vm-wire-check.sh matrix # print the fail-closed matrix as operator steps + expected 503 reasons +# +# Options (all subcommands): +# --env-file F CP env (default deploy/env/proof-challenge.env; process env wins when set) +# --path-map FROM=TO container → host path (default /run/base/proof=deploy/secrets/proof; repeatable) +# --cp URL Proof origin (default: first reachable of the loopback candidates) +# --admin-token-file F operator bearer for /v1/admin/* (default deploy/secrets/proof/admin_tokens, first line) +# --probe-topic ID boot-probe topic id (default wire-probe-; slug, never a real topic) +# --topic ID submit-probe topic id (must be an open custom topic) +# --expect CODE submit-probe expected HTTP status (400 / 503; 2xx needs --allow-live-run) +# --reason SUBSTR submit-probe: the error text must contain this +# --artifact-uri URI submit-probe locator (default https://example.invalid/wire-probe.tar — never fetchable) +# --wait SECS submit-probe --expect 201: how long the synchronous POST may take (default 900) +# +# Exit: 0 all PASS, 1 any FAIL, 2 refused (production host / unsafe request). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +RED() { printf '\033[31m%s\033[0m\n' "$*"; } +GRN() { printf '\033[32m%s\033[0m\n' "$*"; } +YEL() { printf '\033[33m%s\033[0m\n' "$*"; } +LOG() { printf '[wire-check] %s\n' "$*"; } + +FAILS=0 +WARNS=0 +pass() { GRN "PASS $*"; } +warn() { YEL "WARN $*"; WARNS=$((WARNS + 1)); } +fail() { RED "FAIL $*"; FAILS=$((FAILS + 1)); } + +PROD_HOSTS='network\.cortex\.foundation|chain\.joinbase\.ai|api\.cortex\.foundation' +refuse_prod() { + if printf '%s' "$1" | grep -Eq "$PROD_HOSTS"; then + RED "refusing production host: $1" + exit 2 + fi +} + +# --------------------------------------------------------------------------- +# arguments +# --------------------------------------------------------------------------- +ENV_FILE="deploy/env/proof-challenge.env" +PATH_MAPS=() +CP="" +ADMIN_TOKEN_FILE="deploy/secrets/proof/admin_tokens" +PROBE_TOPIC="" +TOPIC="" +EXPECT="" +REASON="" +ALLOW_LIVE_RUN=0 +ARTIFACT_URI="https://example.invalid/wire-probe.tar" +WAIT_SECS=900 + +usage() { sed -n '2,34p' "$0"; } + +[[ $# -ge 1 ]] || { usage; exit 1; } +SUBCOMMAND="$1"; shift +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="${2:?}"; shift 2 ;; + --path-map) PATH_MAPS+=("${2:?}"); shift 2 ;; + --cp) CP="${2:?}"; shift 2 ;; + --admin-token-file) ADMIN_TOKEN_FILE="${2:?}"; shift 2 ;; + --probe-topic) PROBE_TOPIC="${2:?}"; shift 2 ;; + --topic) TOPIC="${2:?}"; shift 2 ;; + --expect) EXPECT="${2:?}"; shift 2 ;; + --reason) REASON="${2:?}"; shift 2 ;; + --allow-live-run) ALLOW_LIVE_RUN=1; shift ;; + --artifact-uri) ARTIFACT_URI="${2:?}"; shift 2 ;; + --wait) WAIT_SECS="${2:?}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) RED "unknown arg: $1"; usage; exit 1 ;; + esac +done +[[ ${#PATH_MAPS[@]} -gt 0 ]] || PATH_MAPS=("/run/base/proof=$ROOT/deploy/secrets/proof") + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- +# jget JSON dotted.path → raw scalar, compact JSON for objects/arrays, "" when absent. +jget() { + printf '%s' "$1" | python3 -c ' +import json, sys +path = sys.argv[1] +try: + v = json.load(sys.stdin) +except Exception: + print(""); sys.exit(0) +for k in [p for p in path.split(".") if p]: + if isinstance(v, list): + try: v = v[int(k)] + except Exception: v = None + elif isinstance(v, dict): + v = v.get(k) + else: + v = None + if v is None: break +if v is None: print("") +elif isinstance(v, bool): print("true" if v else "false") +elif isinstance(v, (dict, list)): print(json.dumps(v, separators=(",", ":"))) +else: print(v) +' "$2" +} + +# Value from the process env when set, else from the env file (last KEY= wins; +# commented lines ignored; surrounding quotes stripped). Empty = unset. +cfg() { + local name="$1" val="${!1:-}" line + if [[ -z "$val" && -f "$ENV_FILE" ]]; then + line="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${name}=" "$ENV_FILE" | tail -n1 || true)" + val="${line#*=}" + val="${val%\"}"; val="${val#\"}"; val="${val%\'}"; val="${val#\'}" + fi + printf '%s' "$val" +} + +# Container path → host path through --path-map (compose bind mounts). +map_path() { + local p="$1" m from to + for m in "${PATH_MAPS[@]}"; do + from="${m%%=*}"; to="${m#*=}" + if [[ "$p" == "$from" || "$p" == "$from/"* ]]; then + printf '%s' "${to}${p#"$from"}" + return 0 + fi + done + printf '%s' "$p" +} + +TMPDIR_WC="$(mktemp -d)" +umask 077 + +# A boot-probe VM must never outlive the probe, even on Ctrl-C. +PROBE_BASE=""; PROBE_VM=""; PROBE_TOPIC_LIVE=""; PROBE_HDR="" +AGENT_ARGS=() +on_exit() { + if [[ -n "$PROBE_VM" ]]; then + RED "boot-probe interrupted with vm $PROBE_VM up; destroying" + curl -sS -m 660 -o /dev/null -X DELETE -H 'content-type: application/json' \ + --data-binary "$(printf '{"topic_id":"%s","policy":"destroy"}' "$PROBE_TOPIC_LIVE")" \ + "${AGENT_ARGS[@]}" -K "$PROBE_HDR" "$PROBE_BASE/v1/vms/$PROBE_VM" || true + fi + rm -rf "$TMPDIR_WC" +} +trap on_exit EXIT + +# curl config carrying the bearer (mode 0600; never in argv, never printed). +bearer_config() { # bearer_config FILE_WITH_TOKEN → path of a curl -K config + local tok cfg_path="$TMPDIR_WC/hdr.$RANDOM$RANDOM" + tok="$(tr -d '[:space:]' < "$1")" + printf 'header = "Authorization: Bearer %s"\n' "$tok" > "$cfg_path" + printf '%s' "$cfg_path" +} + +# http METHOD URL BODY_JSON [CURL_ARGS...] → $HTTP_CODE ("000" = no answer) and $HTTP_BODY. +# Default timeout 30s; a later `-m N` in CURL_ARGS wins. Never fails the script. +HTTP_CODE="" +HTTP_BODY="" +http() { + local method="$1" url="$2" body="$3" out="$TMPDIR_WC/body.$RANDOM$RANDOM" + shift 3 + local -a data=() + [[ -n "$body" ]] && data=(-H 'content-type: application/json' --data-binary "$body") + HTTP_CODE="$(curl -sS -m 30 -o "$out" -w '%{http_code}' -X "$method" "${data[@]}" "$@" "$url" 2>"$TMPDIR_WC/err" || true)" + HTTP_BODY="$(cat "$out" 2>/dev/null || true)" + if [[ -z "$HTTP_CODE" || "$HTTP_CODE" == "000" ]]; then + HTTP_BODY="$(cat "$TMPDIR_WC/err" 2>/dev/null || true)" + HTTP_CODE="000" + fi + rm -f "$out" +} + +# --------------------------------------------------------------------------- +# env: the control plane's side of the wire +# --------------------------------------------------------------------------- +URL=""; TOKEN_PATH=""; CA_PATH=""; DIGEST=""; IDS="" +load_env() { + URL="$(cfg PROOF_VM_ORCHESTRATOR_URL)" + TOKEN_PATH="$(cfg PROOF_VM_ORCHESTRATOR_TOKEN_FILE)" + CA_PATH="$(cfg PROOF_VM_ORCHESTRATOR_CA_FILE)" + DIGEST="$(cfg PROOF_RLM_VM_IMAGE_DIGEST)" + IDS="$(cfg PROOF_VM_RUNNER_CUSTOM_IDS)" + [[ -n "$TOKEN_PATH" ]] && TOKEN_PATH="$(map_path "$TOKEN_PATH")" + [[ -n "$CA_PATH" ]] && CA_PATH="$(map_path "$CA_PATH")" + AGENT_ARGS=() + [[ -n "$CA_PATH" && -f "$CA_PATH" ]] && AGENT_ARGS+=(--cacert "$CA_PATH") + return 0 +} + +split_ids() { # split_ids "a, b,c" → one id per line, trimmed, blanks dropped + printf '%s\n' "$1" | tr ',' '\n' | sed -e 's/[[:space:]]//g' -e '/^$/d' +} + +check_env() { + LOG "env: $ENV_FILE (process env overrides; container paths mapped: ${PATH_MAPS[*]})" + load_env + if [[ -z "$URL" ]]; then + fail "PROOF_VM_ORCHESTRATOR_URL unset → proof-challenge keeps UnwiredVmOrchestrator (every custom topic 503)" + else + refuse_prod "$URL" + case "$URL" in + https://*) pass "PROOF_VM_ORCHESTRATOR_URL is https ($URL)" ;; + http://127.0.0.1*|http://localhost*|http://\[::1\]*) warn "PROOF_VM_ORCHESTRATOR_URL is plain http on loopback (tests / local TLS terminator only)" ;; + http://*) fail "PROOF_VM_ORCHESTRATOR_URL is plain http off loopback → refused at boot, host stays unwired" ;; + *) fail "PROOF_VM_ORCHESTRATOR_URL is not a URL: $URL" ;; + esac + fi + + local raw_token + raw_token="$(cfg PROOF_VM_ORCHESTRATOR_TOKEN_FILE)" + if [[ -z "$raw_token" ]]; then + fail "PROOF_VM_ORCHESTRATOR_TOKEN_FILE unset (the bearer is a FILE, never a value) → boot refuses, host stays unwired" + elif [[ ! -f "$TOKEN_PATH" ]]; then + fail "bearer file $raw_token → $TOKEN_PATH missing on this host → ready() 503 naming PROOF_VM_ORCHESTRATOR_TOKEN_FILE" + elif [[ -z "$(tr -d '[:space:]' < "$TOKEN_PATH")" ]]; then + fail "bearer file $TOKEN_PATH is empty → ready() 503 naming PROOF_VM_ORCHESTRATOR_TOKEN_FILE" + else + pass "bearer file present and non-empty ($raw_token → $TOKEN_PATH; contents not shown)" + local mode owner + mode="$(stat -c '%a' "$TOKEN_PATH" 2>/dev/null || echo '?')" + owner="$(stat -c '%u' "$TOKEN_PATH" 2>/dev/null || echo '?')" + [[ "$mode" == "400" || "$mode" == "600" ]] || warn "bearer file mode is $mode (want 0400)" + [[ "$owner" == "65532" || "$owner" == "?" ]] || warn "bearer file owner uid $owner (proof-challenge reads it as uid 65532)" + fi + + if [[ -z "$DIGEST" ]]; then + fail "PROOF_RLM_VM_IMAGE_DIGEST unset → unpinned → ready() 503; nothing ever boots. Take sha256sum of the RLM rootfs staged on the KVM host; DO NOT INVENT ONE" + elif [[ "$DIGEST" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then + pass "PROOF_RLM_VM_IMAGE_DIGEST is a sha256 pin (${DIGEST:0:19}…); the agent needs images/sha256-.ext4" + else + fail "PROOF_RLM_VM_IMAGE_DIGEST is not sha256:<64 hex>: $DIGEST" + fi + + local raw_ca + raw_ca="$(cfg PROOF_VM_ORCHESTRATOR_CA_FILE)" + if [[ -z "$raw_ca" ]]; then + LOG "PROOF_VM_ORCHESTRATOR_CA_FILE unset: the agent certificate must chain to a public root" + elif [[ ! -f "$CA_PATH" ]]; then + fail "CA file $raw_ca → $CA_PATH missing (boot refuses the client, host stays unwired)" + elif ! grep -q 'BEGIN CERTIFICATE' "$CA_PATH"; then + fail "CA file $CA_PATH is not PEM" + else + pass "CA file present ($raw_ca → $CA_PATH). rustls needs a SAN on the agent cert (CN alone is refused)" + fi + + if [[ -z "$IDS" ]]; then + warn "PROOF_VM_RUNNER_CUSTOM_IDS unset → empty registry → every custom topic 503 (registration is an operator action)" + else + local id bad=0 + while IFS= read -r id; do + [[ "$id" =~ ^[a-z0-9][a-z0-9_-]{1,63}$ ]] || { fail "custom id '$id' is malformed (skipped at boot): want [a-z0-9][a-z0-9_-]{1,63}"; bad=1; } + done < <(split_ids "$IDS") + [[ "$bad" -eq 0 ]] && pass "PROOF_VM_RUNNER_CUSTOM_IDS well-formed: $IDS" + fi + + local vcpus mem + vcpus="$(cfg PROOF_RLM_VM_VCPUS)"; mem="$(cfg PROOF_RLM_VM_MEM_MIB)" + [[ -z "$vcpus" || "$vcpus" == "4" ]] || warn "PROOF_RLM_VM_VCPUS=$vcpus deviates from the locked 4" + [[ -z "$mem" || "$mem" == "8192" ]] || warn "PROOF_RLM_VM_MEM_MIB=$mem deviates from the locked 8192" + if [[ "$(cfg PROOF_FORCE_SIM)" =~ ^(1|true|TRUE|yes)$ ]]; then + fail "PROOF_FORCE_SIM is on: sim never hosts staging/prod scoring (assert-compose-matrix.sh refuses it too)" + fi +} + +# --------------------------------------------------------------------------- +# agent: the KVM-host side, through curl (the CP-side rustls path is `cp`) +# --------------------------------------------------------------------------- +check_agent() { + load_env + if [[ -z "$URL" || -z "$TOKEN_PATH" || ! -s "$TOKEN_PATH" ]]; then + fail "agent probe needs PROOF_VM_ORCHESTRATOR_URL and a non-empty bearer file (run: $0 env)" + return 0 + fi + refuse_prod "$URL" + local base="${URL%/}" code hdr + hdr="$(bearer_config "$TOKEN_PATH")" + LOG "agent: GET $base/v1/health (bearer from $TOKEN_PATH${CA_PATH:+, --cacert $CA_PATH})" + + http GET "$base/v1/health" "" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" + if [[ "$code" != "200" ]]; then + fail "agent health → HTTP $code: $(printf '%s' "$HTTP_BODY" | head -c 300)" + else + local api ready reason hv vms + api="$(jget "$HTTP_BODY" api_version)"; ready="$(jget "$HTTP_BODY" ready)" + reason="$(jget "$HTTP_BODY" reason)"; hv="$(jget "$HTTP_BODY" hypervisor)"; vms="$(jget "$HTTP_BODY" vms)" + [[ "$api" == "1" ]] || fail "agent api_version '$api' (this tree speaks 1)" + if [[ "$ready" == "true" ]]; then + pass "agent ready (hypervisor=$hv vms=$vms)" + else + fail "agent not ready: $reason (firecracker + jailer + /dev/kvm + images on the KVM host)" + fi + [[ "$hv" == "firecracker" ]] || warn "agent hypervisor is '$hv', not firecracker (fake is CI-only)" + fi + + http GET "$base/v1/health" "" "${AGENT_ARGS[@]}"; code="$HTTP_CODE" + if [[ "$code" == "401" ]]; then pass "no bearer → 401 (agent fail-closed)"; else fail "no bearer → HTTP $code (want 401)"; fi + http GET "$base/v1/health" "" "${AGENT_ARGS[@]}" -H 'Authorization: Bearer wire-check-wrong-bearer-not-a-secret'; code="$HTTP_CODE" + if [[ "$code" == "401" ]]; then pass "wrong bearer → 401 (agent fail-closed)"; else fail "wrong bearer → HTTP $code (want 401)"; fi + LOG "curl accepting the certificate is not proof the Rust client does (rustls wants a SAN): '$0 cp' runs the CP-side probe" +} + +# --------------------------------------------------------------------------- +# cp: the control plane's view — status gates, leaks, admin vm-orchestrator probe +# --------------------------------------------------------------------------- +resolve_cp() { + if [[ -z "$CP" ]]; then + local cand + for cand in http://127.0.0.1:8080/challenge/proof http://127.0.0.1:28100 http://127.0.0.1:8100; do + if curl -fsS -m 3 "$cand/health" >/dev/null 2>&1; then CP="$cand"; break; fi + done + fi + [[ -n "$CP" ]] || { fail "no reachable Proof origin (pass --cp URL)"; return 1; } + CP="${CP%/}" + refuse_prod "$CP" +} + +admin_config() { # → curl -K path, or "" when no operator bearer is available + if [[ -n "${PROOF_ADMIN_TOKEN:-}" ]]; then + printf '%s' "$PROOF_ADMIN_TOKEN" > "$TMPDIR_WC/admin_tok" + bearer_config "$TMPDIR_WC/admin_tok" + elif [[ -s "$ADMIN_TOKEN_FILE" ]]; then + grep -m1 -v '^[[:space:]]*$' "$ADMIN_TOKEN_FILE" > "$TMPDIR_WC/admin_tok" + bearer_config "$TMPDIR_WC/admin_tok" + fi +} + +check_cp() { + load_env + resolve_cp || return 0 + LOG "cp: $CP" + local code + http GET "$CP/health" ""; code="$HTTP_CODE" + if [[ "$code" == "200" && "$(jget "$HTTP_BODY" challenge_id)" == "proof" ]]; then + pass "GET /health is proof" + else + fail "GET /health → $code $HTTP_BODY" + fi + + http GET "$CP/v1/status" ""; code="$HTTP_CODE" + if [[ "$code" != "200" ]]; then + fail "GET /v1/status → $code" + return 0 + fi + local status="$HTTP_BODY" harvest can_score registered + harvest="$(jget "$status" live_harvest_wired)"; can_score="$(jget "$status" can_score)" + registered="$(jget "$status" registered_custom)" + LOG "status: eval_backend=$(jget "$status" eval_backend) live_harvest_wired=$harvest can_score=$can_score baseline_sealed=$(jget "$status" baseline_sealed)" + LOG "status: open_topics=$(jget "$status" open_topics) scorable_topics=$(jget "$status" scorable_topics) registered_custom=$registered" + [[ "$(jget "$status" eval_backend)" == "lium" ]] || fail "eval_backend is not lium (sim never hosts staging scoring)" + if [[ "$harvest" == "true" ]]; then + pass "live_harvest_wired: the custom family is routed (LIUM_API_KEY + LIUM_SSH_PUBLIC_KEY_FILE present)" + else + fail "live_harvest_wired=false: the custom family is not routed and registered_custom stays [] whatever PROOF_VM_RUNNER_CUSTOM_IDS says" + fi + local leak leaked=0 + for leak in "vm_orchestrator" "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/run/base" "Bearer " "PROOF_VM_ORCHESTRATOR"; do + if printf '%s' "$status" | grep -qF "$leak"; then fail "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/v1/status leaks '$leak'"; leaked=1; fi + done + if [[ -n "$URL" ]]; then + local host="${URL#*://}"; host="${host%%/*}"; host="${host%%:*}" + if printf '%s' "$status" | grep -qF "$host"; then fail "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/v1/status leaks the agent host $host"; leaked=1; fi + fi + [[ "$leaked" -eq 0 ]] && pass "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/v1/status carries no orchestrator URL, token, or path" + if [[ -n "$IDS" ]]; then + local id missing=0 + while IFS= read -r id; do + printf '%s' "$registered" | grep -qF "\"$id\"" || { fail "registered_custom lacks $id (PROOF_VM_RUNNER_CUSTOM_IDS says it is served)"; missing=1; } + done < <(split_ids "$IDS") + [[ "$missing" -eq 0 ]] && pass "registered_custom lists every id in PROOF_VM_RUNNER_CUSTOM_IDS" + fi + + http GET "$CP/v1/proof/topics" ""; code="$HTTP_CODE" + if [[ "$code" != "200" ]]; then + fail "GET /v1/proof/topics → $code" + elif printf '%s' "$HTTP_BODY" | grep -q 'content_sha256'; then + fail "/v1/proof/topics leaks holdout records" + else + pass "/v1/proof/topics leaks no holdout" + fi + http GET "$CP/v1/proof/executor" ""; code="$HTTP_CODE" + if [[ "$code" == "200" ]]; then + LOG "executor: ready=$(jget "$HTTP_BODY" ready) reason='$(jget "$HTTP_BODY" reason)'" + else + fail "GET /v1/proof/executor → $code" + fi + + local hdr + hdr="$(admin_config)" + if [[ -z "$hdr" ]]; then + warn "no operator bearer (PROOF_ADMIN_TOKEN or --admin-token-file): skipping GET /v1/admin/proof/vm-orchestrator" + return 0 + fi + LOG "admin probe: GET $CP/v1/admin/proof/vm-orchestrator (the CP's own client: bearer file, CA, rustls)" + http GET "$CP/v1/admin/proof/vm-orchestrator" "" -K "$hdr" -m 40; code="$HTTP_CODE" + if [[ "$code" != "200" ]]; then + fail "admin probe → HTTP $code $(printf '%s' "$HTTP_BODY" | head -c 200) (401 = wrong operator bearer; 503 auth_unconfigured = no PROOF_ADMIN_TOKENS_FILE; 404 = proof-challenge predates this route)" + return 0 + fi + local rep="$HTTP_BODY" orch ready reason a_ready a_reason a_hv + orch="$(jget "$rep" orchestrator)"; ready="$(jget "$rep" ready)"; reason="$(jget "$rep" reason)" + a_ready="$(jget "$rep" agent.ready)"; a_reason="$(jget "$rep" agent.reason)"; a_hv="$(jget "$rep" agent.hypervisor)" + LOG "admin probe: orchestrator=$orch ready=$ready image=$(jget "$rep" image_digest | head -c 19)… shape=$(jget "$rep" vcpus)vCPU/$(jget "$rep" mem_mib)MiB agent=$(jget "$rep" agent) agent_error=$(jget "$rep" agent_error)" + if [[ "$orch" == "firecracker" ]]; then pass "CP resolved FirecrackerOrchestrator"; else fail "CP orchestrator is '$orch': $reason"; fi + if [[ "$ready" == "true" ]]; then pass "CP ready(): bearer file + RLM image pin in place"; else fail "CP not ready: $reason"; fi + if [[ -n "$(jget "$rep" agent)" ]]; then + if [[ "$a_ready" == "true" ]]; then pass "agent answered the CP's client: ready (hypervisor=$a_hv)"; else fail "agent answered but not ready: $a_reason"; fi + [[ "$a_hv" == "firecracker" ]] || warn "agent hypervisor '$a_hv' is not firecracker" + else + fail "agent did not answer the CP's client: $(jget "$rep" agent_error)" + fi + if printf '%s' "$rep" | grep -qiE 'authorization|bearer [a-z0-9]'; then fail "admin probe body carries a bearer"; fi +} + +# --------------------------------------------------------------------------- +# boot-probe: create → attach → destroy one RLM VM (no job, no spend) +# --------------------------------------------------------------------------- +boot_probe() { + load_env + if [[ -z "$URL" || ! -s "$TOKEN_PATH" || ! "$DIGEST" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then + fail "boot-probe needs URL + non-empty bearer file + sha256 pin (run: $0 env)" + return 0 + fi + refuse_prod "$URL" + local base="${URL%/}" topic="${PROBE_TOPIC:-wire-probe-$(date +%s)}" hdr code vm_id + [[ "$topic" =~ ^[a-z0-9][a-z0-9-]{1,62}$ ]] || { fail "probe topic '$topic' is not a slug"; return 0; } + local vcpus mem + vcpus="$(cfg PROOF_RLM_VM_VCPUS)"; mem="$(cfg PROOF_RLM_VM_MEM_MIB)" + vcpus="${vcpus:-4}"; mem="${mem:-8192}" + hdr="$(bearer_config "$TOKEN_PATH")" + LOG "boot-probe: POST $base/v1/vms topic=$topic image=${DIGEST:0:19}… ${vcpus}vCPU/${mem}MiB (boots a real RLM VM; up to 10 min)" + + http GET "$base/v1/vms/by-topic/$topic" "" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" + [[ "$code" == "404" ]] || { fail "topic $topic already has a VM or attach failed (HTTP $code): $HTTP_BODY"; return 0; } + + local spec + spec="$(printf '{"spec":{"topic_id":"%s","template":{"image_digest":"%s","vcpus":%s,"mem_mib":%s},"sandbox":{"firecracker_required":true,"deadline_s":60},"retain":"destroy"}}' \ + "$topic" "$DIGEST" "$vcpus" "$mem")" + http POST "$base/v1/vms" "$spec" "${AGENT_ARGS[@]}" -K "$hdr" -m 660; code="$HTTP_CODE" + if [[ "$code" != "201" ]]; then + fail "create → HTTP $code: $(printf '%s' "$HTTP_BODY" | head -c 400) (503 not_ready = image/kernel/kvm on the host; 400 bad_spec; 401 bearer)" + return 0 + fi + vm_id="$(jget "$HTTP_BODY" handle.vm_id)" + PROBE_BASE="$base"; PROBE_VM="$vm_id"; PROBE_TOPIC_LIVE="$topic"; PROBE_HDR="$hdr" + if [[ "$(jget "$HTTP_BODY" handle.topic_id)" == "$topic" ]]; then pass "create bound vm $vm_id to $topic"; else fail "create bound another topic: $HTTP_BODY"; fi + if [[ "$(jget "$HTTP_BODY" image_digest | tr '[:upper:]' '[:lower:]')" == "$(printf '%s' "$DIGEST" | tr '[:upper:]' '[:lower:]')" ]]; then + pass "agent booted the pinned image" + else + fail "agent booted $(jget "$HTTP_BODY" image_digest), pinned $DIGEST" + fi + [[ "$(jget "$HTTP_BODY" state)" == "running" ]] || fail "state after create: $(jget "$HTTP_BODY" state)" + + http GET "$base/v1/vms/by-topic/$topic" "" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" + if [[ "$code" == "200" && "$(jget "$HTTP_BODY" handle.vm_id)" == "$vm_id" ]]; then + pass "attach returns the same vm (one topic ↔ one VM)" + else + fail "attach → HTTP $code $HTTP_BODY" + fi + http POST "$base/v1/vms" "$spec" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" + if [[ "$code" == "409" ]]; then pass "second create for the topic → 409 already_exists"; else fail "second create → HTTP $code (want 409): $HTTP_BODY"; fi + local wrong + wrong="$(printf '{"topic_id":"%s-other","policy":"destroy"}' "$topic")" + http DELETE "$base/v1/vms/$vm_id" "$wrong" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" + if [[ "$code" == "409" ]]; then pass "teardown naming another topic → 409 topic_mismatch"; else fail "teardown for another topic → HTTP $code (want 409): $HTTP_BODY"; fi + + local body + body="$(printf '{"topic_id":"%s","policy":"destroy"}' "$topic")" + http DELETE "$base/v1/vms/$vm_id" "$body" "${AGENT_ARGS[@]}" -K "$hdr" -m 660; code="$HTTP_CODE" + if [[ "$code" == "200" && "$(jget "$HTTP_BODY" state)" == "destroyed" && "$(jget "$HTTP_BODY" confirmed)" == "true" ]]; then + pass "teardown destroyed $vm_id (confirmed)" + PROBE_VM="" + else + fail "teardown → HTTP $code $HTTP_BODY — check the KVM host: /srv/jailer/firecracker/$vm_id, nft list tables" + fi + http GET "$base/v1/vms/by-topic/$topic" "" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" + if [[ "$code" == "404" ]]; then pass "attach after destroy → 404 (nothing left for $topic)"; else fail "attach after destroy → HTTP $code $HTTP_BODY"; fi + LOG "on the KVM host: journalctl -u proof-vm-orchestrator | grep -E 'topic vm booted|torn down'; ls /srv/jailer/firecracker/ must not list $vm_id" +} + +# --------------------------------------------------------------------------- +# submit-probe: POST /v1/submissions on a custom topic, assert the answer +# --------------------------------------------------------------------------- +submit_probe() { + [[ -n "$TOPIC" && -n "$EXPECT" ]] || { RED "submit-probe needs --topic ID --expect CODE"; exit 1; } + if [[ "$EXPECT" =~ ^2 && "$ALLOW_LIVE_RUN" -ne 1 ]]; then + RED "refusing: --expect $EXPECT means a real RLM job (topic VM + sister guest + paid inference). Pass --allow-live-run and a fetchable --artifact-uri." + exit 2 + fi + resolve_cp || return 0 + local hotkey hex body code + hotkey="$(head -c 64 /dev/zero | tr '\0' 'a')" + hex="$(printf '%s' "wire-probe-$TOPIC-$(date +%s)-$$-$RANDOM" | sha256sum | awk '{print $1}')" + body="$(printf '{"miner_hotkey":"%s","artifact_digest":"%s","artifact_uri":"%s","claim":"proof-vm-wire-check probe","declared_flops":1,"topic_id":"%s","manifest":{"train_dataset_ids":["wire-probe-v0"]}}' \ + "$hotkey" "$hex" "$ARTIFACT_URI" "$TOPIC")" + LOG "submit-probe: POST $CP/v1/submissions topic=$TOPIC expect=$EXPECT${REASON:+ reason~'$REASON'}" + http POST "$CP/v1/submissions" "$body" -m "$WAIT_SECS"; code="$HTTP_CODE" + LOG "→ HTTP $code $(printf '%s' "$HTTP_BODY" | head -c 500)" + if [[ "$code" != "$EXPECT" ]]; then + fail "expected HTTP $EXPECT, got $code" + return 0 + fi + if [[ "$code" =~ ^[45] ]]; then + local error + error="$(jget "$HTTP_BODY" error)" + if [[ -n "$error" ]]; then pass "HTTP $code carries an explicit error (fail-closed, no silent empty)"; else fail "HTTP $code with no error field"; fi + if [[ -n "$REASON" ]]; then + if printf '%s' "$error" | grep -qF "$REASON"; then pass "error names '$REASON'"; else fail "error does not name '$REASON': $error"; fi + fi + return 0 + fi + local id state + id="$(jget "$HTTP_BODY" id)" + [[ "$id" == pf_* ]] || { fail "2xx without a pf_ id"; return 0; } + state="$(jget "$HTTP_BODY" state)" + pass "submission scored synchronously: $id (state=$state)" + http GET "$CP/v1/submissions/$id" ""; code="$HTTP_CODE" + [[ "$code" == "200" ]] || { fail "GET /v1/submissions/$id → $code"; return 0; } + local flops pass_flag + flops="$(jget "$HTTP_BODY" verdict.agent.flops_used)"; pass_flag="$(jget "$HTTP_BODY" verdict.pass)" + LOG "row $id: state=$state pass=$pass_flag flops_used=$flops detail=$(jget "$HTTP_BODY" detail) failed=$(jget "$HTTP_BODY" verdict.failed)" + if [[ "$state" == "awaiting_admin" && -n "$flops" && "$flops" != "0" ]]; then + pass "verdict carries the sister-measured flops_used=$flops (host-stamped, never RLM-authored)" + else + fail "no host-measured flops_used on a scored row (a report without a measurement is 503, never a substituted number)" + fi + LOG "sandboxed=true lives in the artefact: docker compose cp proof-challenge:/var/lib/proof/artefacts/$TOPIC/$id.zip /tmp/ && unzip -p /tmp/$id.zip report.json" + LOG "on the KVM host: journalctl -u proof-vm-orchestrator | grep -E 'sister guest booting|sister guest run attested|jail released'" +} + +# --------------------------------------------------------------------------- +# matrix: the fail-closed table as operator steps +# --------------------------------------------------------------------------- +print_matrix() { + load_env + local cp="${CP:-http://127.0.0.1:8080/challenge/proof}" topic="${TOPIC:-}" + local token_host + token_host="$(map_path "${TOKEN_PATH:-/run/base/proof/vm_orchestrator_token}")" + cat < $token_host +$0 submit-probe --cp $cp --topic $topic --expect 503 --reason PROOF_VM_ORCHESTRATOR_TOKEN_FILE +# restore the bearer bytes (same as /etc/proof-vm/token on the KVM host), mode 0400, uid 65532. +# +# 3. Bearer bytes differ from the agent's (no restart) → agent 401 → CP 503. +head -c 32 /dev/urandom | base64 -w0 > $token_host +$0 submit-probe --cp $cp --topic $topic --expect 503 --reason 'refused the bearer' +# restore the bearer bytes. +# +# 4. RLM image digest unpinned (restart required: the pin is read at boot). +# edit $ENV_FILE: PROOF_RLM_VM_IMAGE_DIGEST= ; docker compose ... up -d proof-challenge +$0 submit-probe --cp $cp --topic $topic --expect 503 --reason PROOF_RLM_VM_IMAGE_DIGEST +# restore the digest (sha256sum of the staged rootfs; never invented), restart. +# +# 5. Agent down (on the KVM host: systemctl stop proof-vm-orchestrator). +$0 submit-probe --cp $cp --topic $topic --expect 503 --reason 'orchestrator unreachable' +# systemctl start proof-vm-orchestrator; then: $0 agent +# +# 6. Unknown / closed topic → 400 (no row); a custom topic without artifact_uri → 400. +$0 submit-probe --cp $cp --topic does-not-exist --expect 400 +# +# After every row: $0 cp (the admin probe shows the same root cause: ready / reason / agent_error) +EOF +} + +# --------------------------------------------------------------------------- +summary() { + echo + if [[ "$FAILS" -gt 0 ]]; then + RED "proof-vm-wire-check: $FAILS FAIL, $WARNS WARN" + exit 1 + fi + GRN "proof-vm-wire-check: all PASS ($WARNS WARN)" +} + +case "$SUBCOMMAND" in + env) check_env; summary ;; + agent) check_agent; summary ;; + cp) check_cp; summary ;; + all) check_env; check_agent; check_cp; summary ;; + boot-probe) boot_probe; summary ;; + submit-probe) submit_probe; summary ;; + matrix) print_matrix ;; + -h|--help|help) usage ;; + *) RED "unknown subcommand: $SUBCOMMAND"; usage; exit 1 ;; +esac From 563450d7940067f5fc04db53baee261f23f7555e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:38:38 +0000 Subject: [PATCH 03/15] fix(proof-vm-fc): keep the agent url out of the unreachable error `orchestrator unreachable (...)` travels into the miner-facing 503 body when the KVM-host agent is down; reqwest's Display would print the agent's URL with it. Keep the method + route, strip the URL (`Error::without_url`). Test asserts the route stays and the address does not. Co-authored-by: Mathis --- crates/proof-vm-fc/src/lib.rs | 12 ++++++++---- crates/proof-vm-fc/tests/live_agent.rs | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/proof-vm-fc/src/lib.rs b/crates/proof-vm-fc/src/lib.rs index 9b34c0dc3..8a4f7de37 100644 --- a/crates/proof-vm-fc/src/lib.rs +++ b/crates/proof-vm-fc/src/lib.rs @@ -284,10 +284,14 @@ impl FirecrackerOrchestrator { if let Some(b) = body { req = req.json(b); } - let resp = req - .send() - .await - .map_err(|e| backend(format!("orchestrator unreachable ({method} {path}): {e}")))?; + // The error travels into a miner-facing 503: keep the route, drop the + // agent URL reqwest would otherwise print. + let resp = req.send().await.map_err(|e| { + backend(format!( + "orchestrator unreachable ({method} {path}): {}", + e.without_url() + )) + })?; let status = resp.status(); if status == StatusCode::NOT_FOUND { return Ok(None); diff --git a/crates/proof-vm-fc/tests/live_agent.rs b/crates/proof-vm-fc/tests/live_agent.rs index 5ba6fddc4..7f281e0b0 100644 --- a/crates/proof-vm-fc/tests/live_agent.rs +++ b/crates/proof-vm-fc/tests/live_agent.rs @@ -181,6 +181,15 @@ async fn a_wrong_bearer_or_a_dead_agent_is_a_backend_refusal_without_the_token() .expect_err("agent down is not None"); assert!(matches!(err, VmError::Backend(_)), "{err}"); assert!(err.to_string().contains("unreachable"), "{err}"); + // The text reaches a miner as a 503: the route, never the agent's address. + assert!( + err.to_string().contains("/v1/vms/by-topic/topic-a"), + "{err}" + ); + assert!( + !err.to_string().contains(&dead.addr.port().to_string()), + "agent url leaked: {err}" + ); } #[tokio::test] From f61d92d3bc5116a5cc5c80393fcc182062f08ec2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:40:45 +0000 Subject: [PATCH 04/15] docs(deploy): staging env overlays for the proof topic-vm wire deploy/env/proof-challenge.staging-vm.example: the client-side keys to append to proof-challenge.env on the staging master (https agent URL on the VPC, bearer file, private CA, RLM image pin, locked 4/8192 shape, custom ids, admin tokens file). deploy/env/proof-vm-orchestrator.staging.example: /etc/proof-vm/ orchestrator.env for the dedicated KVM host serving staging (private bind + TLS, bearer file, kernel + sister pins, sizes, egress allowlist for the judge origin / artefact hosts / resolver). Placeholders only, and every REPLACE_WITH_* value fails closed as written: the wire check flags each one, proof-challenge stays unwired or answers 503, the agent refuses to boot. No digest is real. Co-authored-by: Mathis --- deploy/env/proof-challenge.staging-vm.example | 58 +++++++++++++ .../env/proof-vm-orchestrator.staging.example | 81 +++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 deploy/env/proof-challenge.staging-vm.example create mode 100644 deploy/env/proof-vm-orchestrator.staging.example diff --git a/deploy/env/proof-challenge.staging-vm.example b/deploy/env/proof-challenge.staging-vm.example new file mode 100644 index 000000000..90de135e6 --- /dev/null +++ b/deploy/env/proof-challenge.staging-vm.example @@ -0,0 +1,58 @@ +# operator-managed, never committed with values filled in. +# +# STAGING overlay for deploy/env/proof-challenge.env on the staging MASTER +# droplet (`base-staging`, compose role-master + env-staging): the CLIENT side +# of the Proof topic-VM orchestrator. Nothing Firecracker runs on this +# droplet; proof-challenge only talks HTTPS to `proof-vm-orchestrator` on the +# dedicated KVM host (deploy/env/proof-vm-orchestrator.staging.example). +# +# Use: append these keys to the age-encrypted source of proof-challenge.env, +# re-materialize, restart proof-challenge, then +# ./deploy/scripts/proof-vm-wire-check.sh all +# Runbook: docs/runbooks/proof-vm-orchestrator.md § DigitalOcean staging. +# +# Every REPLACE_WITH_* value is a placeholder that FAILS CLOSED as written +# (not https / not 64 hex / not a custom id → unwired or 503). No digest in +# this file is real: take PROOF_RLM_VM_IMAGE_DIGEST from `sha256sum` of the +# RLM rootfs you staged on the KVM host. Do not invent one. + +# HTTPS URL of the agent, reachable from the staging VPC / private network +# only (never a public address, never this droplet). Plain http:// off +# loopback is refused at boot and the host stays unwired. +PROOF_VM_ORCHESTRATOR_URL=https://REPLACE_WITH_KVM_HOST_PRIVATE_ADDRESS:8200 + +# Bearer FILE, container path. Host file: deploy/secrets/proof/vm_orchestrator_token +# (mode 0400, uid 65532), same bytes as /etc/proof-vm/token on the KVM host. +# Re-read per request: rotate by rewriting both files, no restart. +PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/base/proof/vm_orchestrator_token + +# PEM root the agent certificate chains to (staging uses a private CA). +# Remove the line when the certificate chains to a public root. The +# certificate MUST carry a SAN matching the host in the URL: the CP's rustls +# client refuses CN-only certificates even when curl accepts them. +PROOF_VM_ORCHESTRATOR_CA_FILE=/run/base/proof/vm_orchestrator_ca.pem + +# sha256: pin of the RLM rootfs the agent boots, staged on the KVM host as +# PROOF_VM_AGENT_IMAGE_DIR/sha256-.ext4. Empty or malformed = unpinned = +# ready() 503 naming this variable; nothing ever boots. +PROOF_RLM_VM_IMAGE_DIGEST=sha256:REPLACE_WITH_SHA256SUM_OF_THE_STAGED_RLM_ROOTFS + +# Locked RLM VM shape (4 vCPU / 8192 MiB). Leave as is. +PROOF_RLM_VM_VCPUS=4 +PROOF_RLM_VM_MEM_MIB=8192 + +# Custom metric ids the generic VmBackedRunner serves on this host: exactly +# the ids named by the signed staging topics (comma-separated). Registration +# is an operator action — empty = empty registry = every custom topic 503. +# Also requires live_harvest_wired (LIUM_API_KEY + LIUM_SSH_PUBLIC_KEY_FILE): +# without the harvest the custom family is not routed at all. +PROOF_VM_RUNNER_CUSTOM_IDS=REPLACE_WITH_CUSTOM_ID_FROM_THE_SIGNED_STAGING_TOPIC + +# Operator bearers for /v1/admin/* (topics, executor, vm-orchestrator probe). +# The wire check reads the first line of deploy/secrets/proof/admin_tokens. +PROOF_ADMIN_TOKENS_FILE=/run/base/proof/admin_tokens + +# Presence-probed at awaiting_owner_keys only. The MATERIAL the RLM uses is +# staged into the topic VM by the KVM-host agent from its own +# PROOF_VM_AGENT_OWNER_KEY_DIR; this droplet never reads or sends it. +# PROOF_RLM_OWNER_INFERENCE_KEY_FILE=/run/base/proof/rlm_owner_inference_key diff --git a/deploy/env/proof-vm-orchestrator.staging.example b/deploy/env/proof-vm-orchestrator.staging.example new file mode 100644 index 000000000..6095fe454 --- /dev/null +++ b/deploy/env/proof-vm-orchestrator.staging.example @@ -0,0 +1,81 @@ +# operator-managed, never committed with values filled in. +# +# STAGING /etc/proof-vm/orchestrator.env for the DEDICATED KVM HOST that serves +# the staging master (deploy/systemd/proof-vm-orchestrator.service). This is +# not a compose file and never lands on a droplet: the unit needs /dev/kvm +# (ConditionPathExists) and DigitalOcean Droplets expose no nested KVM. Use a +# bare-metal / dedicated KVM host reachable from the staging VPC or a private +# network (WireGuard / peering) and bind the agent on that private address. +# Generic reference with every knob: deploy/env/proof-vm-orchestrator.env.example. +# +# Every REPLACE_WITH_* value FAILS CLOSED as written: a malformed kernel or +# sister pin exits 1 at boot, a non-loopback bind without TLS exits 1. The +# pins are `sha256sum` of the files YOU staged — never copied, never invented. +# Nothing in this file is a secret value: the bearer is a FILE re-read per +# request, owner key material is a DIRECTORY staged over vsock. + +# Listen on the private (VPC / WireGuard) address only; the staging master's +# PROOF_VM_ORCHESTRATOR_URL points here over HTTPS. +PROOF_VM_AGENT_BIND=REPLACE_WITH_KVM_HOST_PRIVATE_IP:8200 +# Certificate with a SAN for that address / name (rustls on the CP refuses +# CN-only). Private CA → its root goes to the CP as vm_orchestrator_ca.pem. +PROOF_VM_AGENT_TLS_CERT=/etc/proof-vm/tls.crt +PROOF_VM_AGENT_TLS_KEY=/etc/proof-vm/tls.key + +# Bearer file (0400 root); same bytes as the master's +# deploy/secrets/proof/vm_orchestrator_token. Missing/empty = every request 401. +PROOF_VM_AGENT_TOKEN_FILE=/etc/proof-vm/token + +# Firecracker + jailer of the SAME release, statically linked (musl). +PROOF_VM_AGENT_FIRECRACKER_BIN=/usr/local/bin/firecracker +PROOF_VM_AGENT_JAILER_BIN=/usr/local/bin/jailer +PROOF_VM_AGENT_CHROOT_BASE=/srv/jailer + +# Rootfs images: sha256-<64 hex>.ext4, one per pin. The RLM image the CP pins +# (PROOF_RLM_VM_IMAGE_DIGEST) must be present here; the agent re-hashes it +# before the first boot. +PROOF_VM_AGENT_IMAGE_DIR=/var/lib/proof-vm/images + +# Guest kernel and its pin. REQUIRED. `sha256sum /var/lib/proof-vm/vmlinux`. +PROOF_VM_AGENT_KERNEL=/var/lib/proof-vm/vmlinux +PROOF_VM_AGENT_KERNEL_DIGEST=sha256:REPLACE_WITH_SHA256SUM_OF_THE_STAGED_VMLINUX + +# Sister (miner) guest rootfs pin, present in PROOF_VM_AGENT_IMAGE_DIR. +# REQUIRED. No network, vsock miner protocol, reports flops_used. +PROOF_VM_AGENT_SISTER_IMAGE_DIGEST=sha256:REPLACE_WITH_SHA256SUM_OF_THE_STAGED_SISTER_ROOTFS + +# uid/gid the jailer drops Firecracker to. +PROOF_VM_AGENT_JAIL_UID=65534 +PROOF_VM_AGENT_JAIL_GID=65534 + +# Sizes. The RLM VM shape (4 vCPU / 8192 MiB) is set by the CP; the sister +# is sized here, never by the RLM. Staging host budget: one RLM VM + one +# sister per open topic → 6 vCPU / 12 GiB RAM / ~10 GiB scratch per topic. +PROOF_VM_AGENT_SCRATCH_MIB=8192 +PROOF_VM_AGENT_SISTER_VCPUS=2 +PROOF_VM_AGENT_SISTER_MEM_MIB=4096 +PROOF_VM_AGENT_SISTER_SCRATCH_MIB=2048 + +# Timeouts (seconds). Guest hello budget; grace past a job's topic deadline +# before the host kills the sister; jobs without a deadline of their own. +PROOF_VM_AGENT_BOOT_TIMEOUT_SECS=120 +PROOF_VM_AGENT_DEADLINE_GRACE_SECS=30 +PROOF_VM_AGENT_DEFAULT_JOB_TIMEOUT_SECS=3600 + +# Owner paid-inference key material staged into the RLM VM over vsock (files +# by name, 0400 root). Optional; the CP never sees these bytes. +# PROOF_VM_AGENT_OWNER_KEY_DIR=/etc/proof-vm/owner-keys + +# Uplink the RLM VMs masquerade through (the interface that reaches the judge +# origin and the artefact hosts — usually the public one, not the VPC one). +PROOF_VM_AGENT_UPLINK=REPLACE_WITH_UPLINK_INTERFACE +# First /30 of the host<->guest pool; must not overlap the VPC range. +PROOF_VM_AGENT_NET_BASE=172.16.0.0 +# Egress the RLM VM may reach and nothing else: CIDR[:port[/tcp|udp]], +# comma-separated. Staging: the RLM judge InferenceOffer origin, the artefact +# locator hosts miners use, and a resolver. EMPTY = NO EGRESS. IPv4 CIDRs +# only — resolve hostnames yourself. The sister guest never has a NIC. +PROOF_VM_AGENT_EGRESS_ALLOW=REPLACE_WITH_JUDGE_ORIGIN_IP/32:443,REPLACE_WITH_ARTEFACT_HOST_IP/32:443,REPLACE_WITH_RESOLVER_IP/32:53/udp + +# Where `retain` teardowns move a topic's jail (scratch, console, config). +PROOF_VM_AGENT_RETAIN_DIR=/var/lib/proof-vm/retained From 0001b859c7b3ad370f91d8bc92bcc00d118aa406 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:46:53 +0000 Subject: [PATCH 05/15] feat(deploy): wire-check submit-probe --no-artifact-uri for the 400 row Co-authored-by: Mathis --- deploy/scripts/proof-vm-wire-check.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/deploy/scripts/proof-vm-wire-check.sh b/deploy/scripts/proof-vm-wire-check.sh index ab933dded..d0efb7181 100755 --- a/deploy/scripts/proof-vm-wire-check.sh +++ b/deploy/scripts/proof-vm-wire-check.sh @@ -29,6 +29,7 @@ # --expect CODE submit-probe expected HTTP status (400 / 503; 2xx needs --allow-live-run) # --reason SUBSTR submit-probe: the error text must contain this # --artifact-uri URI submit-probe locator (default https://example.invalid/wire-probe.tar — never fetchable) +# --no-artifact-uri submit-probe without a locator (a custom topic must answer 400, no row) # --wait SECS submit-probe --expect 201: how long the synchronous POST may take (default 900) # # Exit: 0 all PASS, 1 any FAIL, 2 refused (production host / unsafe request). @@ -71,7 +72,7 @@ ALLOW_LIVE_RUN=0 ARTIFACT_URI="https://example.invalid/wire-probe.tar" WAIT_SECS=900 -usage() { sed -n '2,34p' "$0"; } +usage() { sed -n '2,35p' "$0"; } [[ $# -ge 1 ]] || { usage; exit 1; } SUBCOMMAND="$1"; shift @@ -87,6 +88,7 @@ while [[ $# -gt 0 ]]; do --reason) REASON="${2:?}"; shift 2 ;; --allow-live-run) ALLOW_LIVE_RUN=1; shift ;; --artifact-uri) ARTIFACT_URI="${2:?}"; shift 2 ;; + --no-artifact-uri) ARTIFACT_URI=""; shift ;; --wait) WAIT_SECS="${2:?}"; shift 2 ;; -h|--help) usage; exit 0 ;; *) RED "unknown arg: $1"; usage; exit 1 ;; @@ -507,12 +509,13 @@ submit_probe() { exit 2 fi resolve_cp || return 0 - local hotkey hex body code + local hotkey hex uri_field="" body code hotkey="$(head -c 64 /dev/zero | tr '\0' 'a')" hex="$(printf '%s' "wire-probe-$TOPIC-$(date +%s)-$$-$RANDOM" | sha256sum | awk '{print $1}')" - body="$(printf '{"miner_hotkey":"%s","artifact_digest":"%s","artifact_uri":"%s","claim":"proof-vm-wire-check probe","declared_flops":1,"topic_id":"%s","manifest":{"train_dataset_ids":["wire-probe-v0"]}}' \ - "$hotkey" "$hex" "$ARTIFACT_URI" "$TOPIC")" - LOG "submit-probe: POST $CP/v1/submissions topic=$TOPIC expect=$EXPECT${REASON:+ reason~'$REASON'}" + [[ -n "$ARTIFACT_URI" ]] && uri_field="$(printf '"artifact_uri":"%s",' "$ARTIFACT_URI")" + body="$(printf '{"miner_hotkey":"%s","artifact_digest":"%s",%s"claim":"proof-vm-wire-check probe","declared_flops":1,"topic_id":"%s","manifest":{"train_dataset_ids":["wire-probe-v0"]}}' \ + "$hotkey" "$hex" "$uri_field" "$TOPIC")" + LOG "submit-probe: POST $CP/v1/submissions topic=$TOPIC expect=$EXPECT${REASON:+ reason~'$REASON'}${ARTIFACT_URI:+ artifact_uri=$ARTIFACT_URI}" http POST "$CP/v1/submissions" "$body" -m "$WAIT_SECS"; code="$HTTP_CODE" LOG "→ HTTP $code $(printf '%s' "$HTTP_BODY" | head -c 500)" if [[ "$code" != "$EXPECT" ]]; then @@ -587,8 +590,9 @@ $0 submit-probe --cp $cp --topic $topic --expect 503 --reason PROOF_RLM_VM_IMAGE $0 submit-probe --cp $cp --topic $topic --expect 503 --reason 'orchestrator unreachable' # systemctl start proof-vm-orchestrator; then: $0 agent # -# 6. Unknown / closed topic → 400 (no row); a custom topic without artifact_uri → 400. -$0 submit-probe --cp $cp --topic does-not-exist --expect 400 +# 6. Unknown / closed topic → 400 (no row); a custom topic without artifact_uri → 400 (no row). +$0 submit-probe --cp $cp --topic does-not-exist --expect 400 --reason 'unknown topic' +$0 submit-probe --cp $cp --topic $topic --expect 400 --no-artifact-uri --reason artifact_uri # # After every row: $0 cp (the admin probe shows the same root cause: ready / reason / agent_error) EOF From 04ccf6dd9e5285ba38bb928dcee67e9e506db6b0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:46:53 +0000 Subject: [PATCH 06/15] =?UTF-8?q?docs(runbook):=20proof=20topic-vm=20orche?= =?UTF-8?q?strator=20=E2=80=94=20digitalocean=20staging=20wire=20+=20probe?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New § DigitalOcean staging: the CP stays on the existing staging master droplet; the agent needs a KVM host with /dev/kvm — a DO Droplet is not one (no nested virtualisation; the unit's ConditionPathExists refuses by design; no nested-FC workaround), so use DO bare-metal / dedicated hardware or any bare-metal KVM host on the staging VPC / private network, private bind, TLS + bearer. Host pre-checks, per-topic budget, CP preconditions (lium, live_harvest_wired, judge, executor, a signed custom topic with a sealed baseline), KVM-host and CP steps with the staging env overlays, the wire check (env / agent / cp / boot-probe), the fail-closed matrix (flip → restart? → expected 503 reason → what the admin probe shows), the happy path with the exact log lines and row / artefact evidence (sister sandboxed + flops_used, evidence bind, destroy teardown), a sign-off checklist that records unknown / not run rather than a green box, and rollback. § Wire the control plane documents GET /v1/admin/proof/vm-orchestrator (field → root cause table; loopback only — staging's public API is cleartext). Operate table gains "Is the wire up?". Co-authored-by: Mathis --- docs/runbooks/proof-vm-orchestrator.md | 240 ++++++++++++++++++++++++- 1 file changed, 238 insertions(+), 2 deletions(-) diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index 5f7892241..1c91565b6 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -6,6 +6,8 @@ for every miner run. Product spec: [`../PROOF.md`](../PROOF.md) § Isolation boundary. Code: `crates/proof-vm-proto` (wire), `crates/proof-vm-fc` (control-plane client), `crates/proof-vm-agent` (agent API), `crates/proof-fc-host` (Firecracker backend), `bins/proof-vm-orchestrator`. +Staging wire + probes: § DigitalOcean staging and +[`deploy/scripts/proof-vm-wire-check.sh`](../../deploy/scripts/proof-vm-wire-check.sh). ## What runs where @@ -112,6 +114,9 @@ mode 0400, uid 65532). Restart `proof-challenge`; its boot log must show `firecracker topic-vm orchestrator wired` and one `vm-backed runner registered` line per id. `GET /v1/status` → `registered_custom` lists the ids; an open custom topic with a listed id appears in `scorable_topics`. +Both need `live_harvest_wired: true`: the custom family is routed only over +a wired Lium harvest, so without `LIUM_API_KEY` + `LIUM_SSH_PUBLIC_KEY_FILE` +the registry is never built and `registered_custom` stays `[]`. The Lium harvest is **not** a prerequisite. With these four variables set and no `LIUM_API_KEY` / `LIUM_SSH_PUBLIC_KEY_FILE`, the boot log shows @@ -130,6 +135,234 @@ every submission will 503` instead, the orchestrator URL is unset or refused The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / `PROOF_RLM_VM_MEM_MIB` exist for a deliberate change only. +**Probe the wire from inside the CP** (operator bearer, read-only, no VM, +no spend): `GET /v1/admin/proof/vm-orchestrator` runs the client's own +`ready()` (bearer file + pin, re-read now) and one agent health call through +the very client the runner uses — same bearer file, same CA, same rustls — +and reports the host gates next to it. A broken wire is data, not an error: + +```bash +TOKEN=$(head -n1 deploy/secrets/proof/admin_tokens) +curl -sS -H "Authorization: Bearer $TOKEN" http://127.0.0.1:8080/challenge/proof/v1/admin/proof/vm-orchestrator +# {"orchestrator":"firecracker","ready":true,"reason":"","image_digest":"sha256:…","vcpus":4,"mem_mib":8192, +# "agent":{"api_version":1,"ready":true,"reason":"","hypervisor":"firecracker","vms":0},"agent_error":null, +# "live_harvest_wired":true,"registered_custom":[""]} +unset TOKEN +``` + +| Field | Root cause it names | +|-------|---------------------| +| `orchestrator: "unwired"` + `reason` | `PROOF_VM_ORCHESTRATOR_URL` unset, or set but refused (plain http off loopback, no `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`) — restart after fixing | +| `ready: false` + `reason` | bearer file missing / empty, or `PROOF_RLM_VM_IMAGE_DIGEST` unpinned — fix the file / env; the token needs no restart | +| `agent: null` + `agent_error` | `orchestrator unreachable` (agent down, firewall, VPC route), `orchestrator refused the bearer` (bytes differ from `/etc/proof-vm/token`), TLS refused (CA / SAN) | +| `agent.ready: false` + `agent.reason` | the KVM host: `firecracker` / `jailer` / `/dev/kvm` / image missing | +| `live_harvest_wired: false` | Lium creds absent on the CP → custom family not routed, `registered_custom` empty | + +Run it over SSH + loopback (staging's public API is cleartext; never send +the operator bearer over it). The reason strings name env vars and +container paths — operator data — never the bearer. +[`deploy/scripts/proof-vm-wire-check.sh cp`](../../deploy/scripts/proof-vm-wire-check.sh) +wraps this route with the status / leak checks (§ DigitalOcean staging). + +## DigitalOcean staging (wire + e2e probes) + +Flip staging from `UnwiredVmOrchestrator` to `FirecrackerOrchestrator` and +prove both the fail-closed matrix and one happy path, with evidence. The +operator harness is +[`deploy/scripts/proof-vm-wire-check.sh`](../../deploy/scripts/proof-vm-wire-check.sh) +(bash + curl + python3; runs on the droplet, no cargo; never prints the +bearer; refuses production hosts). Env overlays with placeholders only: +[`deploy/env/proof-challenge.staging-vm.example`](../../deploy/env/proof-challenge.staging-vm.example) +(CP side) and +[`deploy/env/proof-vm-orchestrator.staging.example`](../../deploy/env/proof-vm-orchestrator.staging.example) +(KVM host side). + +### Where things run + +| Piece | Host | Notes | +|-------|------|-------| +| `proof-challenge` (client, `FirecrackerOrchestrator`) | the **existing staging master droplet** `base-staging` (see [`staging-testnet-e2e.md`](staging-testnet-e2e.md)), compose `role-master` + `env-staging` | nothing Firecracker on it: no `/dev/kvm`, no unit, no images; it holds the bearer file, the CA PEM, the RLM image pin, and the custom ids | +| `proof-vm-orchestrator` (agent, Firecracker + jailer) | a **KVM host with `/dev/kvm`**, reachable from the staging VPC / private network | `systemd/proof-vm-orchestrator.service` has `ConditionPathExists=/dev/kvm`; bind on the private address, HTTPS + bearer file | + +**A DigitalOcean Droplet is not that host.** Standard, CPU-optimized, and +dedicated-CPU Droplets are KVM guests that expose no nested virtualisation: +`/dev/kvm` does not exist inside them, `kvm-ok` reports the CPU cannot run +KVM, and the unit refuses to start by design. Do not try to work around it +(no nested-FC redesign, no `--no-kvm` anything): the RLM VM and the sister +guest are the isolation boundary and a software emulator is not one. Use a +DigitalOcean bare-metal / dedicated-hardware host when the account has one, +or any bare-metal KVM host elsewhere (another provider, colo), attached to +the staging VPC's private network — WireGuard from the staging master, or +VPC peering when both sides are DO. Either way the agent listens on the +private address only and TLS + bearer stay mandatory (the bearer never +crosses a network in clear). + +Check the candidate host before installing anything: + +```bash +ls -l /dev/kvm # must exist (crw-rw---- root kvm) +grep -cE '(vmx|svm)' /proc/cpuinfo # > 0 +kvm-ok # "KVM acceleration can be used" (apt install cpu-checker) +modprobe vhost_vsock && ls -l /dev/vhost-vsock # vsock for the guest channels +nft list ruleset >/dev/null && ip -br link # nftables + the uplink you will name +df -T /srv /var/lib | grep -E 'xfs|btrfs' # reflink FS preferred (see Host prerequisites) +``` + +Budget per open topic: RLM VM 4 vCPU / 8192 MiB + sister 2 vCPU / 4096 MiB ++ ~10 GiB scratch. Size the host for the number of custom topics staging +will keep open at once, plus one probe VM. + +### 0. Preconditions on the CP + +The custom family is routed only when the whole live stack is up; check +`GET /v1/status` on the master **before** touching the wire: + +| Gate | Where | Must read | +|------|-------|-----------| +| eval backend | `/v1/status` `eval_backend` | `lium` (`PROOF_FORCE_SIM` off — sim never hosts staging scoring) | +| harvest | `/v1/status` `live_harvest_wired` | `true` (`LIUM_API_KEY` + `LIUM_SSH_PUBLIC_KEY_FILE`); `false` → `registered_custom` stays `[]` whatever the ids say | +| judge | `/v1/status` `inference_offer.status` | `open`, plus `PROOF_INFERENCE_API_KEY_FILE` present | +| executor | `GET /v1/proof/executor` | `ready: true` (open `1x` offer) | +| topic | a **signed custom topic** (`metric.family: custom`, `metric.custom_id: `) with a **sealed baseline**; the RLM of that topic sets it up per [`../PROOF.md`](../PROOF.md) § Dynamic agentic engine | its `custom_id` is what goes into `PROOF_VM_RUNNER_CUSTOM_IDS`; the topic can only **open** once that id is registered | + +Nothing here is challenge content in git: the topic, its rules, and its +images are operator-published documents and staged files. + +### 1. KVM host + +Follow § Host prerequisites and § Install with +`deploy/env/proof-vm-orchestrator.staging.example` as `/etc/proof-vm/orchestrator.env`: + +1. `PROOF_VM_AGENT_BIND=:8200`; a certificate for that address + (a private CA is fine — the CP pins its root) **with a SAN**: the CP's + rustls client refuses CN-only certificates even when curl accepts them. +2. Bearer: `head -c 32 /dev/urandom | base64 -w0 > /etc/proof-vm/token; chmod 0400 /etc/proof-vm/token`. +3. Stage `vmlinux`, the RLM rootfs, and the sister rootfs; `sha256sum` each; + name the rootfs files `images/sha256-.ext4`; put the kernel and + sister digests in the env. The RLM digest goes to the CP. Never copy a + digest from a document; only from `sha256sum` of the file you staged. +4. `PROOF_VM_AGENT_EGRESS_ALLOW`: the judge `InferenceOffer` origin, the + artefact hosts miners use, a resolver — IPv4 CIDRs, nothing else. +5. `systemctl enable --now proof-vm-orchestrator`; the boot log shows + `firecracker + jailer + /dev/kvm present; agent ready` and + `bearer token file present (contents not logged)`. +6. Open `:8200` on the host firewall to the staging master's private + address only. + +### 2. Control plane (staging master) + +```bash +ssh root@ ; cd /opt/base ; umask 077 +# The bearer: copy the KVM host's file over the private network — never paste it on a command line. +scp root@:/etc/proof-vm/token deploy/secrets/proof/vm_orchestrator_token +scp root@:/etc/proof-vm/ca.pem deploy/secrets/proof/vm_orchestrator_ca.pem # private CA only +chown 65532:65532 deploy/secrets/proof/vm_orchestrator_token deploy/secrets/proof/vm_orchestrator_ca.pem +chmod 0400 deploy/secrets/proof/vm_orchestrator_token deploy/secrets/proof/vm_orchestrator_ca.pem +# Append the keys of deploy/env/proof-challenge.staging-vm.example (values filled) to the age source +# of proof-challenge.env (deploy/scripts/age-encrypt-env.sh / age-push-env.sh), then: +./deploy/scripts/materialize-env.sh +docker compose -f docker-compose.yml -f deploy/compose/role-master.yml -f deploy/compose/env-staging.yml --profile master up -d proof-challenge +docker compose logs proof-challenge | grep -E 'topic-vm orchestrator|vm-backed runner' +# firecracker topic-vm orchestrator wired (bearer file present, contents not logged) +# vm-backed runner registered custom_id= +``` + +### 3. Wire check (on the droplet, no cargo) + +```bash +cd /opt/base +./deploy/scripts/proof-vm-wire-check.sh all # env + agent + cp; exit 0 = every check PASS +./deploy/scripts/proof-vm-wire-check.sh boot-probe # one RLM VM: create → attach → 409 → 409 topic_mismatch → destroy → 404 +``` + +| Subcommand | Proves | +|------------|--------| +| `env` | `PROOF_VM_ORCHESTRATOR_URL` is `https://`; the bearer file (container path mapped through the compose bind mount, `--path-map`) exists and is non-empty, mode 0400 / uid 65532; `PROOF_RLM_VM_IMAGE_DIGEST` is `sha256:<64 hex>` (empty or a placeholder = FAIL — never invented); the CA file is PEM when set; every custom id is well-formed; the shape is the locked 4 / 8192; `PROOF_FORCE_SIM` is off | +| `agent` | `GET /v1/health` with the bearer → `ready: true`, `hypervisor: firecracker`; no bearer → 401; wrong bearer → 401 | +| `cp` | `/v1/status`: `lium`, `live_harvest_wired`, `registered_custom` ⊇ ids, no URL / token / path in the body; `/v1/proof/topics` leaks no holdout; `/v1/proof/executor` readiness; then the admin probe above — `orchestrator: firecracker`, `ready: true`, `agent.ready: true` through the CP's own rustls client | +| `boot-probe` | the agent boots the **pinned** image for a probe topic, one topic ↔ one VM, a teardown naming another topic is refused, destroy is confirmed, nothing is left for the topic. Opt-in: it boots a real 4 vCPU / 8 GiB RLM VM on the KVM host (up to 10 min, the RLM guest must say hello); no job runs, nothing is spent; Ctrl-C tears the VM down | + +Every check re-reads the files it names, so a fix to the bearer or the CA +needs no restart; URL / digest / ids are read at boot. + +### 4. Fail-closed matrix (every row: 503, no row, no VM, no rent) + +`./deploy/scripts/proof-vm-wire-check.sh matrix --topic ` +prints these as ready-to-paste steps. Flip one knob, probe, restore, re-run +`cp`. `submit-probe` POSTs a probe submission (64×`a` hotkey, random +artefact digest, `https://example.invalid/…` locator — never fetchable) and +asserts the status **and** the reason text; a 2xx expectation is refused +without `--allow-live-run`. + +| Flip | Restart? | `submit-probe --expect 503 --reason …` | Admin probe shows | +|------|----------|----------------------------------------|-------------------| +| comment out `PROOF_VM_ORCHESTRATOR_URL` | yes | `PROOF_VM_ORCHESTRATOR_URL` (from `runner not wired: no orchestrator configured (…)`) | `orchestrator: unwired` | +| empty the CP bearer file (`: > deploy/secrets/proof/vm_orchestrator_token`) | no | `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` … `missing or empty` | `ready: false` | +| write other bytes into the CP bearer file | no | `refused the bearer` (agent 401 → CP 503) | `agent_error: … refused the bearer` | +| `PROOF_RLM_VM_IMAGE_DIGEST=` | yes | `PROOF_RLM_VM_IMAGE_DIGEST` … `image_digest is missing or out of range` | `ready: false`, `image_digest: ""` | +| `systemctl stop proof-vm-orchestrator` on the KVM host | no | `orchestrator unreachable` (route named, agent address never) | `agent_error: … unreachable` | +| unknown / closed topic → `--expect 400 --reason 'unknown topic'`; custom topic without a locator → `--expect 400 --no-artifact-uri --reason artifact_uri` | no | 400, explicit error, no row | — | + +After each row `docker compose logs proof-challenge` must show no +`topic vm created`, and the KVM host journal no `topic vm booted`; the +`/v1/submissions` list gains no row. Restore the knob and run +`proof-vm-wire-check.sh cp` before the next flip. + +### 5. Happy path (one real run) + +With everything restored and an open custom topic whose id is registered: + +```bash +./deploy/scripts/proof-vm-wire-check.sh submit-probe --topic \ + --expect 201 --allow-live-run --artifact-uri +``` + +The POST is synchronous (the RLM job runs before the 201). Evidence to +collect, in order: + +| Step | Where | Must show | +|------|-------|-----------| +| topic VM created (first job) | CP log · agent journal | `topic vm created` · `topic vm booted`, `rlm guest ready`, `owner key material staged` when a key dir is set | +| inspection (`Inspect` job, no miner code, no sister) | agent journal | the job, no `sister guest` line | +| paid run (`Evaluate`) in the sister | agent journal | `sister guest booting (no network)` → `sister guest run attested` with `sandboxed=true` and the guest's `flops_used` | +| evidence bound to the job | agent + CP | no `evidence_mismatch` (agent 502) and no `orchestrator evidence is not this job's` (CP): the attestation named this job's topic / submission / artefact | +| host-stamped facts on the row | `submit-probe` output · `GET /v1/submissions/` | `state: awaiting_admin`, `verdict.agent.flops_used` > 0 (the sister's measurement, never the RLM's) | +| `sandboxed: true` in the artefact | CP volume | `docker compose cp proof-challenge:/var/lib/proof/artefacts//.zip /tmp/ && unzip -p /tmp/.zip report.json` | +| sister destroyed | agent journal · KVM host | `jail released`; `/srv/jailer/firecracker/` has no `-s` | +| topic VM teardown | close the topic → CP log · agent journal · KVM host | `topic vm teardown` with `state: Destroyed`, `confirmed: true` · `topic vm torn down` · `/srv/jailer/firecracker/` gone, `nft list tables` has no `proof_vm_pfc` | + +Then run the § Verify cleanup probes (failed `ip tuntap`, deadline cut, +`kill -9`) at least once on the staging KVM host. + +### 6. Sign-off + +Staging is "wired and tested" when all of these are in the change log with +dates and the exact commands: + +- [ ] `proof-vm-wire-check.sh all` → all PASS on the staging master. +- [ ] `proof-vm-wire-check.sh boot-probe` → all PASS; KVM host left clean. +- [ ] every row of § 4 → the expected 503 (or 400) with the expected reason, + no row, no VM, no rent; knob restored; `cp` PASS again. +- [ ] § 5 evidence table complete for one submission, including + `flops_used` on the row and `sandboxed: true` in `report.json`. +- [ ] topic close → VM destroyed, host clean. +- [ ] `GET /v1/status` and `GET /v1/proof/topics` still leak nothing + (`cp` checks both). + +Where an item cannot be run yet (no custom topic sealed, no RLM image +built), write **unknown / not run** with the blocker — never a green box +without the evidence. + +### Rollback + +Comment out `PROOF_VM_ORCHESTRATOR_URL` (or all four keys) in the age +source, re-materialize, restart `proof-challenge`: the host logs +`no topic-vm orchestrator (…)` and every custom topic answers 503 with that +reason — the same state as before the flip. `systemctl stop +proof-vm-orchestrator` on the KVM host; live VMs die with the agent (no +`--daemonize`); remove `/srv/jailer/firecracker/*` by hand before the next +start (§ Limitations). + ## Verify a submission end to end (mandatory, see root `AGENTS.md`) 1. `POST /v1/submissions` on a custom topic with a listed id → the first job @@ -140,7 +373,9 @@ The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / and the guest's `flops_used`. 2. The persisted row's verdict carries that `flops_used`; the artefact zip's `report.json` has `sandboxed: true`. -3. Failure probes, each **503 with no row and no rent**: +3. Failure probes, each **503 with no row and no rent** + (`proof-vm-wire-check.sh submit-probe --topic --expect 503 --reason ` + asserts the status and the reason; § DigitalOcean staging has the matrix): - stop the agent → `orchestrator unreachable`; - empty `/etc/proof-vm/token` → `orchestrator refused the bearer`; - remove `PROOF_RLM_VM_IMAGE_DIGEST` → `PROOF_RLM_VM_IMAGE_DIGEST … missing or out of range`; @@ -172,7 +407,8 @@ The RLM VM shape is 4 vCPU / 8192 MiB. `PROOF_RLM_VM_VCPUS` / | Task | How | |------|-----| -| Rotate the bearer | write the new token to `/etc/proof-vm/token` and to the CP's token file; no restart on either side (both re-read per request) | +| Is the wire up? | on the master: `./deploy/scripts/proof-vm-wire-check.sh all` (env + agent + admin probe); or `GET /v1/admin/proof/vm-orchestrator` with the operator bearer over loopback | +| Rotate the bearer | write the new token to `/etc/proof-vm/token` and to the CP's token file; no restart on either side (both re-read per request); confirm with `proof-vm-wire-check.sh agent` | | Rotate the RLM image | stage `images/sha256-.ext4`, set `PROOF_RLM_VM_IMAGE_DIGEST` on the CP, restart `proof-challenge`; running VMs keep the old image until torn down | | Close a topic | the CP tears the VM down with the topic's `retain` policy (default destroy). `retain` moves `/srv/jailer/firecracker/` to `/var/lib/proof-vm/retained/` (scratch, console log, config) | | Agent restart | live VMs die with the agent (no `--daemonize`); `attach` then answers 404 and the CP's next job creates a fresh VM. Rules, checklists, and promotions live in the CP's RLM store, not in the VM | From d63b0b85be8c96e9da687aefcd72c6b9ffe02a59 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 20:49:35 +0000 Subject: [PATCH 07/15] docs: point operators at the proof topic-vm staging harness AGENTS.md verification item 7, deploy/AGENTS.md (staging wire section), docs/AGENTS.md runbook index, docs/PROOF.md HTTP surface (the admin vm-orchestrator probe), deploy/secrets/README.md (admin_tokens now also gates the probe), docs/COMPLETENESS.md (probe + harness + overlays; still not on any host, sign-off unfilled). Co-authored-by: Mathis --- AGENTS.md | 2 +- deploy/AGENTS.md | 14 ++++++++++++++ deploy/secrets/README.md | 2 +- docs/AGENTS.md | 2 +- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 8 ++++++++ 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b7aa67e94..5759a4e4b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). 5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. -7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md). +7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Probe the wire with `GET /v1/admin/proof/vm-orchestrator` (operator bearer, loopback) or `deploy/scripts/proof-vm-wire-check.sh` (`all`, `boot-probe`, `matrix` + `submit-probe --expect 503 --reason …`); a Droplet is never the KVM host. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md) § DigitalOcean staging. 8. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 46efdf564..0a4808d0c 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -91,6 +91,20 @@ files the operator stages (`sha256sum`) — never invented, never in git. Procedure and the mandatory submission verification: [`docs/runbooks/proof-vm-orchestrator.md`](../docs/runbooks/proof-vm-orchestrator.md). +**Staging wire (DO):** the CP is the existing staging master; the agent +needs a KVM host with `/dev/kvm` reachable from the staging VPC — a Droplet +is not one (no nested virt). Overlays with placeholders only: +[`env/proof-challenge.staging-vm.example`](env/proof-challenge.staging-vm.example) +(CP) and +[`env/proof-vm-orchestrator.staging.example`](env/proof-vm-orchestrator.staging.example) +(KVM host); every `REPLACE_WITH_*` fails closed as written. Prove the wire +on the master with +[`scripts/proof-vm-wire-check.sh`](scripts/proof-vm-wire-check.sh) — `all` +(env + agent + `GET /v1/admin/proof/vm-orchestrator` through the CP's own +client), `boot-probe` (one RLM VM created and destroyed, no job), `matrix` ++ `submit-probe --expect 503 --reason …` (every fail-closed flip), and the +one `--allow-live-run` happy path. Runbook § DigitalOcean staging. + ## Local testnet E2E Full procedure: [`docs/runbooks/local-testnet-e2e.md`](../docs/runbooks/local-testnet-e2e.md). diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index bd2a54a4c..0518aa1b0 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -34,7 +34,7 @@ chmod 0400 deploy/secrets/gateway_admin_token | `proof/topics.json` | proof-challenge | Signed topic documents (JSON array). **Never commit secrets**; the documents themselves are operator-published. Mode **0400**, uid **65532** | | `proof/holdouts.json` | proof-challenge | Per-topic holdout records (array or map keyed by `topic_id`). **Never commit.** Verified at boot against each topic's `holdout_commitment`. Mode **0400**, uid **65532** | | `proof/baselines.json` | proof-challenge | Sealed baseline measurements keyed by topic id. **Never commit.** Mode **0400**, uid **65532** | -| `proof/admin_tokens` | proof-challenge | One operator bearer per line for `POST /v1/admin/proof/topics` and `POST /v1/admin/proof/executor` | +| `proof/admin_tokens` | proof-challenge | One operator bearer per line for `POST /v1/admin/proof/topics`, `POST /v1/admin/proof/executor`, and `GET /v1/admin/proof/vm-orchestrator` (`deploy/scripts/proof-vm-wire-check.sh cp` reads the first line) | | `proof/inference_offer.json` | proof-challenge | Live RLM judge `InferenceOffer` (provider kind, origin, mode, model_ref, token caps, `config_commitment`, status). Consumed by proof-eval; **not** a miner training proxy. **Never commit.** Missing/closed → `can_score=false` / 503. Mode **0400**, uid **65532** | | `proof/inference_api_key` | proof-challenge | Provider API key for the eval image. **Never commit, never log.** Mode **0400**, uid **65532** | | `proof/inference_base_url` | proof-challenge | Optional secret-backed origin (`PROOF_INFERENCE_BASE_URL_FILE`) when pin `[inference].base_url` and the topic omit one. **Never commit, never log.** Mode **0400**, uid **65532** | diff --git a/docs/AGENTS.md b/docs/AGENTS.md index ff7469431..4b836f05e 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -20,7 +20,7 @@ When a spike or evidence report conflicts with a frozen spec or runbook, the nor | [`runbooks/local-testnet-e2e.md`](runbooks/local-testnet-e2e.md) | Local laptop/VM full subnet stack on testnet 541 + ephemeral gateway tunnel | | [`runbooks/staging-testnet-e2e.md`](runbooks/staging-testnet-e2e.md) | Staging droplet testnet end-to-end validation | | [`runbooks/proof-submit-e2e.md`](runbooks/proof-submit-e2e.md) | Proof (and Bounty) submit → score: cargo tests, local `--force-sim`, staging curl/ctx | -| [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | Proof topic VMs: Firecracker + jailer agent on the dedicated KVM host, CP wiring, sister-guest verification, security model | +| [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | Proof topic VMs: Firecracker + jailer agent on the dedicated KVM host, CP wiring, DigitalOcean staging wire + fail-closed matrix + happy path (`deploy/scripts/proof-vm-wire-check.sh`), sister-guest verification, security model | | [`runbooks/trust-root-rotation.md`](runbooks/trust-root-rotation.md) | Trust-root key rotation | | [`runbooks/gateway-failover.md`](runbooks/gateway-failover.md) | Gateway kill/restart / failover checks | | [`runbooks/measurement-repin-socket-proxy.md`](runbooks/measurement-repin-socket-proxy.md) | Socket-proxy measurement re-pin | diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index c13eb7657..3708e2471 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -86,7 +86,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | | RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. `/v1/status` reports the families apart — `live_harvest_wired` is Lium only; `custom_family_wired` / `registered_custom` / `custom_ready` are the custom family. | -| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | +| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `live_harvest_wired`, `registered_custom`) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index c87ab33a4..a9ef869ae 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -232,6 +232,14 @@ Trust-root keygen is the throwaway owner path in document. Pin-validated (**400** keeps the previous offer); `status: closed` takes the executor down live. In-memory until restart, like submissions — update `PROOF_EVAL_EXECUTOR_OFFER_FILE` to persist. +- `GET /v1/admin/proof/vm-orchestrator` — operator bearer; read-only probe + of the topic-VM orchestrator through the host's own client: `orchestrator` + (`firecracker` / `unwired`), `ready` + `reason` (bearer file, RLM image + pin), the locked template, one agent health call (`agent` / + `agent_error`), `live_harvest_wired`, `registered_custom`. Always **200** + once authorised — a broken wire is data. Names env vars and container + paths, never the bearer. Run over loopback; wrapped by + [`deploy/scripts/proof-vm-wire-check.sh`](../deploy/scripts/proof-vm-wire-check.sh). - `POST /v1/submissions` **requires** `topic_id`. Missing/unknown/not-open → **400**. Miners do **not** bind the judge offer or the executor offer. Zero open / unsealed baseline / empty digest / missing or closed RLM judge From a3c9433c1e338c5c70b90ee1910aa095d4f5cda2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:01:34 +0000 Subject: [PATCH 08/15] test(proof-rlm-scorer): rlm_e2e app state carries vm_probe Co-authored-by: Mathis --- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index df653c85d..11a60ab06 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -158,6 +158,7 @@ fn stack(register: bool) -> Stack { executor: executor_slot(Some(executor)), judge_api_key: Some("test-judge-key".into()), admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, epoch: 0, }); Stack { From 2bf25b6bca15f9706d87eeb670aef4155afd9095 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:14:38 +0000 Subject: [PATCH 09/15] docs(runbook): staging may colocate the fc agent on the cp when /dev/kvm works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architecte nit on #247: `cortex-staging` (nested DigitalOcean virtualisation) did boot Firecracker and the § 4 fail-closed matrix came back green, so the runbook must not claim a Droplet can never be the KVM host. Corrected wording everywhere the PR (and the #244 lines next to it) said so: - staging may colocate the agent on the control-plane droplet when /dev/kvm works — validated on cortex-staging; - production prefers a dedicated DO bare-metal / KVM host; - nested KVM remains fragile — if the boot fails or /dev/kvm goes away, provision metal, do not patch around it (fragility signs listed); - never Lium, never an emulator, never without /dev/kvm. Runbook § DigitalOcean staging gains the colocated specifics: the CP container reaches the agent on the droplet's VPC address (loopback in the container is the container; FcConfig accepts plain http on loopback only), agent bound on that address with TLS + SAN, :8200 firewalled to the compose network, sizing on top of the compose stack, a local `install` of the bearer copy; sign-off records the placement. The fail-closed probes, the matrix, and the harness are unchanged. Same correction in AGENTS.md item 7, deploy/AGENTS.md, both staging overlays, the generic env examples, the systemd unit comment, docs/PROOF.md, docs/ARCHITECTURE.md, docs/COMPLETENESS.md (staging boot + § 4 green recorded; § 5 / § 6 still to be recorded), and the agent / binary module docs. Co-authored-by: Mathis --- AGENTS.md | 2 +- bins/proof-vm-orchestrator/src/main.rs | 9 +- crates/proof-vm-agent/src/lib.rs | 6 +- deploy/AGENTS.md | 15 ++- deploy/env/proof-challenge.env.example | 13 +- deploy/env/proof-challenge.staging-vm.example | 22 ++-- deploy/env/proof-vm-orchestrator.env.example | 8 +- .../env/proof-vm-orchestrator.staging.example | 28 +++-- deploy/systemd/proof-vm-orchestrator.service | 12 +- docs/ARCHITECTURE.md | 2 +- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 7 +- docs/runbooks/proof-vm-orchestrator.md | 115 +++++++++++++----- 13 files changed, 162 insertions(+), 79 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5759a4e4b..140dd1f9d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). 5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. -7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a **dedicated KVM host** (`proof-vm-orchestrator`, HTTPS + bearer **file**), never on the droplet, never on Lium, never nested; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Probe the wire with `GET /v1/admin/proof/vm-orchestrator` (operator bearer, loopback) or `deploy/scripts/proof-vm-wire-check.sh` (`all`, `boot-probe`, `matrix` + `submit-probe --expect 503 --reason …`); a Droplet is never the KVM host. Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md) § DigitalOcean staging. +7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a host with a working `/dev/kvm` (`proof-vm-orchestrator`, HTTPS + bearer **file**) — production prefers a dedicated DO bare-metal / KVM host; staging may colocate the agent on the CP droplet when nested KVM boots (validated on `cortex-staging`; nested stays fragile — if the boot fails, provision metal); never on Lium, never emulated; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Probe the wire with `GET /v1/admin/proof/vm-orchestrator` (operator bearer, loopback) or `deploy/scripts/proof-vm-wire-check.sh` (`all`, `boot-probe`, `matrix` + `submit-probe --expect 503 --reason …`). Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md) § DigitalOcean staging. 8. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. diff --git a/bins/proof-vm-orchestrator/src/main.rs b/bins/proof-vm-orchestrator/src/main.rs index 6a36f6b86..bf46f2e23 100644 --- a/bins/proof-vm-orchestrator/src/main.rs +++ b/bins/proof-vm-orchestrator/src/main.rs @@ -1,12 +1,13 @@ -//! `proof-vm-orchestrator` — Firecracker topic-VM agent for the **dedicated -//! KVM host** (HTTPS `:8200`). +//! `proof-vm-orchestrator` — Firecracker topic-VM agent for a **KVM host** +//! (HTTPS `:8200`): a dedicated host in production, or the control-plane +//! droplet itself on staging when nested KVM boots there. //! //! The Proof control plane (`proof-challenge`, `FirecrackerOrchestrator`) //! is its only client. It boots one jailed RLM microVM per topic from the //! digest the control plane pins, runs every miner artefact in a sister //! microVM with no network, and stamps what it saw onto the report. It -//! never runs on the control-plane droplet, never on a Lium pod, and never -//! receives a key from the control plane — owner key material is read from +//! never runs on a Lium pod, never without `/dev/kvm`, and never receives a +//! key from the control plane — owner key material is read from //! `--owner-key-dir` on this host and staged over vsock. //! //! Fail-closed at boot: malformed kernel / sister image pins exit 1, a diff --git a/crates/proof-vm-agent/src/lib.rs b/crates/proof-vm-agent/src/lib.rs index 5e4b6cd4e..0e78e9510 100644 --- a/crates/proof-vm-agent/src/lib.rs +++ b/crates/proof-vm-agent/src/lib.rs @@ -1,7 +1,9 @@ //! `proof-vm-orchestrator` agent library. //! -//! The agent runs on a **dedicated KVM host** (never the control-plane -//! droplet, never a Lium pod) and is the only thing that talks to Firecracker. +//! The agent runs on a host with a working `/dev/kvm` — a **dedicated KVM +//! host** in production, the control-plane droplet itself on staging when +//! nested KVM boots there; never a Lium pod — and is the only thing that +//! talks to Firecracker. //! The Proof control plane reaches it over HTTPS with a bearer read from a //! file ([`BearerAuth`]) and drives four verbs (`proof_vm_proto::paths`): //! diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 0a4808d0c..b75d22fc1 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -71,8 +71,11 @@ gaps. See [`docs/WHITEPAPER.md`](../docs/WHITEPAPER.md). Custom-family topics run their RLM in one Firecracker microVM per `topic_id` and every miner run in a **sister** Firecracker guest with no network — on a -**dedicated KVM host**, not on any droplet (DO has no nested virt), not on -Lium. That host runs `proof-vm-orchestrator` as a systemd unit +host with a working `/dev/kvm`: production prefers a **dedicated DO +bare-metal / KVM host**; staging may colocate the agent on the CP droplet +when nested KVM boots (validated on `cortex-staging`; nested stays fragile — +if the boot fails, provision metal); never Lium. That host runs +`proof-vm-orchestrator` as a systemd unit ([`systemd/proof-vm-orchestrator.service`](systemd/proof-vm-orchestrator.service), env [`env/proof-vm-orchestrator.env.example`](env/proof-vm-orchestrator.env.example)), **not** a compose service. The master's `proof-challenge` is only its HTTPS @@ -91,9 +94,11 @@ files the operator stages (`sha256sum`) — never invented, never in git. Procedure and the mandatory submission verification: [`docs/runbooks/proof-vm-orchestrator.md`](../docs/runbooks/proof-vm-orchestrator.md). -**Staging wire (DO):** the CP is the existing staging master; the agent -needs a KVM host with `/dev/kvm` reachable from the staging VPC — a Droplet -is not one (no nested virt). Overlays with placeholders only: +**Staging wire (DO):** the CP is the existing staging master; the agent runs +as a host systemd unit on the same droplet (`cortex-staging`, nested KVM +validated) bound on the VPC address the CP container reaches over HTTPS — or +on a dedicated KVM host when nested KVM does not boot. Overlays with +placeholders only: [`env/proof-challenge.staging-vm.example`](env/proof-challenge.staging-vm.example) (CP) and [`env/proof-vm-orchestrator.staging.example`](env/proof-vm-orchestrator.staging.example) diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 77c893298..f6c6c6751 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -109,11 +109,14 @@ PROOF_SIM_STUB_WIN=false # PROOF_ARTEFACT_ROOT=/var/lib/proof/artefacts # # Topic-VM orchestrator: the RLM runs inside one Firecracker microVM per -# topic on a DEDICATED KVM HOST (never this droplet, never a Lium pod), and -# every miner run happens in a SISTER Firecracker guest with no network. This -# host is only the client (`FirecrackerOrchestrator`, crates/proof-vm-fc); -# the agent is `proof-vm-orchestrator` (deploy/systemd/, runbook -# docs/runbooks/proof-vm-orchestrator.md). Fail-closed: URL unset → unwired +# topic on a host with /dev/kvm (production: a dedicated KVM host; staging +# may colocate the agent on this droplet when nested KVM boots; never a Lium +# pod), and every miner run happens in a SISTER Firecracker guest with no +# network. This container is only the client (`FirecrackerOrchestrator`, +# crates/proof-vm-fc) and reaches the agent on its private address, never on +# the container's loopback; the agent is `proof-vm-orchestrator` +# (deploy/systemd/, runbook docs/runbooks/proof-vm-orchestrator.md). +# Fail-closed: URL unset → unwired # (503, names the vars); URL set but not https:// → refused, stays unwired; # token file missing/empty or image digest unpinned → 503 naming the var # (fixable without a restart). No host-local execution path exists. diff --git a/deploy/env/proof-challenge.staging-vm.example b/deploy/env/proof-challenge.staging-vm.example index 90de135e6..b800da8a2 100644 --- a/deploy/env/proof-challenge.staging-vm.example +++ b/deploy/env/proof-challenge.staging-vm.example @@ -1,10 +1,12 @@ # operator-managed, never committed with values filled in. # # STAGING overlay for deploy/env/proof-challenge.env on the staging MASTER -# droplet (`base-staging`, compose role-master + env-staging): the CLIENT side -# of the Proof topic-VM orchestrator. Nothing Firecracker runs on this -# droplet; proof-challenge only talks HTTPS to `proof-vm-orchestrator` on the -# dedicated KVM host (deploy/env/proof-vm-orchestrator.staging.example). +# droplet (`cortex-staging`, compose role-master + env-staging): the CLIENT +# side of the Proof topic-VM orchestrator. proof-challenge only talks HTTPS +# to `proof-vm-orchestrator`, a host systemd unit — on staging colocated on +# this same droplet (nested KVM validated; fragile — if the boot fails, +# provision a dedicated KVM host), in production on a dedicated DO +# bare-metal / KVM host (deploy/env/proof-vm-orchestrator.staging.example). # # Use: append these keys to the age-encrypted source of proof-challenge.env, # re-materialize, restart proof-challenge, then @@ -16,13 +18,15 @@ # this file is real: take PROOF_RLM_VM_IMAGE_DIGEST from `sha256sum` of the # RLM rootfs you staged on the KVM host. Do not invent one. -# HTTPS URL of the agent, reachable from the staging VPC / private network -# only (never a public address, never this droplet). Plain http:// off -# loopback is refused at boot and the host stays unwired. -PROOF_VM_ORCHESTRATOR_URL=https://REPLACE_WITH_KVM_HOST_PRIVATE_ADDRESS:8200 +# HTTPS URL of the agent on its private (VPC) address — never a public one. +# Colocated: this droplet's VPC IP, not 127.0.0.1 (inside the proof-challenge +# container loopback is the container). Plain http:// off loopback is refused +# at boot and the host stays unwired. +PROOF_VM_ORCHESTRATOR_URL=https://REPLACE_WITH_AGENT_HOST_PRIVATE_ADDRESS:8200 # Bearer FILE, container path. Host file: deploy/secrets/proof/vm_orchestrator_token -# (mode 0400, uid 65532), same bytes as /etc/proof-vm/token on the KVM host. +# (mode 0400, uid 65532), same bytes as the agent's /etc/proof-vm/token (a +# second copy even when colocated: the container reads its own mount). # Re-read per request: rotate by rewriting both files, no restart. PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/base/proof/vm_orchestrator_token diff --git a/deploy/env/proof-vm-orchestrator.env.example b/deploy/env/proof-vm-orchestrator.env.example index 59b5080d4..9163aef56 100644 --- a/deploy/env/proof-vm-orchestrator.env.example +++ b/deploy/env/proof-vm-orchestrator.env.example @@ -1,7 +1,9 @@ # operator-managed, never committed to git. -# Environment for /etc/proof-vm/orchestrator.env on the DEDICATED KVM HOST -# (deploy/systemd/proof-vm-orchestrator.service). Not a compose env file: -# the agent never runs on the control-plane droplet or on a Lium pod. +# Environment for /etc/proof-vm/orchestrator.env on the host that runs the +# agent (deploy/systemd/proof-vm-orchestrator.service): a dedicated KVM host +# in production; on staging the control-plane droplet itself when nested KVM +# boots (see docs/runbooks/proof-vm-orchestrator.md). Not a compose env file; +# never a Lium pod. # # Nothing in this file is a secret value. The bearer is a FILE the agent # re-reads on every request (rotate by rewriting it, no restart). Owner key diff --git a/deploy/env/proof-vm-orchestrator.staging.example b/deploy/env/proof-vm-orchestrator.staging.example index 6095fe454..1f9da7e32 100644 --- a/deploy/env/proof-vm-orchestrator.staging.example +++ b/deploy/env/proof-vm-orchestrator.staging.example @@ -1,12 +1,18 @@ # operator-managed, never committed with values filled in. # -# STAGING /etc/proof-vm/orchestrator.env for the DEDICATED KVM HOST that serves -# the staging master (deploy/systemd/proof-vm-orchestrator.service). This is -# not a compose file and never lands on a droplet: the unit needs /dev/kvm -# (ConditionPathExists) and DigitalOcean Droplets expose no nested KVM. Use a -# bare-metal / dedicated KVM host reachable from the staging VPC or a private -# network (WireGuard / peering) and bind the agent on that private address. -# Generic reference with every knob: deploy/env/proof-vm-orchestrator.env.example. +# STAGING /etc/proof-vm/orchestrator.env for the host that runs the agent +# (deploy/systemd/proof-vm-orchestrator.service — a host systemd unit, not a +# compose service). It needs a working /dev/kvm (ConditionPathExists): +# - staging: the staging master droplet itself (`cortex-staging`), where +# nested DO virtualisation booted Firecracker (validated). Nested KVM is +# fragile — if the boot fails or /dev/kvm goes away, do not patch around +# it: provision a dedicated KVM host and repoint the CP. +# - production: a dedicated DO bare-metal / KVM host on the VPC or a +# private network (WireGuard / peering). +# Either way bind the agent on the host's PRIVATE address with TLS; when +# colocated that is the droplet's VPC IP (the CP container cannot use +# loopback). Generic reference with every knob: +# deploy/env/proof-vm-orchestrator.env.example. # # Every REPLACE_WITH_* value FAILS CLOSED as written: a malformed kernel or # sister pin exits 1 at boot, a non-loopback bind without TLS exits 1. The @@ -15,8 +21,9 @@ # request, owner key material is a DIRECTORY staged over vsock. # Listen on the private (VPC / WireGuard) address only; the staging master's -# PROOF_VM_ORCHESTRATOR_URL points here over HTTPS. -PROOF_VM_AGENT_BIND=REPLACE_WITH_KVM_HOST_PRIVATE_IP:8200 +# PROOF_VM_ORCHESTRATOR_URL points here over HTTPS. Colocated: the droplet's +# own VPC IP, firewalled to the compose network's subnet. +PROOF_VM_AGENT_BIND=REPLACE_WITH_AGENT_HOST_PRIVATE_IP:8200 # Certificate with a SAN for that address / name (rustls on the CP refuses # CN-only). Private CA → its root goes to the CP as vm_orchestrator_ca.pem. PROOF_VM_AGENT_TLS_CERT=/etc/proof-vm/tls.crt @@ -50,7 +57,8 @@ PROOF_VM_AGENT_JAIL_GID=65534 # Sizes. The RLM VM shape (4 vCPU / 8192 MiB) is set by the CP; the sister # is sized here, never by the RLM. Staging host budget: one RLM VM + one -# sister per open topic → 6 vCPU / 12 GiB RAM / ~10 GiB scratch per topic. +# sister per open topic → 6 vCPU / 12 GiB RAM / ~10 GiB scratch per topic — +# on top of the compose stack when colocated on the droplet. PROOF_VM_AGENT_SCRATCH_MIB=8192 PROOF_VM_AGENT_SISTER_VCPUS=2 PROOF_VM_AGENT_SISTER_MEM_MIB=4096 diff --git a/deploy/systemd/proof-vm-orchestrator.service b/deploy/systemd/proof-vm-orchestrator.service index a8e435e51..3f8390a32 100644 --- a/deploy/systemd/proof-vm-orchestrator.service +++ b/deploy/systemd/proof-vm-orchestrator.service @@ -1,9 +1,13 @@ [Unit] -Description=Proof topic-VM orchestrator (Firecracker + jailer agent; dedicated KVM host only) +Description=Proof topic-VM orchestrator (Firecracker + jailer agent; host with /dev/kvm) Documentation=https://github.com/CortexLM/cortex/blob/main/docs/runbooks/proof-vm-orchestrator.md -# Never install this unit on a control-plane droplet or a Lium pod: it needs -# /dev/kvm, a pinned kernel + rootfs images, and CAP_NET_ADMIN for the per-VM -# TAP + nftables allowlist. The control plane reaches it over HTTPS only. +# Runs only where /dev/kvm works: a dedicated KVM host in production; on +# staging it may share the control-plane droplet when nested KVM boots +# (validated on cortex-staging; fragile — if the boot fails, provision metal). +# Never a Lium pod. It needs /dev/kvm, a pinned kernel + rootfs images, and +# CAP_NET_ADMIN for the per-VM TAP + nftables allowlist. The control plane +# reaches it over HTTPS only (on its private address, never loopback from a +# container). After=network-online.target Wants=network-online.target ConditionPathExists=/dev/kvm diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 29ad4223f..1f051740b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -69,7 +69,7 @@ terminates in the host reverse proxy, not in the gateway process. | `validator` | Fetch/mirror bundle, verify, recompute, peer cross-check, CRV4 submit, dissent | | `bounty-challenge` | **Master-only:** internal pair/reports/adjudicate; **reads** CortexLM/backend public API for scoring and signs leaves from those rows. An unreadable feed pays nobody — `E` is covered with `ChallengeInternal`, share burns to uid 0 — rather than scoring offline | | `proof-challenge` | **Master-only:** signed topics, holdout loading, evaluation orchestration. Library payout is a sum of WTA/discovery topic masses; the binary has no automatic leaf-emission loop yet | -| `proof-vm-orchestrator` | **Dedicated KVM host only** (never a droplet, never Lium): Firecracker + jailer agent behind HTTPS + a bearer file. One RLM microVM per Proof topic from the digest the control plane pins, sister miner guest with no network per paid run, host-stamped `sandboxed` / `flops_used`. Client side is `proof-vm-fc::FirecrackerOrchestrator`; runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | +| `proof-vm-orchestrator` | **Host with a working `/dev/kvm`** — a dedicated KVM host in production; staging may colocate it on the CP droplet when nested KVM boots (validated; fragile → provision metal if the boot fails); never Lium: Firecracker + jailer agent behind HTTPS + a bearer file. One RLM microVM per Proof topic from the digest the control plane pins, sister miner guest with no network per paid run, host-stamped `sandboxed` / `flops_used`. Client side is `proof-vm-fc::FirecrackerOrchestrator`; runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | | `updater` | Digest-pinned rollouts via `docker-socket-proxy` (master) | | `trustroot` | Offline keygen / sign / verify for owner-signed TOML | | `bundle` | SCALE types, seal, verify (`PROTOCOL_VERSION`) | diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 3708e2471..2f141cd36 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -86,7 +86,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | | RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. `/v1/status` reports the families apart — `live_harvest_wired` is Lium only; `custom_family_wired` / `registered_custom` / `custom_ready` are the custom family. | -| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `live_harvest_wired`, `registered_custom`) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Not yet on any host:** no RLM / sister image digest is pinned (the operator computes them from images built outside this repo; nothing invents one), so live custom submits still 503. CI runs the fake hypervisor only. mTLS is a follow-up. | +| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `live_harvest_wired`, `registered_custom`) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Staging:** the agent booted Firecracker colocated on `cortex-staging` (nested DO KVM, validated) and the runbook's § 4 fail-closed matrix came back green; image digests are operator state on that host (computed from images built outside this repo; nothing in git invents one), the § 5 happy path and § 6 sign-off are still to be recorded, and nested KVM stays fragile (boot fails → provision a dedicated KVM host, the production preference). Not in production. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index a9ef869ae..c2aa3db59 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -423,8 +423,11 @@ rule set, request) — never a host path, a key, or a judge origin. Two orchestrators exist: `UnwiredVmOrchestrator` (the default; refuses, names `PROOF_VM_ORCHESTRATOR_URL` / `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`) and the live `FirecrackerOrchestrator` (`crates/proof-vm-fc`), a thin HTTPS client of -the `proof-vm-orchestrator` agent on a **dedicated KVM host** (never the -control-plane droplet, never a Lium pod, never nested). The host prefers it +the `proof-vm-orchestrator` agent on a host with a working `/dev/kvm` — a +**dedicated KVM host** in production; on staging the agent may share the +control-plane droplet when nested KVM boots (validated on `cortex-staging`; +fragile — if the boot fails, provision metal); never a Lium pod, never an +emulator. The host prefers it when `PROOF_VM_ORCHESTRATOR_URL` (https; plain http only on loopback) and `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; the bearer is a file re-read per request and never logged; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM VM rootfs diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index 1c91565b6..2c998e4ea 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -1,4 +1,4 @@ -# Runbook — Proof topic-VM orchestrator (Firecracker on a dedicated KVM host) +# Runbook — Proof topic-VM orchestrator (Firecracker + jailer agent on a KVM host) Operator procedure for the `proof-vm-orchestrator` agent that boots one Firecracker RLM microVM per Proof topic and a **sister** Firecracker guest @@ -12,7 +12,7 @@ Staging wire + probes: § DigitalOcean staging and ## What runs where ```text -master droplet (DO) dedicated KVM host (bare metal, /dev/kvm) +control plane (proof-challenge, master) KVM host = wherever /dev/kvm works (see below) ┌─────────────────────────────────┐ ┌────────────────────────────────────────────┐ │ proof-challenge │ HTTPS │ proof-vm-orchestrator (systemd, :8200) │ │ RlmScorer → RunnerRegistry │ bearer │ bearer file · one running VM per topic_id │ @@ -26,11 +26,28 @@ master droplet (DO) dedicated KVM host (bare metal, /de └─────────────────────────────────┘ └────────────────────────────────────────────┘ ``` +**Where the agent runs.** Anywhere `/dev/kvm` works and the agent's +`ready()` is green — the isolation boundary is the RLM microVM plus the +sister guest, not the machine they sit on: + +- **Production prefers a dedicated DigitalOcean bare-metal / KVM host** + (bare metal, its own `/dev/kvm`, reachable from the master over the VPC or + a private network). +- **Staging may colocate the agent on the control-plane droplet** when + `/dev/kvm` works there. Validated on `cortex-staging`: nested DO + virtualisation booted Firecracker and the § 4 fail-closed matrix came back + green. Nested virtualisation stays **fragile** (it depends on what the + hypervisor underneath exposes and can change with a resize or a + migration): if the boot fails or `/dev/kvm` disappears, do not patch + around it — provision a dedicated KVM host and point the CP at it. +- Never a Lium pod, never a software emulator, never anything without + `/dev/kvm` (the unit's `ConditionPathExists` refuses). + Locked by design (do not move any of it): | Rule | Where it is enforced | |------|----------------------| -| Firecracker microVMs, **sisters** (RLM VM + miner guest) on one dedicated KVM host — not the control-plane droplet, not Lium, not nested | agent runs only where `/dev/kvm` exists (`ConditionPathExists`); nothing in `proof-challenge` can exec | +| Firecracker microVMs, **sisters** (RLM VM + miner guest) run only where `/dev/kvm` works — production on a dedicated KVM host, staging colocated on the CP droplet when nested KVM boots (validated); never Lium, never emulated | agent runs only where `/dev/kvm` exists (`ConditionPathExists`); nothing in `proof-challenge` can exec | | The RLM never sees the host filesystem or secrets — only `VmJob` payloads | `proof-vm-proto` types; jobs are the signed topic, digests, rule versions; tests assert no path / key / origin in any body | | RLM image pin `PROOF_RLM_VM_IMAGE_DIGEST=sha256:…`, empty = fail-closed | `FirecrackerOrchestrator::ready()` → `NotWired` naming the var; agent re-hashes `images/sha256-.ext4` before boot | | Auth: `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`, bearer file first, never logged | client reads the file per request; agent compares SHA-256 digests in constant time, re-reads its file per request | @@ -181,23 +198,50 @@ bearer; refuses production hosts). Env overlays with placeholders only: | Piece | Host | Notes | |-------|------|-------| -| `proof-challenge` (client, `FirecrackerOrchestrator`) | the **existing staging master droplet** `base-staging` (see [`staging-testnet-e2e.md`](staging-testnet-e2e.md)), compose `role-master` + `env-staging` | nothing Firecracker on it: no `/dev/kvm`, no unit, no images; it holds the bearer file, the CA PEM, the RLM image pin, and the custom ids | -| `proof-vm-orchestrator` (agent, Firecracker + jailer) | a **KVM host with `/dev/kvm`**, reachable from the staging VPC / private network | `systemd/proof-vm-orchestrator.service` has `ConditionPathExists=/dev/kvm`; bind on the private address, HTTPS + bearer file | - -**A DigitalOcean Droplet is not that host.** Standard, CPU-optimized, and -dedicated-CPU Droplets are KVM guests that expose no nested virtualisation: -`/dev/kvm` does not exist inside them, `kvm-ok` reports the CPU cannot run -KVM, and the unit refuses to start by design. Do not try to work around it -(no nested-FC redesign, no `--no-kvm` anything): the RLM VM and the sister -guest are the isolation boundary and a software emulator is not one. Use a -DigitalOcean bare-metal / dedicated-hardware host when the account has one, -or any bare-metal KVM host elsewhere (another provider, colo), attached to -the staging VPC's private network — WireGuard from the staging master, or -VPC peering when both sides are DO. Either way the agent listens on the -private address only and TLS + bearer stay mandatory (the bearer never -crosses a network in clear). - -Check the candidate host before installing anything: +| `proof-challenge` (client, `FirecrackerOrchestrator`) | the **existing staging master droplet** (`cortex-staging`; topology in [`staging-testnet-e2e.md`](staging-testnet-e2e.md)), compose `role-master` + `env-staging` | holds the bearer file, the CA PEM, the RLM image pin, and the custom ids; it is only the HTTPS client — nothing in the container can exec Firecracker | +| `proof-vm-orchestrator` (agent, Firecracker + jailer) | **colocated on that same droplet** (validated) — or a dedicated KVM host when nested KVM does not boot | `systemd/proof-vm-orchestrator.service` on the host, `ConditionPathExists=/dev/kvm`; bind on the droplet's private address, HTTPS + bearer file | + +**Colocated staging (validated).** `cortex-staging` exposes a working +`/dev/kvm` through nested DigitalOcean virtualisation; the agent booted +Firecracker there and § 4 came back green. This is the staging default: one +droplet runs the compose stack **and** the agent as a host systemd unit. Two +things follow from the CP living in a container on the same machine: + +- The CP must reach the agent on the **droplet's private (VPC) address**, + never on loopback: `127.0.0.1` inside the `proof-challenge` container is + the container, and `FcConfig` only accepts plain `http://` on loopback + anyway. Bind the agent on the VPC IP with TLS (`PROOF_VM_AGENT_BIND=:8200`, + a certificate with that IP or name as SAN) and set + `PROOF_VM_ORCHESTRATOR_URL=https://:8200` on the CP. Open `:8200` + on the host firewall to the compose network only (`docker network ls` → + the stack's `base` network → `docker network inspect ` for its + subnet); the wire check runs on the host itself and needs nothing more. +- The RLM VM (4 vCPU / 8 GiB) and each sister (2 vCPU / 4 GiB, no NIC) + share CPU, RAM, and disk with gateway, Postgres, and both challenges. Size + the droplet for it and keep `/srv/jailer` + `/var/lib/proof-vm` on a disk + with room for the images plus ~10 GiB scratch per open topic. + +**Nested virtualisation is fragile.** What the droplet's hypervisor exposes +is not under our control and can change with a resize or a live migration. +Treat any of these as "provision metal", not as something to patch around +(no nested-FC redesign, no `--no-kvm` anything — a software emulator is not +the isolation boundary): + +- `/dev/kvm` missing or `kvm-ok` reporting KVM cannot be used → the unit + does not start (`ConditionPathExists`), agent health `ready: false`. +- `topic vm boot failed; releasing its jail` with a KVM ioctl error from + Firecracker in the agent journal / the jail's `console.log`, or the guest + never saying hello inside `PROOF_VM_AGENT_BOOT_TIMEOUT_SECS`. +- Sisters cut at the deadline on a run that fits comfortably on metal. + +**Dedicated KVM host (production preference, staging fallback).** A +DigitalOcean bare-metal / dedicated-hardware host, or any bare-metal KVM +host attached to the staging VPC's private network (WireGuard from the +master, or VPC peering when both sides are DO). Same unit, same env file; +the agent listens on the private address only and TLS + bearer stay +mandatory (the bearer never crosses a network in clear). + +Check whichever host you pick before installing anything: ```bash ls -l /dev/kvm # must exist (crw-rw---- root kvm) @@ -210,7 +254,8 @@ df -T /srv /var/lib | grep -E 'xfs|btrfs' # reflink FS preferred (see Host Budget per open topic: RLM VM 4 vCPU / 8192 MiB + sister 2 vCPU / 4096 MiB + ~10 GiB scratch. Size the host for the number of custom topics staging -will keep open at once, plus one probe VM. +will keep open at once, plus one probe VM — on the colocated droplet, on +top of the compose stack. ### 0. Preconditions on the CP @@ -228,14 +273,16 @@ The custom family is routed only when the whole live stack is up; check Nothing here is challenge content in git: the topic, its rules, and its images are operator-published documents and staged files. -### 1. KVM host +### 1. KVM host (the staging droplet itself, or the dedicated host) Follow § Host prerequisites and § Install with `deploy/env/proof-vm-orchestrator.staging.example` as `/etc/proof-vm/orchestrator.env`: -1. `PROOF_VM_AGENT_BIND=:8200`; a certificate for that address - (a private CA is fine — the CP pins its root) **with a SAN**: the CP's - rustls client refuses CN-only certificates even when curl accepts them. +1. `PROOF_VM_AGENT_BIND=:8200` — the droplet's VPC address when + colocated (the CP container cannot use loopback), the private address of + the dedicated host otherwise; a certificate for that address (a private CA + is fine — the CP pins its root) **with a SAN**: the CP's rustls client + refuses CN-only certificates even when curl accepts them. 2. Bearer: `head -c 32 /dev/urandom | base64 -w0 > /etc/proof-vm/token; chmod 0400 /etc/proof-vm/token`. 3. Stage `vmlinux`, the RLM rootfs, and the sister rootfs; `sha256sum` each; name the rootfs files `images/sha256-.ext4`; put the kernel and @@ -246,18 +293,19 @@ Follow § Host prerequisites and § Install with 5. `systemctl enable --now proof-vm-orchestrator`; the boot log shows `firecracker + jailer + /dev/kvm present; agent ready` and `bearer token file present (contents not logged)`. -6. Open `:8200` on the host firewall to the staging master's private - address only. +6. Open `:8200` on the host firewall to the CP only: the compose network's + subnet when colocated, the staging master's private address when the + host is dedicated. ### 2. Control plane (staging master) ```bash ssh root@ ; cd /opt/base ; umask 077 -# The bearer: copy the KVM host's file over the private network — never paste it on a command line. -scp root@:/etc/proof-vm/token deploy/secrets/proof/vm_orchestrator_token -scp root@:/etc/proof-vm/ca.pem deploy/secrets/proof/vm_orchestrator_ca.pem # private CA only -chown 65532:65532 deploy/secrets/proof/vm_orchestrator_token deploy/secrets/proof/vm_orchestrator_ca.pem -chmod 0400 deploy/secrets/proof/vm_orchestrator_token deploy/secrets/proof/vm_orchestrator_ca.pem +# The bearer: a second copy of the agent's file — never pasted on a command line. +install -m 0400 -o 65532 -g 65532 /etc/proof-vm/token deploy/secrets/proof/vm_orchestrator_token # colocated +install -m 0400 -o 65532 -g 65532 /etc/proof-vm/ca.pem deploy/secrets/proof/vm_orchestrator_ca.pem # private CA only +# Dedicated host instead: scp root@:/etc/proof-vm/{token,ca.pem} over the private +# network to the same paths, then chown 65532:65532 + chmod 0400 both files. # Append the keys of deploy/env/proof-challenge.staging-vm.example (values filled) to the age source # of proof-challenge.env (deploy/scripts/age-encrypt-env.sh / age-push-env.sh), then: ./deploy/scripts/materialize-env.sh @@ -339,6 +387,9 @@ Then run the § Verify cleanup probes (failed `ip tuntap`, deadline cut, Staging is "wired and tested" when all of these are in the change log with dates and the exact commands: +- [ ] placement recorded: colocated on `cortex-staging` (nested KVM boots) + or a dedicated KVM host, with `ls -l /dev/kvm` + `kvm-ok` output from + that host; none of the fragility signs above appeared during the run. - [ ] `proof-vm-wire-check.sh all` → all PASS on the staging master. - [ ] `proof-vm-wire-check.sh boot-probe` → all PASS; KVM host left clean. - [ ] every row of § 4 → the expected 503 (or 400) with the expected reason, From 58a3fcd2820c9cddb7081648ddf169ebc7348f8a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:21:02 +0000 Subject: [PATCH 10/15] docs(proof-vm): staging colo = allowed proven exception; prod = dedicated do metal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architecte GO conditionnel on #247: one wording across every placement statement (runbook, AGENTS.md item 7, deploy/AGENTS.md, both staging overlays, the generic env examples, the systemd unit comment, docs/PROOF.md, docs/ARCHITECTURE.md, docs/COMPLETENESS.md, agent and binary module docs): - staging: colocating the agent on the CP droplet with nested /dev/kvm is an allowed exception, proven on cortex-staging (§ 4 green); nested stays fragile — if the boot fails, provision metal; - production: dedicated DO metal preferred — never colocated on the CP. No "a Droplet is not one / no nested" claim remains. Fail-closed probes, matrix, and harness unchanged. Co-authored-by: Mathis --- AGENTS.md | 2 +- bins/proof-vm-orchestrator/src/main.rs | 5 +- crates/proof-vm-agent/src/lib.rs | 9 ++-- deploy/AGENTS.md | 18 ++++--- deploy/env/proof-challenge.env.example | 7 +-- deploy/env/proof-challenge.staging-vm.example | 7 +-- deploy/env/proof-vm-orchestrator.env.example | 9 ++-- .../env/proof-vm-orchestrator.staging.example | 13 ++--- deploy/systemd/proof-vm-orchestrator.service | 9 ++-- docs/ARCHITECTURE.md | 2 +- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 11 +++-- docs/runbooks/proof-vm-orchestrator.md | 49 ++++++++++--------- 13 files changed, 78 insertions(+), 65 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 140dd1f9d..58a69675c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 4. **Bounty — fail-closed scorer:** the CortexLM/backend public feed is the only scorer. With no readable `BOUNTY_BACKEND_PUBLIC_URL`, `POST /v1/reports` must answer **503** and the emitter must pay **nobody** — it still covers `E` with `NoScore(ChallengeInternal)`, because a paid challenge with no leaves 409s the seal for every challenge. `BOUNTY_FORCE_SIM` is retired — do not reintroduce an offline bounty scorer. See [`docs/BOUNTY.md`](docs/BOUNTY.md). 5. **Proof — submit:** `POST /v1/submissions` with a `topic_id`. Missing/unknown/not-open → **400** (no row); a custom topic without `artifact_uri` → **400** (no row). Empty `eval_image_digest`, missing/closed/misconfigured RLM judge `InferenceOffer`, missing judge API key, spoofed topic origin, missing/closed/non-`1x` `EvalExecutorOffer` (Lium path), zero open topics, or an unsealed baseline → **503**. Miners submit claim + code + FLOPs + artifact; they do not bind the judge offer or the executor offer. Contamination / empty manifest persist **rejected** without rent; on custom topics the runner's measured `flops_used` over the budget or over the miner's `declared_flops` persists **rejected** after the run, and a report without a measurement is **503** (no row). `GET /v1/proof/topics` must never leak holdout records. 6. **Proof — executor:** `GET /v1/proof/executor` is always 200 (`ready` + `reason`); `POST /v1/admin/proof/executor` (operator bearer) rotates or closes the live `1x` offer and 400s anything the pin refuses. Harvest rents the offer's `lium_template_id` at exactly `1x` (any other `rent_gpu_count` aborts before the rent) under `max_proof_deadline_s`; a run cut at the deadline is **503 + `stdout_tail`**. `PROOF_HARVEST_*` env only hot-swaps under the pin ceilings. Never a live Lium rent in CI. -7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a host with a working `/dev/kvm` (`proof-vm-orchestrator`, HTTPS + bearer **file**) — production prefers a dedicated DO bare-metal / KVM host; staging may colocate the agent on the CP droplet when nested KVM boots (validated on `cortex-staging`; nested stays fragile — if the boot fails, provision metal); never on Lium, never emulated; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Probe the wire with `GET /v1/admin/proof/vm-orchestrator` (operator bearer, loopback) or `deploy/scripts/proof-vm-wire-check.sh` (`all`, `boot-probe`, `matrix` + `submit-probe --expect 503 --reason …`). Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md) § DigitalOcean staging. +7. **Proof — topic VMs (custom family):** the RLM runs in one Firecracker microVM per `topic_id` on a host with a working `/dev/kvm` (`proof-vm-orchestrator`, HTTPS + bearer **file**) — production: dedicated DO metal preferred, never colocated on the CP; staging: colocating the agent on the CP droplet with nested `/dev/kvm` is an allowed exception, proven on `cortex-staging` (nested stays fragile — if the boot fails, provision metal); never on Lium, never emulated; every paid run is a **sister** Firecracker guest with **no network**, and the host stamps `sandboxed` / guest-measured `flops_used` on the report. `PROOF_VM_ORCHESTRATOR_URL` unset → `UnwiredVmOrchestrator` (503); URL set but token file missing/empty, `PROOF_RLM_VM_IMAGE_DIGEST` unpinned, agent down, or a `firecracker_required` run without the sister attestation → **503, no row, no host fallback**. `PROOF_VM_RUNNER_CUSTOM_IDS` is the only thing that registers a runner. The custom family is wired from that env alone — live orchestrator selected + ≥1 id → `FamilyMux::custom_only` when no Lium harvest is wired (custom topics score; `nll` / `throughput` → **503**, no row); never stage a placeholder Lium key to open custom topics, and the unwired stub never carries a mux. `/v1/status` keeps the families apart: `live_harvest_wired` is the **Lium harvest only** (never true because a custom mux exists); the custom family is `custom_family_wired` / `registered_custom` / `custom_ready`. Hard `topic_id ↔ VM` bind on both sides (agent 409 `topic_mismatch`). Do not invent an RLM / sister image digest. **Zero live Firecracker in CI** — every test uses the fake hypervisor. Probe the wire with `GET /v1/admin/proof/vm-orchestrator` (operator bearer, loopback) or `deploy/scripts/proof-vm-wire-check.sh` (`all`, `boot-probe`, `matrix` + `submit-probe --expect 503 --reason …`). Runbook: [`docs/runbooks/proof-vm-orchestrator.md`](docs/runbooks/proof-vm-orchestrator.md) § DigitalOcean staging. 8. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). **Never host Sim in staging/prod** for live scoring. `PROOF_FORCE_SIM=1` is CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live Proof rent requires a digest pin in `config/proof-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. Do not invent `eval_image_digest`. diff --git a/bins/proof-vm-orchestrator/src/main.rs b/bins/proof-vm-orchestrator/src/main.rs index bf46f2e23..d0b9dcace 100644 --- a/bins/proof-vm-orchestrator/src/main.rs +++ b/bins/proof-vm-orchestrator/src/main.rs @@ -1,6 +1,7 @@ //! `proof-vm-orchestrator` — Firecracker topic-VM agent for a **KVM host** -//! (HTTPS `:8200`): a dedicated host in production, or the control-plane -//! droplet itself on staging when nested KVM boots there. +//! (HTTPS `:8200`): dedicated DO metal in production (never colocated on the +//! control plane); on staging the control-plane droplet itself with nested +//! `/dev/kvm` is an allowed, proven exception. //! //! The Proof control plane (`proof-challenge`, `FirecrackerOrchestrator`) //! is its only client. It boots one jailed RLM microVM per topic from the diff --git a/crates/proof-vm-agent/src/lib.rs b/crates/proof-vm-agent/src/lib.rs index 0e78e9510..c18b84a4d 100644 --- a/crates/proof-vm-agent/src/lib.rs +++ b/crates/proof-vm-agent/src/lib.rs @@ -1,9 +1,10 @@ //! `proof-vm-orchestrator` agent library. //! -//! The agent runs on a host with a working `/dev/kvm` — a **dedicated KVM -//! host** in production, the control-plane droplet itself on staging when -//! nested KVM boots there; never a Lium pod — and is the only thing that -//! talks to Firecracker. +//! The agent runs on a host with a working `/dev/kvm` — **dedicated DO +//! metal** in production (never colocated on the control plane); on staging +//! the control-plane droplet itself with nested `/dev/kvm` is an allowed, +//! proven exception; never a Lium pod — and is the only thing that talks to +//! Firecracker. //! The Proof control plane reaches it over HTTPS with a bearer read from a //! file ([`BearerAuth`]) and drives four verbs (`proof_vm_proto::paths`): //! diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index b75d22fc1..88779d392 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -71,10 +71,11 @@ gaps. See [`docs/WHITEPAPER.md`](../docs/WHITEPAPER.md). Custom-family topics run their RLM in one Firecracker microVM per `topic_id` and every miner run in a **sister** Firecracker guest with no network — on a -host with a working `/dev/kvm`: production prefers a **dedicated DO -bare-metal / KVM host**; staging may colocate the agent on the CP droplet -when nested KVM boots (validated on `cortex-staging`; nested stays fragile — -if the boot fails, provision metal); never Lium. That host runs +host with a working `/dev/kvm`: production — **dedicated DO metal +preferred, never colocated on the CP**; staging — colocating the agent on +the CP droplet with nested `/dev/kvm` is an **allowed exception, proven** on +`cortex-staging` (nested stays fragile — if the boot fails, provision metal); +never Lium. That host runs `proof-vm-orchestrator` as a systemd unit ([`systemd/proof-vm-orchestrator.service`](systemd/proof-vm-orchestrator.service), env [`env/proof-vm-orchestrator.env.example`](env/proof-vm-orchestrator.env.example)), @@ -95,10 +96,11 @@ Procedure and the mandatory submission verification: [`docs/runbooks/proof-vm-orchestrator.md`](../docs/runbooks/proof-vm-orchestrator.md). **Staging wire (DO):** the CP is the existing staging master; the agent runs -as a host systemd unit on the same droplet (`cortex-staging`, nested KVM -validated) bound on the VPC address the CP container reaches over HTTPS — or -on a dedicated KVM host when nested KVM does not boot. Overlays with -placeholders only: +as a host systemd unit on the same droplet (`cortex-staging`, nested +`/dev/kvm` — the allowed, proven staging exception) bound on the VPC address +the CP container reaches over HTTPS — or on dedicated DO metal when nested +KVM does not boot (production never colocates). Overlays with placeholders +only: [`env/proof-challenge.staging-vm.example`](env/proof-challenge.staging-vm.example) (CP) and [`env/proof-vm-orchestrator.staging.example`](env/proof-vm-orchestrator.staging.example) diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index f6c6c6751..80d997b66 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -109,9 +109,10 @@ PROOF_SIM_STUB_WIN=false # PROOF_ARTEFACT_ROOT=/var/lib/proof/artefacts # # Topic-VM orchestrator: the RLM runs inside one Firecracker microVM per -# topic on a host with /dev/kvm (production: a dedicated KVM host; staging -# may colocate the agent on this droplet when nested KVM boots; never a Lium -# pod), and every miner run happens in a SISTER Firecracker guest with no +# topic on a host with /dev/kvm (production: dedicated DO metal, never +# colocated on the CP; staging: colocating the agent on this droplet with +# nested /dev/kvm is an allowed, proven exception; never a Lium pod), and +# every miner run happens in a SISTER Firecracker guest with no # network. This container is only the client (`FirecrackerOrchestrator`, # crates/proof-vm-fc) and reaches the agent on its private address, never on # the container's loopback; the agent is `proof-vm-orchestrator` diff --git a/deploy/env/proof-challenge.staging-vm.example b/deploy/env/proof-challenge.staging-vm.example index b800da8a2..c954d39cf 100644 --- a/deploy/env/proof-challenge.staging-vm.example +++ b/deploy/env/proof-challenge.staging-vm.example @@ -4,9 +4,10 @@ # droplet (`cortex-staging`, compose role-master + env-staging): the CLIENT # side of the Proof topic-VM orchestrator. proof-challenge only talks HTTPS # to `proof-vm-orchestrator`, a host systemd unit — on staging colocated on -# this same droplet (nested KVM validated; fragile — if the boot fails, -# provision a dedicated KVM host), in production on a dedicated DO -# bare-metal / KVM host (deploy/env/proof-vm-orchestrator.staging.example). +# this same droplet with nested /dev/kvm (allowed exception, proven on +# cortex-staging; fragile — if the boot fails, provision metal), in +# production on dedicated DO metal, never colocated on the CP +# (deploy/env/proof-vm-orchestrator.staging.example). # # Use: append these keys to the age-encrypted source of proof-challenge.env, # re-materialize, restart proof-challenge, then diff --git a/deploy/env/proof-vm-orchestrator.env.example b/deploy/env/proof-vm-orchestrator.env.example index 9163aef56..c41cd44c6 100644 --- a/deploy/env/proof-vm-orchestrator.env.example +++ b/deploy/env/proof-vm-orchestrator.env.example @@ -1,9 +1,10 @@ # operator-managed, never committed to git. # Environment for /etc/proof-vm/orchestrator.env on the host that runs the -# agent (deploy/systemd/proof-vm-orchestrator.service): a dedicated KVM host -# in production; on staging the control-plane droplet itself when nested KVM -# boots (see docs/runbooks/proof-vm-orchestrator.md). Not a compose env file; -# never a Lium pod. +# agent (deploy/systemd/proof-vm-orchestrator.service): dedicated DO metal in +# production (never colocated on the CP); on staging the control-plane +# droplet itself with nested /dev/kvm is an allowed, proven exception (see +# docs/runbooks/proof-vm-orchestrator.md). Not a compose env file; never a +# Lium pod. # # Nothing in this file is a secret value. The bearer is a FILE the agent # re-reads on every request (rotate by rewriting it, no restart). Owner key diff --git a/deploy/env/proof-vm-orchestrator.staging.example b/deploy/env/proof-vm-orchestrator.staging.example index 1f9da7e32..fd6a05a25 100644 --- a/deploy/env/proof-vm-orchestrator.staging.example +++ b/deploy/env/proof-vm-orchestrator.staging.example @@ -3,12 +3,13 @@ # STAGING /etc/proof-vm/orchestrator.env for the host that runs the agent # (deploy/systemd/proof-vm-orchestrator.service — a host systemd unit, not a # compose service). It needs a working /dev/kvm (ConditionPathExists): -# - staging: the staging master droplet itself (`cortex-staging`), where -# nested DO virtualisation booted Firecracker (validated). Nested KVM is -# fragile — if the boot fails or /dev/kvm goes away, do not patch around -# it: provision a dedicated KVM host and repoint the CP. -# - production: a dedicated DO bare-metal / KVM host on the VPC or a -# private network (WireGuard / peering). +# - staging: the staging master droplet itself (`cortex-staging`) with +# nested /dev/kvm — an allowed exception, proven (nested DO +# virtualisation booted Firecracker there). Nested KVM is fragile — if +# the boot fails or /dev/kvm goes away, do not patch around it: +# provision dedicated metal and repoint the CP. +# - production: dedicated DO metal preferred, never colocated on the CP; +# on the VPC or a private network (WireGuard / peering). # Either way bind the agent on the host's PRIVATE address with TLS; when # colocated that is the droplet's VPC IP (the CP container cannot use # loopback). Generic reference with every knob: diff --git a/deploy/systemd/proof-vm-orchestrator.service b/deploy/systemd/proof-vm-orchestrator.service index 3f8390a32..a37362f33 100644 --- a/deploy/systemd/proof-vm-orchestrator.service +++ b/deploy/systemd/proof-vm-orchestrator.service @@ -1,10 +1,11 @@ [Unit] Description=Proof topic-VM orchestrator (Firecracker + jailer agent; host with /dev/kvm) Documentation=https://github.com/CortexLM/cortex/blob/main/docs/runbooks/proof-vm-orchestrator.md -# Runs only where /dev/kvm works: a dedicated KVM host in production; on -# staging it may share the control-plane droplet when nested KVM boots -# (validated on cortex-staging; fragile — if the boot fails, provision metal). -# Never a Lium pod. It needs /dev/kvm, a pinned kernel + rootfs images, and +# Runs only where /dev/kvm works. Production: dedicated DO metal preferred, +# never colocated on the control plane. Staging: sharing the control-plane +# droplet with nested /dev/kvm is an allowed exception, proven on +# cortex-staging (fragile — if the boot fails, provision metal). Never a +# Lium pod. It needs /dev/kvm, a pinned kernel + rootfs images, and # CAP_NET_ADMIN for the per-VM TAP + nftables allowlist. The control plane # reaches it over HTTPS only (on its private address, never loopback from a # container). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1f051740b..5021f0e1c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -69,7 +69,7 @@ terminates in the host reverse proxy, not in the gateway process. | `validator` | Fetch/mirror bundle, verify, recompute, peer cross-check, CRV4 submit, dissent | | `bounty-challenge` | **Master-only:** internal pair/reports/adjudicate; **reads** CortexLM/backend public API for scoring and signs leaves from those rows. An unreadable feed pays nobody — `E` is covered with `ChallengeInternal`, share burns to uid 0 — rather than scoring offline | | `proof-challenge` | **Master-only:** signed topics, holdout loading, evaluation orchestration. Library payout is a sum of WTA/discovery topic masses; the binary has no automatic leaf-emission loop yet | -| `proof-vm-orchestrator` | **Host with a working `/dev/kvm`** — a dedicated KVM host in production; staging may colocate it on the CP droplet when nested KVM boots (validated; fragile → provision metal if the boot fails); never Lium: Firecracker + jailer agent behind HTTPS + a bearer file. One RLM microVM per Proof topic from the digest the control plane pins, sister miner guest with no network per paid run, host-stamped `sandboxed` / `flops_used`. Client side is `proof-vm-fc::FirecrackerOrchestrator`; runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | +| `proof-vm-orchestrator` | **Host with a working `/dev/kvm`** — production: dedicated DO metal preferred, never colocated on the CP; staging: colocation on the CP droplet with nested `/dev/kvm` is an allowed exception, proven on `cortex-staging` (fragile → provision metal if the boot fails); never Lium: Firecracker + jailer agent behind HTTPS + a bearer file. One RLM microVM per Proof topic from the digest the control plane pins, sister miner guest with no network per paid run, host-stamped `sandboxed` / `flops_used`. Client side is `proof-vm-fc::FirecrackerOrchestrator`; runbook [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md) | | `updater` | Digest-pinned rollouts via `docker-socket-proxy` (master) | | `trustroot` | Offline keygen / sign / verify for owner-signed TOML | | `bundle` | SCALE types, seal, verify (`PROTOCOL_VERSION`) | diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 2f141cd36..182d6f087 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -86,7 +86,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | | RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. `/v1/status` reports the families apart — `live_harvest_wired` is Lium only; `custom_family_wired` / `registered_custom` / `custom_ready` are the custom family. | -| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `live_harvest_wired`, `registered_custom`) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Staging:** the agent booted Firecracker colocated on `cortex-staging` (nested DO KVM, validated) and the runbook's § 4 fail-closed matrix came back green; image digests are operator state on that host (computed from images built outside this repo; nothing in git invents one), the § 5 happy path and § 6 sign-off are still to be recorded, and nested KVM stays fragile (boot fails → provision a dedicated KVM host, the production preference). Not in production. CI runs the fake hypervisor only. mTLS is a follow-up. | +| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `live_harvest_wired`, `registered_custom`) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Staging:** the agent booted Firecracker colocated on `cortex-staging` (nested DO `/dev/kvm` — an allowed exception, proven) and the runbook's § 4 fail-closed matrix came back green; image digests are operator state on that host (computed from images built outside this repo; nothing in git invents one), the § 5 happy path and § 6 sign-off are still to be recorded, and nested KVM stays fragile (boot fails → provision metal). **Production:** dedicated DO metal preferred, never colocated on the CP; not deployed. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index c2aa3db59..c15298eac 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -423,11 +423,12 @@ rule set, request) — never a host path, a key, or a judge origin. Two orchestrators exist: `UnwiredVmOrchestrator` (the default; refuses, names `PROOF_VM_ORCHESTRATOR_URL` / `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`) and the live `FirecrackerOrchestrator` (`crates/proof-vm-fc`), a thin HTTPS client of -the `proof-vm-orchestrator` agent on a host with a working `/dev/kvm` — a -**dedicated KVM host** in production; on staging the agent may share the -control-plane droplet when nested KVM boots (validated on `cortex-staging`; -fragile — if the boot fails, provision metal); never a Lium pod, never an -emulator. The host prefers it +the `proof-vm-orchestrator` agent on a host with a working `/dev/kvm` — +production: **dedicated DO metal preferred, never colocated on the CP**; +staging: colocating the agent on the control-plane droplet with nested +`/dev/kvm` is an **allowed exception, proven** on `cortex-staging` (fragile +— if the boot fails, provision metal); never a Lium pod, never an emulator. +The host prefers it when `PROOF_VM_ORCHESTRATOR_URL` (https; plain http only on loopback) and `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; the bearer is a file re-read per request and never logged; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM VM rootfs diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index 2c998e4ea..548ade940 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -26,20 +26,20 @@ control plane (proof-challenge, master) KVM host = wherever /dev/kvm works └─────────────────────────────────┘ └────────────────────────────────────────────┘ ``` -**Where the agent runs.** Anywhere `/dev/kvm` works and the agent's +**Where the agent runs.** Only where `/dev/kvm` works and the agent's `ready()` is green — the isolation boundary is the RLM microVM plus the sister guest, not the machine they sit on: -- **Production prefers a dedicated DigitalOcean bare-metal / KVM host** - (bare metal, its own `/dev/kvm`, reachable from the master over the VPC or - a private network). -- **Staging may colocate the agent on the control-plane droplet** when - `/dev/kvm` works there. Validated on `cortex-staging`: nested DO +- **Production: dedicated DigitalOcean metal preferred — never colocated on + the CP.** A bare-metal / dedicated KVM host with its own `/dev/kvm`, + reachable from the master over the VPC or a private network. +- **Staging: colocating the agent on the CP droplet with nested `/dev/kvm` + is an allowed exception, proven.** On `cortex-staging` nested DO virtualisation booted Firecracker and the § 4 fail-closed matrix came back green. Nested virtualisation stays **fragile** (it depends on what the hypervisor underneath exposes and can change with a resize or a migration): if the boot fails or `/dev/kvm` disappears, do not patch - around it — provision a dedicated KVM host and point the CP at it. + around it — provision dedicated metal and point the CP at it. - Never a Lium pod, never a software emulator, never anything without `/dev/kvm` (the unit's `ConditionPathExists` refuses). @@ -47,7 +47,7 @@ Locked by design (do not move any of it): | Rule | Where it is enforced | |------|----------------------| -| Firecracker microVMs, **sisters** (RLM VM + miner guest) run only where `/dev/kvm` works — production on a dedicated KVM host, staging colocated on the CP droplet when nested KVM boots (validated); never Lium, never emulated | agent runs only where `/dev/kvm` exists (`ConditionPathExists`); nothing in `proof-challenge` can exec | +| Firecracker microVMs, **sisters** (RLM VM + miner guest) run only where `/dev/kvm` works — production on dedicated DO metal (never colocated on the CP); staging colocated on the CP droplet with nested `/dev/kvm` as the allowed, proven exception; never Lium, never emulated | agent runs only where `/dev/kvm` exists (`ConditionPathExists`); nothing in `proof-challenge` can exec | | The RLM never sees the host filesystem or secrets — only `VmJob` payloads | `proof-vm-proto` types; jobs are the signed topic, digests, rule versions; tests assert no path / key / origin in any body | | RLM image pin `PROOF_RLM_VM_IMAGE_DIGEST=sha256:…`, empty = fail-closed | `FirecrackerOrchestrator::ready()` → `NotWired` naming the var; agent re-hashes `images/sha256-.ext4` before boot | | Auth: `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE`, bearer file first, never logged | client reads the file per request; agent compares SHA-256 digests in constant time, re-reads its file per request | @@ -199,13 +199,14 @@ bearer; refuses production hosts). Env overlays with placeholders only: | Piece | Host | Notes | |-------|------|-------| | `proof-challenge` (client, `FirecrackerOrchestrator`) | the **existing staging master droplet** (`cortex-staging`; topology in [`staging-testnet-e2e.md`](staging-testnet-e2e.md)), compose `role-master` + `env-staging` | holds the bearer file, the CA PEM, the RLM image pin, and the custom ids; it is only the HTTPS client — nothing in the container can exec Firecracker | -| `proof-vm-orchestrator` (agent, Firecracker + jailer) | **colocated on that same droplet** (validated) — or a dedicated KVM host when nested KVM does not boot | `systemd/proof-vm-orchestrator.service` on the host, `ConditionPathExists=/dev/kvm`; bind on the droplet's private address, HTTPS + bearer file | +| `proof-vm-orchestrator` (agent, Firecracker + jailer) | **colocated on that same droplet** — the allowed, proven staging exception — or dedicated DO metal when nested KVM does not boot | `systemd/proof-vm-orchestrator.service` on the host, `ConditionPathExists=/dev/kvm`; bind on the droplet's private address, HTTPS + bearer file | -**Colocated staging (validated).** `cortex-staging` exposes a working -`/dev/kvm` through nested DigitalOcean virtualisation; the agent booted -Firecracker there and § 4 came back green. This is the staging default: one -droplet runs the compose stack **and** the agent as a host systemd unit. Two -things follow from the CP living in a container on the same machine: +**Colocated staging (allowed exception, proven).** `cortex-staging` exposes +a working `/dev/kvm` through nested DigitalOcean virtualisation; the agent +booted Firecracker there and § 4 came back green. This is the staging +default — and staging only: production never colocates the agent on the CP. +One droplet runs the compose stack **and** the agent as a host systemd unit. +Two things follow from the CP living in a container on the same machine: - The CP must reach the agent on the **droplet's private (VPC) address**, never on loopback: `127.0.0.1` inside the `proof-challenge` container is @@ -234,12 +235,13 @@ the isolation boundary): never saying hello inside `PROOF_VM_AGENT_BOOT_TIMEOUT_SECS`. - Sisters cut at the deadline on a run that fits comfortably on metal. -**Dedicated KVM host (production preference, staging fallback).** A -DigitalOcean bare-metal / dedicated-hardware host, or any bare-metal KVM -host attached to the staging VPC's private network (WireGuard from the -master, or VPC peering when both sides are DO). Same unit, same env file; -the agent listens on the private address only and TLS + bearer stay -mandatory (the bearer never crosses a network in clear). +**Dedicated DO metal (production — preferred, never colocated on the CP; +staging fallback when nested does not boot).** A DigitalOcean bare-metal / +dedicated-hardware host, or any bare-metal KVM host attached to the VPC's +private network (WireGuard from the master, or VPC peering when both sides +are DO). Same unit, same env file; the agent listens on the private address +only and TLS + bearer stay mandatory (the bearer never crosses a network in +clear). Check whichever host you pick before installing anything: @@ -387,9 +389,10 @@ Then run the § Verify cleanup probes (failed `ip tuntap`, deadline cut, Staging is "wired and tested" when all of these are in the change log with dates and the exact commands: -- [ ] placement recorded: colocated on `cortex-staging` (nested KVM boots) - or a dedicated KVM host, with `ls -l /dev/kvm` + `kvm-ok` output from - that host; none of the fragility signs above appeared during the run. +- [ ] placement recorded: colocated on `cortex-staging` (the allowed, + proven nested-KVM exception — staging only) or dedicated DO metal, with + `ls -l /dev/kvm` + `kvm-ok` output from that host; none of the + fragility signs above appeared during the run. - [ ] `proof-vm-wire-check.sh all` → all PASS on the staging master. - [ ] `proof-vm-wire-check.sh boot-probe` → all PASS; KVM host left clean. - [ ] every row of § 4 → the expected 503 (or 400) with the expected reason, From 83b85f7cc81f6bd6d3554d31895d3de4eb5d444e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:32:45 +0000 Subject: [PATCH 11/15] =?UTF-8?q?fix(deploy):=20wire-check=20=E2=80=94=20c?= =?UTF-8?q?ase-insensitive=20prod=20guard,=20strand-proof=20boot-probe,=20?= =?UTF-8?q?budget=20flops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile on #247: P1 production guard bypass: refuse_prod parsed nothing and matched the URL case-sensitively, so https://NETWORK.CORTEX.FOUNDATION reached authenticated create / submit requests. The guard now parses the host (scheme, userinfo, port, path, query, trailing dot stripped; IPv6 kept bracketed), lower-cases it, and refuses a protected host or any subdomain of one, plus a case-insensitive whole-URL match as belt and braces. It runs at env load, before any other check, on every probe. P2 lost response strands VM: PROBE_TOPIC_LIVE is set before the create goes out; a create whose answer is 000 / 5xx / unparseable is reconciled through GET /v1/vms/by-topic and any VM the agent reports is destroyed in line (probe_reconcile_destroy), and the EXIT trap does the same for anything still in flight (Ctrl-C, unconfirmed teardown). Topic state is cleared only after a confirmed destroy followed by a 404, so a retry on the probe topic is never blocked. PROOF_VM_WIRE_CHECK_FAULT= lose-create-answer is a test-only hook that drops the create's answer. P1 live probe under-declares FLOPs: submit-probe declared 1, so a live run measuring more was a flops_under_declared reject. Fail-closed probes still send 1 (nothing runs); --expect 2xx declares the topic's flops_budget read from GET /v1/proof/topics/ (--declared-flops N overrides; unreadable budget = FAIL naming the flag). Tests (fake agent): uppercase / userinfo / trailing-dot / subdomain production origins exit 2 before any request, for submit-probe and for the agent URL at env load; the lost-answer create boots one VM that is found by topic and destroyed, the topic is free, a retry passes. Runbook § 3 / § 5 updated. Co-authored-by: Mathis --- crates/proof-vm-fc/tests/wire_check_script.rs | 130 ++++++++++++++- deploy/scripts/proof-vm-wire-check.sh | 149 +++++++++++++++--- docs/runbooks/proof-vm-orchestrator.md | 11 +- 3 files changed, 265 insertions(+), 25 deletions(-) diff --git a/crates/proof-vm-fc/tests/wire_check_script.rs b/crates/proof-vm-fc/tests/wire_check_script.rs index 873a3775e..a8536923d 100644 --- a/crates/proof-vm-fc/tests/wire_check_script.rs +++ b/crates/proof-vm-fc/tests/wire_check_script.rs @@ -53,10 +53,11 @@ fn cp_env(tag: &str, agent_url: &str, digest: &str) -> (PathBuf, PathBuf) { (env, secrets) } -fn run_script(args: &[&str]) -> (bool, String) { +fn run_script_env(args: &[&str], env: &[(&str, &str)]) -> (i32, String) { let out = Command::new("bash") .arg(repo_root().join("deploy/scripts/proof-vm-wire-check.sh")) .args(args) + .envs(env.iter().copied()) .current_dir(repo_root()) .output() .expect("run script"); @@ -65,7 +66,12 @@ fn run_script(args: &[&str]) -> (bool, String) { String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr) ); - (out.status.success(), text) + (out.status.code().unwrap_or(-1), text) +} + +fn run_script(args: &[&str]) -> (bool, String) { + let (code, text) = run_script_env(args, &[]); + (code == 0, text) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -154,6 +160,70 @@ async fn agent_and_boot_probe_speak_the_router_json_and_never_print_the_bearer() "nothing outlives the probe" ); + // The agent committed the create but its answer never arrived: the probe + // must find the VM by topic and destroy it, never leave it running. + let (code, text) = tokio::task::spawn_blocking({ + let env_s = env_s.clone(); + let map = map.clone(); + move || { + run_script_env( + &[ + "boot-probe", + "--env-file", + &env_s, + "--path-map", + &map, + "--probe-topic", + "wire-probe-lost", + ], + &[("PROOF_VM_WIRE_CHECK_FAULT", "lose-create-answer")], + ) + } + }) + .await + .expect("join"); + assert_eq!(code, 1, "a lost answer is a FAIL, not a pass:\n{text}"); + assert!(text.contains("create → HTTP 000"), "{text}"); + assert!( + text.contains("reconcile: agent reports vm wire-probe-lost-"), + "{text}" + ); + assert!( + text.contains("destroyed; topic wire-probe-lost free"), + "{text}" + ); + assert!( + !text.contains("left topic wire-probe-lost in flight"), + "the in-line reconcile must clear the topic before exit:\n{text}" + ); + assert_eq!(hv.boots().len(), 2, "the lost create still booted a VM"); + assert_eq!(hv.boots()[1].topic_id, "wire-probe-lost"); + assert_eq!(hv.teardowns().len(), 2, "and it was destroyed"); + assert_eq!(hv.teardowns()[1].1, RetainPolicy::Destroy); + assert!( + agent.state.running().await.is_empty(), + "nothing outlives an ambiguous create" + ); + // A retry on the same topic is not blocked by a stranded VM. + let (ok, text) = tokio::task::spawn_blocking({ + let env_s = env_s.clone(); + let map = map.clone(); + move || { + run_script(&[ + "boot-probe", + "--env-file", + &env_s, + "--path-map", + &map, + "--probe-topic", + "wire-probe-lost", + ]) + } + }) + .await + .expect("join"); + assert!(ok, "retry after reconcile:\n{text}"); + // A stopped agent is reported, not swallowed. agent.stop(); let (ok, text) = tokio::task::spawn_blocking(move || { @@ -166,6 +236,62 @@ async fn agent_and_boot_probe_speak_the_router_json_and_never_print_the_bearer() let _ = std::fs::remove_dir_all(env.parent().expect("dir")); } +/// DNS is case-insensitive, so is the guard: a production origin is refused +/// however it is spelled, before any authenticated request, on every probe +/// that would send one. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn production_hosts_are_refused_case_insensitively_before_any_request() { + if !tools_present() { + eprintln!("skipping: bash / curl / python3 not all present"); + return; + } + for cp in [ + "https://NETWORK.CORTEX.FOUNDATION/challenge/proof", + "http://user@Chain.JoinBase.AI:8080/challenge/proof", + "https://api.cortex.foundation./v1", + "https://sub.network.cortex.foundation:443/challenge/proof", + ] { + let (code, text) = tokio::task::spawn_blocking(move || { + run_script_env( + &["submit-probe", "--cp", cp, "--topic", "x", "--expect", "400"], + &[], + ) + }) + .await + .expect("join"); + assert_eq!(code, 2, "{cp} must be refused:\n{text}"); + assert!(text.contains("refusing production host"), "{cp}: {text}"); + assert!(!text.contains("POST"), "{cp}: no request may go out:\n{text}"); + } + // The agent URL is refused at env load, before boot-probe reads the bearer. + let (secrets, _) = { + let (env, secrets) = cp_env( + "prod-agent", + "https://NETWORK.Cortex.Foundation:8200", + &pinned_template().image_digest, + ); + (secrets, env) + }; + let env_s = secrets + .parent() + .expect("dir") + .join("proof-challenge.env") + .to_string_lossy() + .into_owned(); + let map = format!("/run/base/proof={}", secrets.display()); + let (code, text) = tokio::task::spawn_blocking(move || { + run_script_env( + &["boot-probe", "--env-file", &env_s, "--path-map", &map], + &[], + ) + }) + .await + .expect("join"); + assert_eq!(code, 2, "{text}"); + assert!(text.contains("refusing production host"), "{text}"); + let _ = std::fs::remove_dir_all(secrets.parent().expect("dir")); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn env_check_fails_closed_on_unpinned_digest_and_empty_bearer() { if !tools_present() { diff --git a/deploy/scripts/proof-vm-wire-check.sh b/deploy/scripts/proof-vm-wire-check.sh index d0efb7181..998bb3e91 100755 --- a/deploy/scripts/proof-vm-wire-check.sh +++ b/deploy/scripts/proof-vm-wire-check.sh @@ -30,6 +30,8 @@ # --reason SUBSTR submit-probe: the error text must contain this # --artifact-uri URI submit-probe locator (default https://example.invalid/wire-probe.tar — never fetchable) # --no-artifact-uri submit-probe without a locator (a custom topic must answer 400, no row) +# --declared-flops N submit-probe declaration (default: 1 for fail-closed probes; the topic's +# flops_budget for --expect 2xx so a real run is not rejected flops_under_declared) # --wait SECS submit-probe --expect 201: how long the synchronous POST may take (default 900) # # Exit: 0 all PASS, 1 any FAIL, 2 refused (production host / unsafe request). @@ -50,9 +52,25 @@ warn() { YEL "WARN $*"; WARNS=$((WARNS + 1)); } fail() { RED "FAIL $*"; FAILS=$((FAILS + 1)); } PROD_HOSTS='network\.cortex\.foundation|chain\.joinbase\.ai|api\.cortex\.foundation' +# url_host URL → the host part, lower-cased (no scheme, userinfo, port, path, +# query; a trailing dot dropped; IPv6 literal kept bracketed). +url_host() { + local h="$1" + h="${h#*://}"; h="${h%%/*}"; h="${h%%\?*}"; h="${h%%\#*}"; h="${h##*@}" + if [[ "$h" == \[* ]]; then h="${h%%]*}]"; else h="${h%%:*}"; fi + h="${h%.}" + printf '%s' "$h" | tr '[:upper:]' '[:lower:]' +} +# Refuse a production origin however it is spelled: the parsed host (or any +# subdomain of a protected host) is compared lower-cased, and the whole URL is +# also matched case-insensitively as belt and braces. DNS is case-insensitive; +# the guard must be too. refuse_prod() { - if printf '%s' "$1" | grep -Eq "$PROD_HOSTS"; then - RED "refusing production host: $1" + local url="$1" host + host="$(url_host "$url")" + if printf '%s\n' "$host" | grep -Eq "^(.*\.)?(${PROD_HOSTS})$" \ + || printf '%s' "$url" | grep -Eiq "$PROD_HOSTS"; then + RED "refusing production host: $url" exit 2 fi } @@ -71,8 +89,9 @@ REASON="" ALLOW_LIVE_RUN=0 ARTIFACT_URI="https://example.invalid/wire-probe.tar" WAIT_SECS=900 +DECLARED_FLOPS="" -usage() { sed -n '2,35p' "$0"; } +usage() { sed -n '2,37p' "$0"; } [[ $# -ge 1 ]] || { usage; exit 1; } SUBCOMMAND="$1"; shift @@ -90,6 +109,7 @@ while [[ $# -gt 0 ]]; do --artifact-uri) ARTIFACT_URI="${2:?}"; shift 2 ;; --no-artifact-uri) ARTIFACT_URI=""; shift ;; --wait) WAIT_SECS="${2:?}"; shift 2 ;; + --declared-flops) DECLARED_FLOPS="${2:?}"; shift 2 ;; -h|--help) usage; exit 0 ;; *) RED "unknown arg: $1"; usage; exit 1 ;; esac @@ -152,15 +172,17 @@ map_path() { TMPDIR_WC="$(mktemp -d)" umask 077 -# A boot-probe VM must never outlive the probe, even on Ctrl-C. +# A boot-probe VM must never outlive the probe: not on Ctrl-C, and not when +# the agent committed a create whose answer never reached us. PROBE_TOPIC_LIVE +# is set BEFORE the create goes out, so the exit path can always reconcile by +# topic (GET /v1/vms/by-topic) even without a vm id; PROBE_VM is the id once +# an answer named it. Both are cleared only after a confirmed destroy + 404. PROBE_BASE=""; PROBE_VM=""; PROBE_TOPIC_LIVE=""; PROBE_HDR="" AGENT_ARGS=() on_exit() { - if [[ -n "$PROBE_VM" ]]; then - RED "boot-probe interrupted with vm $PROBE_VM up; destroying" - curl -sS -m 660 -o /dev/null -X DELETE -H 'content-type: application/json' \ - --data-binary "$(printf '{"topic_id":"%s","policy":"destroy"}' "$PROBE_TOPIC_LIVE")" \ - "${AGENT_ARGS[@]}" -K "$PROBE_HDR" "$PROBE_BASE/v1/vms/$PROBE_VM" || true + if [[ -n "$PROBE_TOPIC_LIVE" ]]; then + RED "boot-probe left topic $PROBE_TOPIC_LIVE in flight; reconciling on the agent" + probe_reconcile_destroy || true fi rm -rf "$TMPDIR_WC" } @@ -190,6 +212,59 @@ http() { HTTP_CODE="000" fi rm -f "$out" + # Test hook (crates/proof-vm-fc/tests/wire_check_script.rs): the agent + # committed the create but its answer never arrived. Never set by operators. + if [[ "${PROOF_VM_WIRE_CHECK_FAULT:-}" == "lose-create-answer" && "$method" == "POST" && "$url" == */v1/vms ]]; then + HTTP_CODE="000"; HTTP_BODY="(simulated: answer to POST /v1/vms lost)" + fi +} + +# The VM the agent holds for the probe topic, if any → $PROBE_VM ("" when none). +probe_attach() { + http GET "$PROBE_BASE/v1/vms/by-topic/$PROBE_TOPIC_LIVE" "" "${AGENT_ARGS[@]}" -K "$PROBE_HDR" + if [[ "$HTTP_CODE" == "200" ]]; then + PROBE_VM="$(jget "$HTTP_BODY" handle.vm_id)" + elif [[ "$HTTP_CODE" == "404" ]]; then + PROBE_VM="" + fi + return 0 +} + +# Destroy whatever VM the probe topic holds — the id we were told, or the one +# the agent reports for the topic when the create's answer was lost. Clears +# PROBE_VM / PROBE_TOPIC_LIVE only on a confirmed destroy followed by a 404. +# Returns 0 when the topic is verifiably free, 1 otherwise. +probe_reconcile_destroy() { + [[ -n "$PROBE_TOPIC_LIVE" ]] || return 0 + if [[ -z "$PROBE_VM" ]]; then + probe_attach + if [[ "$HTTP_CODE" == "404" ]]; then + LOG "reconcile: agent holds no vm for $PROBE_TOPIC_LIVE" + PROBE_TOPIC_LIVE="" + return 0 + fi + if [[ -z "$PROBE_VM" ]]; then + RED "reconcile: attach for $PROBE_TOPIC_LIVE → HTTP $HTTP_CODE $(printf '%s' "$HTTP_BODY" | head -c 200); check the agent by hand" + return 1 + fi + LOG "reconcile: agent reports vm $PROBE_VM for $PROBE_TOPIC_LIVE" + fi + local body + body="$(printf '{"topic_id":"%s","policy":"destroy"}' "$PROBE_TOPIC_LIVE")" + http DELETE "$PROBE_BASE/v1/vms/$PROBE_VM" "$body" "${AGENT_ARGS[@]}" -K "$PROBE_HDR" -m 660 + if [[ "$HTTP_CODE" == "200" && "$(jget "$HTTP_BODY" state)" == "destroyed" && "$(jget "$HTTP_BODY" confirmed)" == "true" ]] \ + || [[ "$HTTP_CODE" == "404" ]]; then + local destroyed="$PROBE_VM" + PROBE_VM="" + probe_attach + if [[ "$HTTP_CODE" == "404" ]]; then + LOG "reconcile: vm $destroyed destroyed; topic $PROBE_TOPIC_LIVE free" + PROBE_TOPIC_LIVE="" + return 0 + fi + fi + RED "reconcile: vm $PROBE_VM for $PROBE_TOPIC_LIVE not confirmed destroyed (HTTP $HTTP_CODE $(printf '%s' "$HTTP_BODY" | head -c 200)); on the agent host: journalctl -u proof-vm-orchestrator, ls /srv/jailer/firecracker/" + return 1 } # --------------------------------------------------------------------------- @@ -206,6 +281,8 @@ load_env() { [[ -n "$CA_PATH" ]] && CA_PATH="$(map_path "$CA_PATH")" AGENT_ARGS=() [[ -n "$CA_PATH" && -f "$CA_PATH" ]] && AGENT_ARGS+=(--cacert "$CA_PATH") + # Before any other check or request: a production agent is never probed. + [[ -n "$URL" ]] && refuse_prod "$URL" return 0 } @@ -379,8 +456,9 @@ check_cp() { if printf '%s' "$status" | grep -qF "$leak"; then fail "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/v1/status leaks '$leak'"; leaked=1; fi done if [[ -n "$URL" ]]; then - local host="${URL#*://}"; host="${host%%/*}"; host="${host%%:*}" - if printf '%s' "$status" | grep -qF "$host"; then fail "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/v1/status leaks the agent host $host"; leaked=1; fi + local host + host="$(url_host "$URL")" + if printf '%s' "$status" | grep -qiF "$host"; then fail "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/v1/status leaks the agent host $host"; leaked=1; fi fi [[ "$leaked" -eq 0 ]] && pass "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/v1/status carries no orchestrator URL, token, or path" if [[ -n "$IDS" ]]; then @@ -457,13 +535,24 @@ boot_probe() { local spec spec="$(printf '{"spec":{"topic_id":"%s","template":{"image_digest":"%s","vcpus":%s,"mem_mib":%s},"sandbox":{"firecracker_required":true,"deadline_s":60},"retain":"destroy"}}' \ "$topic" "$DIGEST" "$vcpus" "$mem")" + # From here the topic is in flight: whatever the create's answer, the exit + # path reconciles by topic and destroys what the agent holds for it. + PROBE_BASE="$base"; PROBE_HDR="$hdr"; PROBE_TOPIC_LIVE="$topic"; PROBE_VM="" http POST "$base/v1/vms" "$spec" "${AGENT_ARGS[@]}" -K "$hdr" -m 660; code="$HTTP_CODE" - if [[ "$code" != "201" ]]; then - fail "create → HTTP $code: $(printf '%s' "$HTTP_BODY" | head -c 400) (503 not_ready = image/kernel/kvm on the host; 400 bad_spec; 401 bearer)" + vm_id="" + [[ "$code" == "201" ]] && vm_id="$(jget "$HTTP_BODY" handle.vm_id)" + if [[ -z "$vm_id" ]]; then + # 000 (timeout / connection lost), 5xx, or a 201 we could not parse: the + # agent may have committed the VM. Ask it by topic and destroy any hit. + fail "create → HTTP $code: $(printf '%s' "$HTTP_BODY" | head -c 400) (503 not_ready = image/kernel/kvm on the host; 400 bad_spec; 401 bearer; 000 = answer lost)" + if probe_reconcile_destroy; then + LOG "reconciled: the agent holds nothing for $topic" + else + fail "reconcile after the ambiguous create did not free $topic" + fi return 0 fi - vm_id="$(jget "$HTTP_BODY" handle.vm_id)" - PROBE_BASE="$base"; PROBE_VM="$vm_id"; PROBE_TOPIC_LIVE="$topic"; PROBE_HDR="$hdr" + PROBE_VM="$vm_id" if [[ "$(jget "$HTTP_BODY" handle.topic_id)" == "$topic" ]]; then pass "create bound vm $vm_id to $topic"; else fail "create bound another topic: $HTTP_BODY"; fi if [[ "$(jget "$HTTP_BODY" image_digest | tr '[:upper:]' '[:lower:]')" == "$(printf '%s' "$DIGEST" | tr '[:upper:]' '[:lower:]')" ]]; then pass "agent booted the pinned image" @@ -490,12 +579,17 @@ boot_probe() { http DELETE "$base/v1/vms/$vm_id" "$body" "${AGENT_ARGS[@]}" -K "$hdr" -m 660; code="$HTTP_CODE" if [[ "$code" == "200" && "$(jget "$HTTP_BODY" state)" == "destroyed" && "$(jget "$HTTP_BODY" confirmed)" == "true" ]]; then pass "teardown destroyed $vm_id (confirmed)" - PROBE_VM="" else fail "teardown → HTTP $code $HTTP_BODY — check the KVM host: /srv/jailer/firecracker/$vm_id, nft list tables" fi http GET "$base/v1/vms/by-topic/$topic" "" "${AGENT_ARGS[@]}" -K "$hdr"; code="$HTTP_CODE" - if [[ "$code" == "404" ]]; then pass "attach after destroy → 404 (nothing left for $topic)"; else fail "attach after destroy → HTTP $code $HTTP_BODY"; fi + if [[ "$code" == "404" ]]; then + pass "attach after destroy → 404 (nothing left for $topic)" + PROBE_VM=""; PROBE_TOPIC_LIVE="" + else + fail "attach after destroy → HTTP $code $HTTP_BODY" + # The exit path retries the destroy by topic before the script ends. + fi LOG "on the KVM host: journalctl -u proof-vm-orchestrator | grep -E 'topic vm booted|torn down'; ls /srv/jailer/firecracker/ must not list $vm_id" } @@ -509,13 +603,28 @@ submit_probe() { exit 2 fi resolve_cp || return 0 + # A real run measures FLOPs in the sister and the CP rejects a run over its + # declaration (flops_under_declared). The fail-closed probes never run, so + # they declare 1; a live run declares the topic's whole budget unless told + # otherwise (over the budget is a 400 before anything runs). + local flops="${DECLARED_FLOPS:-1}" + if [[ -z "$DECLARED_FLOPS" && "$EXPECT" =~ ^2 ]]; then + http GET "$CP/v1/proof/topics/$TOPIC" "" + flops="$(jget "$HTTP_BODY" flops_budget)" + if [[ "$HTTP_CODE" != "200" || ! "$flops" =~ ^[0-9]+$ || "$flops" == "0" ]]; then + fail "cannot read flops_budget of topic $TOPIC (HTTP $HTTP_CODE); pass --declared-flops N for the live run" + return 0 + fi + LOG "live run declares the topic budget: declared_flops=$flops" + fi + [[ "$flops" =~ ^[0-9]+$ ]] || { RED "--declared-flops must be an integer"; exit 1; } local hotkey hex uri_field="" body code hotkey="$(head -c 64 /dev/zero | tr '\0' 'a')" hex="$(printf '%s' "wire-probe-$TOPIC-$(date +%s)-$$-$RANDOM" | sha256sum | awk '{print $1}')" [[ -n "$ARTIFACT_URI" ]] && uri_field="$(printf '"artifact_uri":"%s",' "$ARTIFACT_URI")" - body="$(printf '{"miner_hotkey":"%s","artifact_digest":"%s",%s"claim":"proof-vm-wire-check probe","declared_flops":1,"topic_id":"%s","manifest":{"train_dataset_ids":["wire-probe-v0"]}}' \ - "$hotkey" "$hex" "$uri_field" "$TOPIC")" - LOG "submit-probe: POST $CP/v1/submissions topic=$TOPIC expect=$EXPECT${REASON:+ reason~'$REASON'}${ARTIFACT_URI:+ artifact_uri=$ARTIFACT_URI}" + body="$(printf '{"miner_hotkey":"%s","artifact_digest":"%s",%s"claim":"proof-vm-wire-check probe","declared_flops":%s,"topic_id":"%s","manifest":{"train_dataset_ids":["wire-probe-v0"]}}' \ + "$hotkey" "$hex" "$uri_field" "$flops" "$TOPIC")" + LOG "submit-probe: POST $CP/v1/submissions topic=$TOPIC expect=$EXPECT declared_flops=$flops${REASON:+ reason~'$REASON'}${ARTIFACT_URI:+ artifact_uri=$ARTIFACT_URI}" http POST "$CP/v1/submissions" "$body" -m "$WAIT_SECS"; code="$HTTP_CODE" LOG "→ HTTP $code $(printf '%s' "$HTTP_BODY" | head -c 500)" if [[ "$code" != "$EXPECT" ]]; then diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index 548ade940..f8f890f83 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -330,7 +330,7 @@ cd /opt/base | `env` | `PROOF_VM_ORCHESTRATOR_URL` is `https://`; the bearer file (container path mapped through the compose bind mount, `--path-map`) exists and is non-empty, mode 0400 / uid 65532; `PROOF_RLM_VM_IMAGE_DIGEST` is `sha256:<64 hex>` (empty or a placeholder = FAIL — never invented); the CA file is PEM when set; every custom id is well-formed; the shape is the locked 4 / 8192; `PROOF_FORCE_SIM` is off | | `agent` | `GET /v1/health` with the bearer → `ready: true`, `hypervisor: firecracker`; no bearer → 401; wrong bearer → 401 | | `cp` | `/v1/status`: `lium`, `live_harvest_wired`, `registered_custom` ⊇ ids, no URL / token / path in the body; `/v1/proof/topics` leaks no holdout; `/v1/proof/executor` readiness; then the admin probe above — `orchestrator: firecracker`, `ready: true`, `agent.ready: true` through the CP's own rustls client | -| `boot-probe` | the agent boots the **pinned** image for a probe topic, one topic ↔ one VM, a teardown naming another topic is refused, destroy is confirmed, nothing is left for the topic. Opt-in: it boots a real 4 vCPU / 8 GiB RLM VM on the KVM host (up to 10 min, the RLM guest must say hello); no job runs, nothing is spent; Ctrl-C tears the VM down | +| `boot-probe` | the agent boots the **pinned** image for a probe topic, one topic ↔ one VM, a teardown naming another topic is refused, destroy is confirmed, nothing is left for the topic. Opt-in: it boots a real 4 vCPU / 8 GiB RLM VM on the KVM host (up to 10 min, the RLM guest must say hello); no job runs, nothing is spent. Nothing outlives it: Ctrl-C, a lost `201` (timeout, dropped connection), or an unconfirmed teardown all end in a by-topic attach + destroy before the script exits, so a retry on the same probe topic is never blocked by a stranded VM | Every check re-reads the files it names, so a fix to the bearer or the CA needs no restart; URL / digest / ids are read at boot. @@ -367,8 +367,13 @@ With everything restored and an open custom topic whose id is registered: --expect 201 --allow-live-run --artifact-uri ``` -The POST is synchronous (the RLM job runs before the 201). Evidence to -collect, in order: +The POST is synchronous (the RLM job runs before the 201). The live probe +declares the topic's whole `flops_budget` (read from `GET +/v1/proof/topics/`; `--declared-flops N` overrides) so the sister's +measurement is judged against the budget, not against the token `1` the +fail-closed probes send — a run over its own declaration is a +`flops_under_declared` reject, which is the miner rule, not a wire fault. +Evidence to collect, in order: | Step | Where | Must show | |------|-------|-----------| From 27554b142b0f701858afd00cfe7c4885e42f358b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:33:01 +0000 Subject: [PATCH 12/15] style(proof-vm-fc): rustfmt the wire-check script test Co-authored-by: Mathis --- crates/proof-vm-fc/tests/wire_check_script.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/proof-vm-fc/tests/wire_check_script.rs b/crates/proof-vm-fc/tests/wire_check_script.rs index a8536923d..7e92998b0 100644 --- a/crates/proof-vm-fc/tests/wire_check_script.rs +++ b/crates/proof-vm-fc/tests/wire_check_script.rs @@ -253,7 +253,15 @@ async fn production_hosts_are_refused_case_insensitively_before_any_request() { ] { let (code, text) = tokio::task::spawn_blocking(move || { run_script_env( - &["submit-probe", "--cp", cp, "--topic", "x", "--expect", "400"], + &[ + "submit-probe", + "--cp", + cp, + "--topic", + "x", + "--expect", + "400", + ], &[], ) }) @@ -261,7 +269,10 @@ async fn production_hosts_are_refused_case_insensitively_before_any_request() { .expect("join"); assert_eq!(code, 2, "{cp} must be refused:\n{text}"); assert!(text.contains("refusing production host"), "{cp}: {text}"); - assert!(!text.contains("POST"), "{cp}: no request may go out:\n{text}"); + assert!( + !text.contains("POST"), + "{cp}: no request may go out:\n{text}" + ); } // The agent URL is refused at env load, before boot-probe reads the bearer. let (secrets, _) = { From 6b25fb78e271df3b9d74dd91e51e2ab93ed20b1e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:34:20 +0000 Subject: [PATCH 13/15] test(proof-vm-fc): split the lost-create reconcile probe into its own test Co-authored-by: Mathis --- crates/proof-vm-fc/tests/wire_check_script.rs | 100 +++++++++++------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/crates/proof-vm-fc/tests/wire_check_script.rs b/crates/proof-vm-fc/tests/wire_check_script.rs index 7e92998b0..b1c24bea0 100644 --- a/crates/proof-vm-fc/tests/wire_check_script.rs +++ b/crates/proof-vm-fc/tests/wire_check_script.rs @@ -160,22 +160,54 @@ async fn agent_and_boot_probe_speak_the_router_json_and_never_print_the_bearer() "nothing outlives the probe" ); - // The agent committed the create but its answer never arrived: the probe - // must find the VM by topic and destroy it, never leave it running. + // A stopped agent is reported, not swallowed. + agent.stop(); + let (ok, text) = tokio::task::spawn_blocking(move || { + run_script(&["agent", "--env-file", &env_s, "--path-map", &map]) + }) + .await + .expect("join"); + assert!(!ok, "a dead agent must fail the check:\n{text}"); + assert!(text.contains("agent health → HTTP 000"), "{text}"); + let _ = std::fs::remove_dir_all(env.parent().expect("dir")); +} + +/// The agent committed the create but its answer never arrived (timeout, +/// dropped connection): the probe must find the VM by topic and destroy it, +/// never leave it running, and a retry on the same topic must not be blocked. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_lost_create_answer_is_reconciled_by_topic_and_destroyed() { + if !tools_present() { + eprintln!("skipping: bash / curl / python3 not all present"); + return; + } + let agent_token = token_file("wire-script-lost", TOKEN); + let agent = FakeAgent::serve(FakeHypervisor::new(0.8), &agent_token).await; + let digest = pinned_template().image_digest; + let (env, secrets) = cp_env("lost", &agent.url(), &digest); + let env_s = env.to_string_lossy().into_owned(); + let map = format!("/run/base/proof={}", secrets.display()); + let probe_args = |env_s: &str, map: &str| -> Vec { + [ + "boot-probe", + "--env-file", + env_s, + "--path-map", + map, + "--probe-topic", + "wire-probe-lost", + ] + .iter() + .map(ToString::to_string) + .collect() + }; + let (code, text) = tokio::task::spawn_blocking({ - let env_s = env_s.clone(); - let map = map.clone(); + let argv = probe_args(&env_s, &map); move || { + let refs: Vec<&str> = argv.iter().map(String::as_str).collect(); run_script_env( - &[ - "boot-probe", - "--env-file", - &env_s, - "--path-map", - &map, - "--probe-topic", - "wire-probe-lost", - ], + &refs, &[("PROOF_VM_WIRE_CHECK_FAULT", "lose-create-answer")], ) } @@ -196,43 +228,29 @@ async fn agent_and_boot_probe_speak_the_router_json_and_never_print_the_bearer() !text.contains("left topic wire-probe-lost in flight"), "the in-line reconcile must clear the topic before exit:\n{text}" ); - assert_eq!(hv.boots().len(), 2, "the lost create still booted a VM"); - assert_eq!(hv.boots()[1].topic_id, "wire-probe-lost"); - assert_eq!(hv.teardowns().len(), 2, "and it was destroyed"); - assert_eq!(hv.teardowns()[1].1, RetainPolicy::Destroy); + let hv = &agent.hypervisor; + assert_eq!(hv.boots().len(), 1, "the lost create still booted a VM"); + assert_eq!(hv.boots()[0].topic_id, "wire-probe-lost"); + assert_eq!(hv.teardowns().len(), 1, "and it was destroyed"); + assert_eq!(hv.teardowns()[0].1, RetainPolicy::Destroy); assert!( agent.state.running().await.is_empty(), "nothing outlives an ambiguous create" ); - // A retry on the same topic is not blocked by a stranded VM. - let (ok, text) = tokio::task::spawn_blocking({ - let env_s = env_s.clone(); - let map = map.clone(); + + let (code, text) = tokio::task::spawn_blocking({ + let argv = probe_args(&env_s, &map); move || { - run_script(&[ - "boot-probe", - "--env-file", - &env_s, - "--path-map", - &map, - "--probe-topic", - "wire-probe-lost", - ]) + let refs: Vec<&str> = argv.iter().map(String::as_str).collect(); + run_script_env(&refs, &[]) } }) .await .expect("join"); - assert!(ok, "retry after reconcile:\n{text}"); - - // A stopped agent is reported, not swallowed. - agent.stop(); - let (ok, text) = tokio::task::spawn_blocking(move || { - run_script(&["agent", "--env-file", &env_s, "--path-map", &map]) - }) - .await - .expect("join"); - assert!(!ok, "a dead agent must fail the check:\n{text}"); - assert!(text.contains("agent health → HTTP 000"), "{text}"); + assert_eq!(code, 0, "retry after reconcile:\n{text}"); + assert_eq!(hv.boots().len(), 2); + assert_eq!(hv.teardowns().len(), 2); + assert!(agent.state.running().await.is_empty()); let _ = std::fs::remove_dir_all(env.parent().expect("dir")); } From 2acff57d6f12b3b589ca8a2bcec35568fba16efe Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 21:57:37 +0000 Subject: [PATCH 14/15] fix(deploy): wire-check gates custom on custom_family_wired / custom_ready, not lium MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After #246 the custom family is wired from the topic-VM env alone and `live_harvest_wired` is the Lium harvest only (nll / throughput), so a custom-only host reads false by design. The harness `cp` step no longer FAILs on it: the flag is logged as informational, and the custom family is gated on `custom_family_wired` (FAIL when ids are set but the family is not routed), `registered_custom` ⊇ ids, and `custom_ready` ⊇ ids (registered but not ready = bearer file / image pin on this host). The admin probe log line shows the same three fields. `VmOrchestratorReport` carries `custom_family_wired` next to `registered_custom`, and its `live_harvest_wired` is the host's Lium-only answer. Runbook (§ Wire the control plane, admin-probe field table, § 0 preconditions, § 3 cp row), the CP staging overlay, docs/PROOF.md, and docs/COMPLETENESS.md say the same. Verified against a custom-only loopback proof-challenge wired to a loopback agent: custom gates PASS with live_harvest_wired=false logged, not failed. Co-authored-by: Mathis --- deploy/env/proof-challenge.staging-vm.example | 6 ++-- deploy/scripts/proof-vm-wire-check.sh | 28 ++++++++++++++----- docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 6 ++-- docs/runbooks/proof-vm-orchestrator.md | 21 +++++++------- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/deploy/env/proof-challenge.staging-vm.example b/deploy/env/proof-challenge.staging-vm.example index c954d39cf..61bd9ca0b 100644 --- a/deploy/env/proof-challenge.staging-vm.example +++ b/deploy/env/proof-challenge.staging-vm.example @@ -49,8 +49,10 @@ PROOF_RLM_VM_MEM_MIB=8192 # Custom metric ids the generic VmBackedRunner serves on this host: exactly # the ids named by the signed staging topics (comma-separated). Registration # is an operator action — empty = empty registry = every custom topic 503. -# Also requires live_harvest_wired (LIUM_API_KEY + LIUM_SSH_PUBLIC_KEY_FILE): -# without the harvest the custom family is not routed at all. +# The custom family is wired from these four variables alone: no Lium +# credentials needed (live_harvest_wired is Lium-only and reads false on a +# custom-only host — expected). Check custom_family_wired / registered_custom / +# custom_ready on GET /v1/status instead. PROOF_VM_RUNNER_CUSTOM_IDS=REPLACE_WITH_CUSTOM_ID_FROM_THE_SIGNED_STAGING_TOPIC # Operator bearers for /v1/admin/* (topics, executor, vm-orchestrator probe). diff --git a/deploy/scripts/proof-vm-wire-check.sh b/deploy/scripts/proof-vm-wire-check.sh index 998bb3e91..37c1191e3 100755 --- a/deploy/scripts/proof-vm-wire-check.sh +++ b/deploy/scripts/proof-vm-wire-check.sh @@ -440,16 +440,27 @@ check_cp() { fail "GET /v1/status → $code" return 0 fi - local status="$HTTP_BODY" harvest can_score registered + local status="$HTTP_BODY" harvest can_score registered custom_wired custom_ready harvest="$(jget "$status" live_harvest_wired)"; can_score="$(jget "$status" can_score)" - registered="$(jget "$status" registered_custom)" - LOG "status: eval_backend=$(jget "$status" eval_backend) live_harvest_wired=$harvest can_score=$can_score baseline_sealed=$(jget "$status" baseline_sealed)" - LOG "status: open_topics=$(jget "$status" open_topics) scorable_topics=$(jget "$status" scorable_topics) registered_custom=$registered" + registered="$(jget "$status" registered_custom)"; custom_wired="$(jget "$status" custom_family_wired)" + custom_ready="$(jget "$status" custom_ready)" + LOG "status: eval_backend=$(jget "$status" eval_backend) live_harvest_wired=$harvest custom_family_wired=$custom_wired can_score=$can_score baseline_sealed=$(jget "$status" baseline_sealed)" + LOG "status: open_topics=$(jget "$status" open_topics) scorable_topics=$(jget "$status" scorable_topics) registered_custom=$registered custom_ready=$custom_ready" [[ "$(jget "$status" eval_backend)" == "lium" ]] || fail "eval_backend is not lium (sim never hosts staging scoring)" + # live_harvest_wired is the Lium harvest (nll / throughput) and nothing else: + # the custom family is wired from the topic-VM env on its own, so on a + # custom-only host this reads false by design. Informational, never a FAIL. if [[ "$harvest" == "true" ]]; then - pass "live_harvest_wired: the custom family is routed (LIUM_API_KEY + LIUM_SSH_PUBLIC_KEY_FILE present)" + LOG "live_harvest_wired=true: the Lium harvest scores nll / throughput as well" else - fail "live_harvest_wired=false: the custom family is not routed and registered_custom stays [] whatever PROOF_VM_RUNNER_CUSTOM_IDS says" + LOG "live_harvest_wired=false: Lium-only flag; nll / throughput topics 503 here, the custom family is judged by custom_family_wired / registered_custom / custom_ready" + fi + if [[ -n "$IDS" ]]; then + if [[ "$custom_wired" == "true" ]]; then + pass "custom_family_wired: the custom family is routed over the topic-vm orchestrator" + else + fail "custom_family_wired=$custom_wired with PROOF_VM_RUNNER_CUSTOM_IDS set: URL unset or refused, or no id registered (boot log: 'firecracker topic-vm orchestrator wired' + 'vm-backed runner registered')" + fi fi local leak leaked=0 for leak in "vm_orchestrator" "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/run/base" "Bearer " "PROOF_VM_ORCHESTRATOR"; do @@ -462,11 +473,13 @@ check_cp() { fi [[ "$leaked" -eq 0 ]] && pass "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/v1/status carries no orchestrator URL, token, or path" if [[ -n "$IDS" ]]; then - local id missing=0 + local id missing=0 not_ready=0 while IFS= read -r id; do printf '%s' "$registered" | grep -qF "\"$id\"" || { fail "registered_custom lacks $id (PROOF_VM_RUNNER_CUSTOM_IDS says it is served)"; missing=1; } + printf '%s' "$custom_ready" | grep -qF "\"$id\"" || { fail "custom_ready lacks $id: registered but its runner cannot run now (bearer file / image pin on this host — see the admin probe)"; not_ready=1; } done < <(split_ids "$IDS") [[ "$missing" -eq 0 ]] && pass "registered_custom lists every id in PROOF_VM_RUNNER_CUSTOM_IDS" + [[ "$not_ready" -eq 0 ]] && pass "custom_ready lists every id: the runner over the orchestrator is ready now" fi http GET "$CP/v1/proof/topics" ""; code="$HTTP_CODE" @@ -500,6 +513,7 @@ check_cp() { orch="$(jget "$rep" orchestrator)"; ready="$(jget "$rep" ready)"; reason="$(jget "$rep" reason)" a_ready="$(jget "$rep" agent.ready)"; a_reason="$(jget "$rep" agent.reason)"; a_hv="$(jget "$rep" agent.hypervisor)" LOG "admin probe: orchestrator=$orch ready=$ready image=$(jget "$rep" image_digest | head -c 19)… shape=$(jget "$rep" vcpus)vCPU/$(jget "$rep" mem_mib)MiB agent=$(jget "$rep" agent) agent_error=$(jget "$rep" agent_error)" + LOG "admin probe: custom_family_wired=$(jget "$rep" custom_family_wired) registered_custom=$(jget "$rep" registered_custom) live_harvest_wired=$(jget "$rep" live_harvest_wired) (Lium-only, informational)" if [[ "$orch" == "firecracker" ]]; then pass "CP resolved FirecrackerOrchestrator"; else fail "CP orchestrator is '$orch': $reason"; fi if [[ "$ready" == "true" ]]; then pass "CP ready(): bearer file + RLM image pin in place"; else fail "CP not ready: $reason"; fi if [[ -n "$(jget "$rep" agent)" ]]; then diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 182d6f087..a47d89777 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -86,7 +86,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Configured allocation | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Payout splits equally across currently `open` topics, then `wta` or `discovery`. Empty digest / missing evaluation prerequisites still fail closed. | | Automatic emission | **lib-only** | `proof-challenge::emit_epoch` signs payout leaves, but `bins/proof-challenge` does not call it or run an emission loop; the HTTP state starts at epoch `0`. Do not infer payments from `can_score`. | | RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. `/v1/status` reports the families apart — `live_harvest_wired` is Lium only; `custom_family_wired` / `registered_custom` / `custom_ready` are the custom family. | -| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `live_harvest_wired`, `registered_custom`) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Staging:** the agent booted Firecracker colocated on `cortex-staging` (nested DO `/dev/kvm` — an allowed exception, proven) and the runbook's § 4 fail-closed matrix came back green; image digests are operator state on that host (computed from images built outside this repo; nothing in git invents one), the § 5 happy path and § 6 sign-off are still to be recorded, and nested KVM stays fragile (boot fails → provision metal). **Production:** dedicated DO metal preferred, never colocated on the CP; not deployed. CI runs the fake hypervisor only. mTLS is a follow-up. | +| Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `custom_family_wired` / `registered_custom`, `live_harvest_wired` Lium-only) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Staging:** the agent booted Firecracker colocated on `cortex-staging` (nested DO `/dev/kvm` — an allowed exception, proven) and the runbook's § 4 fail-closed matrix came back green; image digests are operator state on that host (computed from images built outside this repo; nothing in git invents one), the § 5 happy path and § 6 sign-off are still to be recorded, and nested KVM stays fragile (boot fails → provision metal). **Production:** dedicated DO metal preferred, never colocated on the CP; not deployed. CI runs the fake hypervisor only. mTLS is a follow-up. | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index c15298eac..c8329ab76 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -236,8 +236,10 @@ Trust-root keygen is the throwaway owner path in of the topic-VM orchestrator through the host's own client: `orchestrator` (`firecracker` / `unwired`), `ready` + `reason` (bearer file, RLM image pin), the locked template, one agent health call (`agent` / - `agent_error`), `live_harvest_wired`, `registered_custom`. Always **200** - once authorised — a broken wire is data. Names env vars and container + `agent_error`), plus the host gates `custom_family_wired`, + `registered_custom`, and `live_harvest_wired` (Lium only — informational + for the custom family). Always **200** once authorised — a broken wire is + data. Names env vars and container paths, never the bearer. Run over loopback; wrapped by [`deploy/scripts/proof-vm-wire-check.sh`](../deploy/scripts/proof-vm-wire-check.sh). - `POST /v1/submissions` **requires** `topic_id`. Missing/unknown/not-open → diff --git a/docs/runbooks/proof-vm-orchestrator.md b/docs/runbooks/proof-vm-orchestrator.md index f8f890f83..f6aa68ac0 100644 --- a/docs/runbooks/proof-vm-orchestrator.md +++ b/docs/runbooks/proof-vm-orchestrator.md @@ -129,11 +129,10 @@ PROOF_VM_RUNNER_CUSTOM_IDS= Put the token under `deploy/secrets/proof/` (mounted at `/run/base/proof`, mode 0400, uid 65532). Restart `proof-challenge`; its boot log must show `firecracker topic-vm orchestrator wired` and one `vm-backed runner -registered` line per id. `GET /v1/status` → `registered_custom` lists the -ids; an open custom topic with a listed id appears in `scorable_topics`. -Both need `live_harvest_wired: true`: the custom family is routed only over -a wired Lium harvest, so without `LIUM_API_KEY` + `LIUM_SSH_PUBLIC_KEY_FILE` -the registry is never built and `registered_custom` stays `[]`. +registered` line per id. `GET /v1/status` → `custom_family_wired: true`, +`registered_custom` lists the ids, `custom_ready` lists the ones whose runner +can run now; an open custom topic with a ready id appears in +`scorable_topics`. The Lium harvest is **not** a prerequisite. With these four variables set and no `LIUM_API_KEY` / `LIUM_SSH_PUBLIC_KEY_FILE`, the boot log shows @@ -163,7 +162,7 @@ TOKEN=$(head -n1 deploy/secrets/proof/admin_tokens) curl -sS -H "Authorization: Bearer $TOKEN" http://127.0.0.1:8080/challenge/proof/v1/admin/proof/vm-orchestrator # {"orchestrator":"firecracker","ready":true,"reason":"","image_digest":"sha256:…","vcpus":4,"mem_mib":8192, # "agent":{"api_version":1,"ready":true,"reason":"","hypervisor":"firecracker","vms":0},"agent_error":null, -# "live_harvest_wired":true,"registered_custom":[""]} +# "live_harvest_wired":false,"custom_family_wired":true,"registered_custom":[""]} unset TOKEN ``` @@ -173,7 +172,8 @@ unset TOKEN | `ready: false` + `reason` | bearer file missing / empty, or `PROOF_RLM_VM_IMAGE_DIGEST` unpinned — fix the file / env; the token needs no restart | | `agent: null` + `agent_error` | `orchestrator unreachable` (agent down, firewall, VPC route), `orchestrator refused the bearer` (bytes differ from `/etc/proof-vm/token`), TLS refused (CA / SAN) | | `agent.ready: false` + `agent.reason` | the KVM host: `firecracker` / `jailer` / `/dev/kvm` / image missing | -| `live_harvest_wired: false` | Lium creds absent on the CP → custom family not routed, `registered_custom` empty | +| `custom_family_wired: false` (ids set) | the custom family is not routed: orchestrator env unset / refused, or no id registered — `registered_custom` says which | +| `live_harvest_wired` | **Lium only** (`nll` / `throughput`); informational for the custom family — `false` on a custom-only host is expected, not a fault | Run it over SSH + loopback (staging's public API is cleartext; never send the operator bearer over it). The reason strings name env vars and @@ -261,13 +261,14 @@ top of the compose stack. ### 0. Preconditions on the CP -The custom family is routed only when the whole live stack is up; check +The custom family scores only when the rest of the live stack is up; check `GET /v1/status` on the master **before** touching the wire: | Gate | Where | Must read | |------|-------|-----------| | eval backend | `/v1/status` `eval_backend` | `lium` (`PROOF_FORCE_SIM` off — sim never hosts staging scoring) | -| harvest | `/v1/status` `live_harvest_wired` | `true` (`LIUM_API_KEY` + `LIUM_SSH_PUBLIC_KEY_FILE`); `false` → `registered_custom` stays `[]` whatever the ids say | +| custom family | `/v1/status` `custom_family_wired`, `registered_custom`, `custom_ready` | `true`, your ids, your ids — wired from the topic-VM env alone (§ Wire the control plane); an id registered but not ready = bearer file / image pin on this host | +| harvest (Lium, informational here) | `/v1/status` `live_harvest_wired` | Lium only (`nll` / `throughput`): `true` with `LIUM_API_KEY` + `LIUM_SSH_PUBLIC_KEY_FILE`, `false` on a custom-only host — expected, not a fault; never stage a placeholder Lium key to open custom topics | | judge | `/v1/status` `inference_offer.status` | `open`, plus `PROOF_INFERENCE_API_KEY_FILE` present | | executor | `GET /v1/proof/executor` | `ready: true` (open `1x` offer) | | topic | a **signed custom topic** (`metric.family: custom`, `metric.custom_id: `) with a **sealed baseline**; the RLM of that topic sets it up per [`../PROOF.md`](../PROOF.md) § Dynamic agentic engine | its `custom_id` is what goes into `PROOF_VM_RUNNER_CUSTOM_IDS`; the topic can only **open** once that id is registered | @@ -329,7 +330,7 @@ cd /opt/base |------------|--------| | `env` | `PROOF_VM_ORCHESTRATOR_URL` is `https://`; the bearer file (container path mapped through the compose bind mount, `--path-map`) exists and is non-empty, mode 0400 / uid 65532; `PROOF_RLM_VM_IMAGE_DIGEST` is `sha256:<64 hex>` (empty or a placeholder = FAIL — never invented); the CA file is PEM when set; every custom id is well-formed; the shape is the locked 4 / 8192; `PROOF_FORCE_SIM` is off | | `agent` | `GET /v1/health` with the bearer → `ready: true`, `hypervisor: firecracker`; no bearer → 401; wrong bearer → 401 | -| `cp` | `/v1/status`: `lium`, `live_harvest_wired`, `registered_custom` ⊇ ids, no URL / token / path in the body; `/v1/proof/topics` leaks no holdout; `/v1/proof/executor` readiness; then the admin probe above — `orchestrator: firecracker`, `ready: true`, `agent.ready: true` through the CP's own rustls client | +| `cp` | `/v1/status`: `lium`, `custom_family_wired`, `registered_custom` ⊇ ids, `custom_ready` ⊇ ids (`live_harvest_wired` is logged, Lium-only, never a FAIL), no URL / token / path in the body; `/v1/proof/topics` leaks no holdout; `/v1/proof/executor` readiness; then the admin probe above — `orchestrator: firecracker`, `ready: true`, `agent.ready: true` through the CP's own rustls client | | `boot-probe` | the agent boots the **pinned** image for a probe topic, one topic ↔ one VM, a teardown naming another topic is refused, destroy is confirmed, nothing is left for the topic. Opt-in: it boots a real 4 vCPU / 8 GiB RLM VM on the KVM host (up to 10 min, the RLM guest must say hello); no job runs, nothing is spent. Nothing outlives it: Ctrl-C, a lost `201` (timeout, dropped connection), or an unconfirmed teardown all end in a by-topic attach + destroy before the script exits, so a retry on the same probe topic is never blocked by a stranded VM | Every check re-reads the files it names, so a fix to the bearer or the CA From b01d606c8fef0cfc5784305c2643ed8e9c9b58c5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 22:17:37 +0000 Subject: [PATCH 15/15] fix(deploy): wire-check refuses a zero declared_flops override Greptile P2 on #247: `--declared-flops 0` passed the override check and reached the control plane for a live probe, where any measured usage is a flops_under_declared reject instead of the documented awaiting_admin path. Explicit overrides must now be positive integers; zero, negative, and non-numeric values exit 1 with an actionable message before any request. Test covers 0 / abc / -1 on a live-run invocation. Co-authored-by: Mathis --- crates/proof-vm-fc/tests/wire_check_script.rs | 33 +++++++++++++++++++ deploy/scripts/proof-vm-wire-check.sh | 11 +++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/crates/proof-vm-fc/tests/wire_check_script.rs b/crates/proof-vm-fc/tests/wire_check_script.rs index b1c24bea0..a0b9b0fb8 100644 --- a/crates/proof-vm-fc/tests/wire_check_script.rs +++ b/crates/proof-vm-fc/tests/wire_check_script.rs @@ -292,6 +292,39 @@ async fn production_hosts_are_refused_case_insensitively_before_any_request() { "{cp}: no request may go out:\n{text}" ); } + // A zero (or non-numeric) declaration can only end rejected + // (flops_under_declared): refused before any request, live run or not. + for bad in ["0", "abc", "-1"] { + let (code, text) = tokio::task::spawn_blocking(move || { + run_script_env( + &[ + "submit-probe", + "--cp", + "http://127.0.0.1:9", + "--topic", + "x", + "--expect", + "201", + "--allow-live-run", + "--declared-flops", + bad, + ], + &[], + ) + }) + .await + .expect("join"); + assert_eq!(code, 1, "--declared-flops {bad} must be refused:\n{text}"); + assert!( + text.contains("--declared-flops must be a positive integer"), + "{bad}: {text}" + ); + assert!( + !text.contains("POST"), + "{bad}: no request may go out:\n{text}" + ); + } + // The agent URL is refused at env load, before boot-probe reads the bearer. let (secrets, _) = { let (env, secrets) = cp_env( diff --git a/deploy/scripts/proof-vm-wire-check.sh b/deploy/scripts/proof-vm-wire-check.sh index 37c1191e3..f39be3b02 100755 --- a/deploy/scripts/proof-vm-wire-check.sh +++ b/deploy/scripts/proof-vm-wire-check.sh @@ -30,8 +30,8 @@ # --reason SUBSTR submit-probe: the error text must contain this # --artifact-uri URI submit-probe locator (default https://example.invalid/wire-probe.tar — never fetchable) # --no-artifact-uri submit-probe without a locator (a custom topic must answer 400, no row) -# --declared-flops N submit-probe declaration (default: 1 for fail-closed probes; the topic's -# flops_budget for --expect 2xx so a real run is not rejected flops_under_declared) +# --declared-flops N submit-probe declaration, positive integer (default: 1 for fail-closed probes; the +# topic's flops_budget for --expect 2xx so a real run is not rejected flops_under_declared) # --wait SECS submit-probe --expect 201: how long the synchronous POST may take (default 900) # # Exit: 0 all PASS, 1 any FAIL, 2 refused (production host / unsafe request). @@ -631,7 +631,12 @@ submit_probe() { fi LOG "live run declares the topic budget: declared_flops=$flops" fi - [[ "$flops" =~ ^[0-9]+$ ]] || { RED "--declared-flops must be an integer"; exit 1; } + # Zero is never a useful declaration: any measured usage would be + # flops_under_declared, so a live probe could only end rejected. + if [[ ! "$flops" =~ ^[0-9]+$ || "$flops" == "0" ]]; then + RED "--declared-flops must be a positive integer (got '$flops'); a live run measuring anything above the declaration is rejected flops_under_declared" + exit 1 + fi local hotkey hex uri_field="" body code hotkey="$(head -c 64 /dev/zero | tr '\0' 'a')" hex="$(printf '%s' "wire-probe-$TOPIC-$(date +%s)-$$-$RANDOM" | sha256sum | awk '{print $1}')"