diff --git a/Cargo.lock b/Cargo.lock index 30efa1280..cbba96274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3614,8 +3614,10 @@ dependencies = [ "axum", "challenge-keys", "clap", + "crypto", "db", "harvest-pod", + "hex", "prism-lium", "proof-autonomy-http", "proof-autonomy-pg", @@ -3624,7 +3626,9 @@ dependencies = [ "proof-harvest", "proof-store", "proof-task", + "reqwest 0.12.28", "serde_json", + "sha2 0.10.9", "telemetry", "tokio", "tracing", @@ -3741,6 +3745,7 @@ dependencies = [ "proof-score", "proof-store", "proof-task", + "reqwest 0.12.28", "serde", "serde_json", "sha2 0.10.9", diff --git a/bins/ctx/src/proof.rs b/bins/ctx/src/proof.rs index 84876084a..f811807b7 100644 --- a/bins/ctx/src/proof.rs +++ b/bins/ctx/src/proof.rs @@ -113,10 +113,7 @@ pub async fn topics(client: &Client, json_out: bool) -> Result<(), String> { if json_out { return Ok(()); } - let items = reply - .body - .as_array() - .or_else(|| reply.body.get("topics").and_then(Value::as_array)); + let items = topic_list_items(&reply.body); match items { Some(list) if list.is_empty() => { println!("No open topics. Submits answer 503 until an operator publishes one."); @@ -245,6 +242,14 @@ fn print_fields(body: &Value) { } } +/// `GET /v1/proof/topics` returns `{ "items": [...] }`. Older shapes used +/// a bare array or `{ "topics": [...] }`. +fn topic_list_items(body: &Value) -> Option<&Vec> { + body.as_array() + .or_else(|| body.get("items").and_then(Value::as_array)) + .or_else(|| body.get("topics").and_then(Value::as_array)) +} + fn explain_failure(status: u16, message: &str) -> String { match status { 400 => format!("refused ({message}). Nothing was stored and nothing was rented."), @@ -287,4 +292,18 @@ mod tests { let ok = "a".repeat(64); assert_eq!(normalize_hex64(&ok, "hotkey").unwrap(), ok); } + + #[test] + fn topic_list_reads_the_items_wrapper() { + let body = serde_json::json!({ + "items": [ + { "id": "dt-no-ib-v0", "status": "open", "payout_mode": "wta" }, + { "id": "muon-vs-adamw-10m-v0", "status": "open", "payout_mode": "wta" } + ] + }); + let items = topic_list_items(&body).expect("items"); + assert_eq!(items.len(), 2); + assert_eq!(items[0]["id"], "dt-no-ib-v0"); + assert_eq!(items[1]["id"], "muon-vs-adamw-10m-v0"); + } } diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index ca7848c5e..e017ed625 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -31,5 +31,12 @@ telemetry = { path = "../../crates/telemetry" } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } tracing = "0.1" +[dev-dependencies] +crypto = { path = "../../crates/crypto" } +hex = "0.4" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "process", "time"] } + [lints] workspace = true diff --git a/bins/proof-challenge/tests/submit_e2e.rs b/bins/proof-challenge/tests/submit_e2e.rs new file mode 100644 index 000000000..11760cb1b --- /dev/null +++ b/bins/proof-challenge/tests/submit_e2e.rs @@ -0,0 +1,441 @@ +//! Process-level Proof submit → sim score. +//! +//! Spawns `proof-challenge --force-sim` with disposable synthetic +//! topic / holdout / baseline / offer files. No Lium, no compose, no secrets. +//! Topic ids match the staging open set. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use proof_eval::{sim_document, BaselineMeasurement, BASELINE_SKILL}; +use proof_task::{ + default_adamw, holdout_commitment, inference_config_commitment, synthetic_holdout, Constraints, + InferenceConfig, InferenceMode, InferenceOffer, InferenceProvider, InferenceProviderKind, + MetricDirection, MetricFamily, MetricSpec, OfferStatus, ProofPin, TopicDocument, TopicStatus, + EVAL_IMAGE, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, METRIC_TOKENS_PER_SEC, PRIMARY_HOLDOUT_NLL, + STRATUM_SIZE, +}; +use serde_json::Value; +use tokio::process::Command; + +const DT: &str = "dt-no-ib-v0"; +const MUON: &str = "muon-vs-adamw-10m-v0"; +const OFFER_ID: &str = "openrouter-glm53flash-v0"; + +fn sk() -> [u8; 32] { + let mut s = [3u8; 32]; + s[0] = 17; + s +} + +fn pk_hex() -> String { + hex::encode(crypto::public_key_from_mini_secret(&sk()).expect("pk")) +} + +fn digest(label: &str) -> String { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) +} + +fn pin() -> ProofPin { + let mut p = ProofPin { + topic_pubkey: pk_hex(), + ..ProofPin::default() + }; + p.inference.model = "master-proxy-v0".into(); + p +} + +fn offer() -> InferenceOffer { + let config = InferenceConfig { + mode: InferenceMode::Chat, + model_ref: "master-proxy-v0".into(), + max_input_tokens: 32_768, + max_output_tokens: 8_192, + temperature: Some(0.0), + top_p: None, + timeout_ms: None, + }; + InferenceOffer { + offer_id: OFFER_ID.into(), + provider: InferenceProvider { + kind: InferenceProviderKind::OpenaiCompatible, + base_url: "http://127.0.0.1:8000/v1".into(), + }, + config_commitment: inference_config_commitment(&config, "http://127.0.0.1:8000/v1"), + config, + status: OfferStatus::Open, + } +} + +fn dt_topic() -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.optimizer = "nccl-ib-reference".into(); + baseline.wall_budget_s = 14_400; + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: DT.into(), + statement: "No IB/NVLink; 12.5 Gbit/s cap; beat sealed comms baseline.".into(), + payout_mode: proof_task::PayoutMode::Wta, + constraints: Constraints { + no_infiniband: true, + no_nvlink: true, + no_nccl_fast_fabric: true, + max_inter_node_gbps: Some(12.5), + }, + metric: MetricSpec { + family: MetricFamily::Throughput, + primary: METRIC_TOKENS_PER_SEC.into(), + direction: MetricDirection::Max, + unit: "tokens_per_second".into(), + epsilon_rel: 0.05, + quality_floor_nll: 0.02, + wall_budget_s: 14_400, + custom_id: String::new(), + }, + baseline, + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } +} + +fn muon_topic() -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: MUON.into(), + statement: + "Beat sealed AdamW holdout NLL with Muon at ~10M params under the same FLOP budget." + .into(), + payout_mode: proof_task::PayoutMode::Wta, + metric: MetricSpec { + family: MetricFamily::Nll, + primary: PRIMARY_HOLDOUT_NLL.into(), + direction: MetricDirection::Min, + unit: "nll".into(), + epsilon_rel: 0.0, + quality_floor_nll: 0.0, + wall_budget_s: 0, + custom_id: String::new(), + }, + baseline, + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } +} + +fn seal( + pin: &ProofPin, + mut topic: TopicDocument, +) -> ( + TopicDocument, + BaselineMeasurement, + Vec, +) { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + topic.holdout_commitment = holdout_commitment(&recs); + let doc = sim_document(pin, &topic, "base", "base-art", BASELINE_SKILL, true); + let meas = BaselineMeasurement { + eval_image_digest: pin.eval_image_digest.clone(), + topic_id: topic.id.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + holdout_nll: doc.harness.holdout_nll, + split_nll: doc.harness.split_nll.clone(), + tokens_per_sec: doc.harness.tokens_per_sec, + step_latency_ms: doc.harness.step_latency_ms, + custom_value: doc.harness.custom_value, + }; + topic.baseline.metrics_commitment = meas.commitment(); + topic.signature = topic.sign_with(&sk()).expect("sign"); + topic.validate(pin, &[]).expect("valid"); + topic.verify_signature(pin).expect("sig"); + (topic, meas, recs) +} + +fn write_pin(dir: &Path, pin: &ProofPin) -> PathBuf { + let path = dir.join("pin.toml"); + let body = format!( + r#"challenge_id = "proof" +scoring_version = 1 +base_model_family = "Qwen/Qwen3.8" +proxy_model = "" +proxy_models = [] +inference_config_schema_version = 1 +allowed_modes = ["chat", "completions", "embeddings"] +max_input_tokens_ceiling = 32768 +max_output_tokens_ceiling = 8192 +inference_offer_commitment_alg = "sha256" +eval_image = "{EVAL_IMAGE}" +eval_image_digest = "{digest}" +proof_git = "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/CortexLM/cortex" +proof_git_sha = "" +topic_pubkey = "{pk}" +flops_budget_max = 2000000000000000000 +epsilon_nll_min = 0.02 +epsilon_topic_max_regress_min = 0.05 +epsilon_throughput_rel_min = 0.05 +quality_floor_nll_max = 0.02 +holdout_size = 120 +stratum_size = 24 + +[inference] +provider = "openai_compatible" +base_url = "" +model = "master-proxy-v0" +mode = "chat" +max_input_tokens = 32768 +max_output_tokens = 8192 +"#, + digest = pin.eval_image_digest, + pk = pin.topic_pubkey, + ); + fs::write(&path, body).expect("pin"); + path +} + +fn workdir() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "proof-submit-e2e-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&dir).expect("dir"); + dir +} + +struct Host { + child: tokio::process::Child, + base: String, + dir: PathBuf, +} + +impl Host { + async fn spawn() -> Self { + let dir = workdir(); + let p = pin(); + let (dt, dt_meas, dt_recs) = seal(&p, dt_topic()); + let (muon, muon_meas, muon_recs) = seal(&p, muon_topic()); + fs::write( + dir.join("topics.json"), + serde_json::to_vec(&[&dt, &muon]).expect("topics"), + ) + .expect("write topics"); + fs::write( + dir.join("holdouts.json"), + serde_json::to_vec(&serde_json::json!({ + DT: dt_recs, + MUON: muon_recs, + })) + .expect("holdouts"), + ) + .expect("write holdouts"); + fs::write( + dir.join("baselines.json"), + serde_json::to_vec(&serde_json::json!({ + DT: dt_meas, + MUON: muon_meas, + })) + .expect("baselines"), + ) + .expect("write baselines"); + fs::write( + dir.join("offer.json"), + serde_json::to_vec(&offer()).expect("offer"), + ) + .expect("write offer"); + let pin_path = write_pin(&dir, &p); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind probe"); + let addr = listener.local_addr().expect("addr"); + drop(listener); + + let mut child = Command::new(env!("CARGO_BIN_EXE_proof-challenge")) + .arg("--bind") + .arg(addr.to_string()) + .arg("--force-sim") + .arg("--pin-file") + .arg(&pin_path) + .arg("--topics-file") + .arg(dir.join("topics.json")) + .arg("--holdout-file") + .arg(dir.join("holdouts.json")) + .arg("--baseline-file") + .arg(dir.join("baselines.json")) + .arg("--inference-offer-file") + .arg(dir.join("offer.json")) + .env("PROOF_FORCE_SIM", "true") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("spawn proof-challenge"); + + let base = format!("http://{addr}"); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("client"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if tokio::time::Instant::now() > deadline { + let _ = child.start_kill(); + let out = child.wait_with_output().await.expect("wait"); + panic!( + "proof-challenge did not become healthy\nstderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + if client + .get(format!("{base}/health")) + .send() + .await + .ok() + .is_some_and(|r| r.status().is_success()) + { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + Self { child, base, dir } + } +} + +impl Drop for Host { + fn drop(&mut self) { + let _ = self.child.start_kill(); + let _ = fs::remove_dir_all(&self.dir); + } +} + +async fn json(method: reqwest::Method, url: &str, body: Option) -> (u16, Value) { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .expect("client"); + let mut req = client.request(method, url); + if let Some(b) = body { + req = req.json(&b); + } + let resp = req.send().await.expect("http"); + let status = resp.status().as_u16(); + let v = resp.json::().await.unwrap_or(Value::Null); + (status, v) +} + +fn submit_body(topic_id: &str, extra: &Value) -> Value { + let mut v = serde_json::json!({ + "miner_hotkey": digest("e2e-miner"), + "artifact_digest": digest(topic_id), + "claim": "e2e sim claim + artifact + declared_flops", + "declared_flops": 1_000_000u64, + "topic_id": topic_id, + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] }, + }); + if let Some(obj) = extra.as_object() { + if let Some(dst) = v.as_object_mut() { + for (k, val) in obj { + dst.insert(k.clone(), val.clone()); + } + } + } + v +} + +#[tokio::test] +async fn force_sim_binary_scores_staging_topic_ids() { + let host = Host::spawn().await; + let (st, status) = json( + reqwest::Method::GET, + &format!("{}/v1/status", host.base), + None, + ) + .await; + assert_eq!(st, 200, "{status}"); + assert_eq!(status["challenge_id"], "proof"); + assert_eq!(status["can_score"], true, "{status}"); + assert_eq!(status["eval_backend"], "sim", "{status}"); + assert_eq!(status["force_sim"], true, "{status}"); + assert_eq!(status["sim_stub_win"], true, "{status}"); + assert_eq!(status["baseline_sealed"], true, "{status}"); + assert_eq!(status["inference_offer"]["offer_id"], OFFER_ID); + let open = status["open_topics"].as_array().expect("open_topics"); + let ids: Vec<&str> = open.iter().filter_map(Value::as_str).collect(); + assert!(ids.contains(&DT), "{status}"); + assert!(ids.contains(&MUON), "{status}"); + assert!(!status.to_string().contains("api_key"), "{status}"); + assert!(!status.to_string().contains("8000"), "{status}"); + + let (st, topics) = json( + reqwest::Method::GET, + &format!("{}/v1/proof/topics", host.base), + None, + ) + .await; + assert_eq!(st, 200, "{topics}"); + assert!(!topics.to_string().contains("content_sha256"), "{topics}"); + + for topic_id in [DT, MUON] { + let (st, created) = json( + reqwest::Method::POST, + &format!("{}/v1/submissions", host.base), + Some(submit_body(topic_id, &serde_json::json!({}))), + ) + .await; + assert_eq!(st, 201, "{topic_id}: {created}"); + assert!( + created["id"] + .as_str() + .is_some_and(|id| id.starts_with("pf_")), + "{created}" + ); + assert_eq!(created["topic_id"], topic_id); + assert_eq!(created["eval_backend"], "sim"); + assert_eq!(created["state"], "awaiting_admin", "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + + let id = created["id"].as_str().expect("id"); + let (st, row) = json( + reqwest::Method::GET, + &format!("{}/v1/submissions/{id}", host.base), + None, + ) + .await; + assert_eq!(st, 200, "{row}"); + assert_eq!(row["declared_flops"], 1_000_000); + assert!(row["verdict"]["agent"].is_object(), "judge missing: {row}"); + assert!( + row["verdict"]["harness"]["holdout_nll"].is_number(), + "{row}" + ); + assert_eq!(row["verdict"]["pass"], true, "{row}"); + assert_eq!(row["verdict"]["agent"]["rationale"], "sim stub win"); + assert!( + row["receipt_json"] + .as_str() + .is_some_and(|s| s.contains("sim")), + "{row}" + ); + } + + let (st, bad) = json( + reqwest::Method::POST, + &format!("{}/v1/submissions", host.base), + Some(submit_body("", &serde_json::json!({ "topic_id": "" }))), + ) + .await; + assert_eq!(st, 400, "{bad}"); + assert_eq!(bad["error"], "topic_id is required"); +} diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index a07a40590..7e5e3ab3e 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -16,8 +16,8 @@ use proof_score::{payout_lattices, MinerTopicRun, SealedBaseline}; use proof_task::{CHALLENGE_ID_BYTES, SCORE_MAX}; pub use proof_eval::{ - force_sim, resolve_eval_backend, scoring_readiness, supported_custom, BaselineMeasurement, - EvalBackend, LiveScorer, + force_sim, resolve_eval_backend, scoring_readiness, sim_stub_win, supported_custom, + BaselineMeasurement, EvalBackend, LiveScorer, }; pub use proof_http::{hash_admin_token, proof_router, AppState}; pub use proof_store::{ArtifactManifest, MemoryStore, StoreError}; diff --git a/crates/proof-eval/src/lib.rs b/crates/proof-eval/src/lib.rs index 3c17aa0d3..23f35ea3b 100644 --- a/crates/proof-eval/src/lib.rs +++ b/crates/proof-eval/src/lib.rs @@ -29,8 +29,8 @@ use proof_score::{AgentVerdict, HarnessMetrics, ProofCheatCode, ProofKind, Seale use proof_store::ArtifactManifest; use proof_task::{ canonical_json, contamination, require_open_offer, resolve_inference, HoldoutRecord, - HoldoutSplit, InferenceOffer, MetricFamily, OfferError, ProofPin, TopicDocument, - BASELINE_DOMAIN, + HoldoutSplit, InferenceOffer, MetricDirection, MetricFamily, OfferError, ProofPin, + TopicDocument, BASELINE_DOMAIN, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -58,6 +58,17 @@ pub fn force_sim() -> bool { ) } +/// True when this host opted into sim (`PROOF_FORCE_SIM`). +/// +/// Under [`EvalBackend::Sim`] a sealed baseline scores with +/// [`sim_win_document`] (harness relative to the seal). Skill-only +/// [`sim_document`] cannot beat a real ~0.29 NLL seal. The Lium path never +/// uses either helper. `PROOF_SIM_STUB_WIN` is a leftover no-op. +#[must_use] +pub fn sim_stub_win() -> bool { + force_sim() +} + /// Resolve the scoring backend for this host. Sim is never implicit. #[must_use] pub fn resolve_eval_backend() -> EvalBackend { @@ -462,6 +473,11 @@ fn unit(parts: &[&str], index: u32) -> f64 { } /// Deterministic sim scores. Only used when the host opted into sim. +/// +/// Holdout NLL is `(3.10 - 0.40 * skill).max(1.0)`. Skill=1.0 still yields +/// NLL ≥ 1.0, so this **cannot** clear `quality_floor` against a real sealed +/// baseline near 0.29. Test wins against sim-derived baselines +/// ([`BASELINE_SKILL`]) do not apply on staging. Use [`sim_win_document`]. #[must_use] pub fn sim_document( pin: &ProofPin, @@ -528,6 +544,96 @@ pub fn sim_document( /// Skill of the sealed AdamW / comms reference in sim (so a strong miner wins). pub const BASELINE_SKILL: f64 = 0.40; +fn beat(baseline: f64, direction: MetricDirection, epsilon: f64) -> f64 { + let margin = 0.01; + match direction { + MetricDirection::Max => baseline * (1.0 + epsilon + margin), + MetricDirection::Min => (baseline * (1.0 - epsilon - margin)).max(0.0), + } +} + +/// Sim harness **relative to `sealed`**, not a higher [`sim_document`] skill. +/// +/// Inequalities (option A): holdout NLL ≤ baseline + quality floor, each +/// scored split ≤ baseline + `epsilon_topic_max_regress`, and +/// `tokens_per_sec` ≥ ref × (1 + `epsilon_rel`) when that is the primary. +/// Used for every [`EvalBackend::Sim`] score that has a sealed baseline. +/// Never called from the Lium path. +#[must_use] +pub fn sim_win_document( + pin: &ProofPin, + topic: &TopicDocument, + frozen: &str, + artifact: &str, + sealed: &SealedBaseline, +) -> ProofEvalDocument { + let nll_eps = topic.epsilon_nll.max(0.01); + let holdout = match topic.metric.family { + MetricFamily::Nll => (sealed.holdout_nll - nll_eps).max(0.0), + _ => sealed.holdout_nll, + }; + let mut split = BTreeMap::new(); + for s in HoldoutSplit::SCORED { + let b = sealed.split_nll.get(s.as_str()).copied().unwrap_or(holdout); + let v = match topic.metric.family { + MetricFamily::Nll => (b - nll_eps).max(0.0), + _ => b, + }; + split.insert(s.as_str().to_owned(), v); + } + let tps = match topic.metric.primary.as_str() { + proof_task::METRIC_TOKENS_PER_SEC => Some(beat( + sealed.tokens_per_sec.unwrap_or(100.0), + topic.metric.direction, + topic.metric.epsilon_rel, + )), + _ => sealed.tokens_per_sec, + }; + let latency = match topic.metric.primary.as_str() { + proof_task::METRIC_STEP_LATENCY_MS => Some(beat( + sealed.step_latency_ms.unwrap_or(100.0), + topic.metric.direction, + topic.metric.epsilon_rel, + )), + _ => sealed.step_latency_ms, + }; + let custom = sealed + .custom_value + .map(|b| beat(b, topic.metric.direction, topic.metric.epsilon_rel)); + ProofEvalDocument { + schema_version: PROOF_METRICS_SCHEMA, + submission_digest: frozen.to_owned(), + artifact_digest: artifact.to_owned(), + topic_id: topic.id.clone(), + eval_image_digest: pin.eval_image_digest.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + agent: AgentVerdict { + verdict: ProofKind::Clean, + reproduced: true, + claim_holds_public: true, + contamination: false, + canary_hit: false, + flops_used: topic.flops_budget / 2, + flops_budget: topic.flops_budget, + cheat_codes: Vec::new(), + rationale: "sim stub win".into(), + topic_id: topic.id.clone(), + family: topic.metric.family, + }, + harness: HarnessMetrics { + holdout_nll: holdout, + split_nll: split, + public_nll: Some(holdout), + tokens_per_sec: tps, + step_latency_ms: latency, + wall_s: (topic.metric.family == MetricFamily::Throughput) + .then_some(topic.metric.wall_budget_s / 2), + custom_value: custom, + canary_nll: None, + }, + } +} + /// Score only after the submission digest is frozen and a topic is open. #[allow(clippy::too_many_arguments)] pub async fn eval_after_freeze( @@ -541,6 +647,7 @@ pub async fn eval_after_freeze( backend: EvalBackend, live: Option<&dyn LiveScorer>, judge_api_key: Option<&str>, + sealed: Option<&SealedBaseline>, ) -> Result { if frozen_digest.trim().is_empty() || holdout.is_empty() { return Err(EvalError::HoldoutSealed); @@ -567,8 +674,12 @@ pub async fn eval_after_freeze( } let doc = match backend { EvalBackend::Sim => { - let skill = unit(&[artifact_digest, "skill"], 0); - sim_document(pin, topic, frozen_digest, artifact_digest, skill, true) + if let Some(sealed) = sealed { + sim_win_document(pin, topic, frozen_digest, artifact_digest, sealed) + } else { + let skill = unit(&[artifact_digest, "skill"], 0); + sim_document(pin, topic, frozen_digest, artifact_digest, skill, true) + } } EvalBackend::Lium => { let scorer = live.ok_or(EvalError::LiveHarvestUnavailable)?; @@ -731,6 +842,7 @@ mod tests { EvalBackend::Lium, None, None, + None, ) .await .expect_err("no digest"); @@ -750,6 +862,7 @@ mod tests { EvalBackend::Lium, None, None, + None, ) .await .expect_err("no harvest"); @@ -775,6 +888,7 @@ mod tests { EvalBackend::Lium, Some(&Harvest { reproduced: true }), Some("test-judge-key"), + None, ) .await .expect("live"); @@ -891,4 +1005,154 @@ mod tests { let doc = sim_document(&pin, &t, "f", "art", 1.0, true); assert!(doc.harness.custom_value.is_none()); } + + fn tight_sealed() -> SealedBaseline { + let mut split = BTreeMap::new(); + for s in HoldoutSplit::SCORED { + split.insert(s.as_str().to_owned(), 0.29); + } + SealedBaseline { + holdout_nll: 0.29, + split_nll: split, + tokens_per_sec: Some(80.0), + step_latency_ms: None, + custom_value: None, + } + } + + fn throughput_topic() -> TopicDocument { + let mut t = topic(); + t.id = "dt-no-ib-v0".into(); + t.metric.family = MetricFamily::Throughput; + t.metric.primary = proof_task::METRIC_TOKENS_PER_SEC.into(); + t.metric.direction = MetricDirection::Max; + t.metric.epsilon_rel = 0.05; + t.metric.quality_floor_nll = 0.02; + t.metric.wall_budget_s = 14_400; + t + } + + #[test] + fn stub_win_clears_quality_floor_against_a_tight_sealed_baseline() { + let pin = pin(""); + let t = throughput_topic(); + let sealed = tight_sealed(); + let floor = sealed.holdout_nll + t.metric.quality_floor_nll; + let stub_win_skill = sim_document(&pin, &t, "f", "art", 0.95, true); + let max_skill = sim_document(&pin, &t, "f", "art", 1.0, true); + assert!( + max_skill.harness.holdout_nll >= 1.0, + "skill=1.0 must not dip below the sim NLL floor: {}", + max_skill.harness.holdout_nll + ); + assert!( + stub_win_skill.harness.holdout_nll >= 1.0, + "StubScorer::win skill=0.95 is still NLL≥1.0: {}", + stub_win_skill.harness.holdout_nll + ); + for skill_doc in [&stub_win_skill, &max_skill] { + let reject = proof_score::judge_topic( + &t, + &skill_doc.agent, + &skill_doc.harness, + &sealed, + &[], + &[], + ); + assert!(!reject.pass, "{reject:?}"); + assert!( + reject + .failed + .iter() + .any(|g| matches!(g, proof_score::GateFail::QualityFloor { .. })), + "{reject:?}" + ); + } + + let win = sim_win_document(&pin, &t, "f", "art", &sealed); + assert_eq!(win.agent.rationale, "sim stub win"); + assert!( + win.harness.holdout_nll <= floor, + "holdout {} > baseline+floor {}", + win.harness.holdout_nll, + floor + ); + for s in HoldoutSplit::SCORED { + let h = win.harness.split_nll[s.as_str()]; + let b = sealed.split_nll[s.as_str()]; + assert!( + h <= b + t.epsilon_topic_max_regress, + "split {} {h} > {b}+eps", + s.as_str() + ); + } + let tps = win.harness.tokens_per_sec.expect("tps"); + let ref_tps = sealed.tokens_per_sec.expect("ref tps"); + assert!( + tps >= ref_tps * (1.0 + t.metric.epsilon_rel), + "tps {tps} < ref*(1+eps) {}", + ref_tps * (1.0 + t.metric.epsilon_rel) + ); + let verdict = proof_score::judge_topic(&t, &win.agent, &win.harness, &sealed, &[], &[]); + assert!(verdict.pass, "{verdict:?}"); + assert!(verdict.failed.is_empty(), "{verdict:?}"); + } + + #[tokio::test] + async fn sim_plus_sealed_uses_relative_harness() { + let t = throughput_topic(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let p = pin(""); + let sealed = tight_sealed(); + let out = eval_after_freeze( + &p, + &t, + &offer(), + "digest-a", + "art", + &recs, + "claim", + EvalBackend::Sim, + None, + None, + Some(&sealed), + ) + .await + .expect("sim"); + assert_eq!(out.backend, EvalBackend::Sim); + assert_eq!(out.receipt.provider, "sim"); + assert_eq!(out.agent.rationale, "sim stub win"); + assert!(out.harness.holdout_nll <= sealed.holdout_nll + t.metric.quality_floor_nll); + assert!( + out.harness.tokens_per_sec.expect("tps") + >= sealed.tokens_per_sec.expect("ref") * (1.0 + t.metric.epsilon_rel) + ); + let verdict = proof_score::judge_topic(&t, &out.agent, &out.harness, &sealed, &[], &[]); + assert!(verdict.pass, "{verdict:?}"); + } + + #[tokio::test] + async fn stub_win_is_ignored_on_the_lium_path() { + let t = throughput_topic(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let p = pin(&format!("sha256:{}", "ab".repeat(32))); + let out = eval_after_freeze( + &p, + &t, + &offer(), + "digest-a", + "art", + &recs, + "claim", + EvalBackend::Lium, + Some(&Harvest { reproduced: true }), + Some("test-judge-key"), + Some(&tight_sealed()), + ) + .await + .expect("live"); + assert_eq!(out.backend, EvalBackend::Lium); + assert_eq!(out.receipt.provider, "lium"); + assert!(out.harness.holdout_nll > 1.0, "must not emit stub-win NLL"); + } } diff --git a/crates/proof-http/Cargo.toml b/crates/proof-http/Cargo.toml index 391595438..ff0b9e1c6 100644 --- a/crates/proof-http/Cargo.toml +++ b/crates/proof-http/Cargo.toml @@ -25,6 +25,7 @@ db = { path = "../db", features = ["testing"] } async-trait = "0.1" crypto = { path = "../crypto" } http-body-util = "0.1" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync"] } tower = { version = "0.5", features = ["util"] } diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 613e6f0a5..212d23cdf 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -141,6 +141,7 @@ async fn status(State(st): State) -> impl IntoResponse { }, "eval_backend": st.backend, "force_sim": force_sim(), + "sim_stub_win": st.backend == EvalBackend::Sim, "can_score": st.can_score(), "live_harvest_wired": st.live_scorer.is_some(), "baseline_sealed": baseline_sealed, @@ -316,6 +317,7 @@ async fn submit( st.backend, st.live(), st.judge_api_key.as_deref(), + Some(&sealed), ) .await .map_err(|e| eval_err(&e))?; @@ -618,10 +620,10 @@ mod tests { use proof_eval::{sim_document, BaselineMeasurement, BASELINE_SKILL}; use proof_task::{ default_adamw, holdout_commitment, inference_config_commitment, synthetic_holdout, - Constraints, InferenceConfig, InferenceMode, InferenceOffer, InferenceProvider, - InferenceProviderKind, MetricDirection, MetricFamily, MetricSpec, OfferStatus, - TopicDocument, TopicStatus, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, METRIC_TOKENS_PER_SEC, - STRATUM_SIZE, + Constraints, HoldoutSplit, InferenceConfig, InferenceMode, InferenceOffer, + InferenceProvider, InferenceProviderKind, MetricDirection, MetricFamily, MetricSpec, + OfferStatus, TopicDocument, TopicStatus, FLOPS_BUDGET_MAX, HOLDOUT_SIZE, + METRIC_TOKENS_PER_SEC, STRATUM_SIZE, }; use tower::ServiceExt; @@ -654,6 +656,15 @@ mod tests { } fn offer() -> InferenceOffer { + named_offer("master-v0") + } + + /// Staging sim offer id (operator-published; miners do not bind it). + fn staging_offer() -> InferenceOffer { + named_offer("openrouter-glm53flash-v0") + } + + fn named_offer(offer_id: &str) -> InferenceOffer { let config = InferenceConfig { mode: InferenceMode::Chat, model_ref: "master-proxy-v0".into(), @@ -664,7 +675,7 @@ mod tests { timeout_ms: None, }; InferenceOffer { - offer_id: "master-v0".into(), + offer_id: offer_id.into(), provider: InferenceProvider { kind: InferenceProviderKind::OpenaiCompatible, base_url: "http://127.0.0.1:8000/v1".into(), @@ -708,6 +719,33 @@ mod tests { } } + fn unsigned_muon_topic(recs: &[proof_task::HoldoutRecord]) -> TopicDocument { + let mut baseline = default_adamw(FLOPS_BUDGET_MAX); + baseline.script_sha256 = "11".repeat(32); + TopicDocument { + id: "muon-vs-adamw-10m-v0".into(), + statement: + "Beat sealed AdamW holdout NLL with Muon at ~10M params under the same FLOP budget." + .into(), + payout_mode: proof_task::PayoutMode::Wta, + metric: MetricSpec { + family: MetricFamily::Nll, + primary: proof_task::PRIMARY_HOLDOUT_NLL.into(), + direction: MetricDirection::Min, + unit: "nll".into(), + epsilon_rel: 0.0, + quality_floor_nll: 0.0, + wall_budget_s: 0, + custom_id: String::new(), + }, + baseline, + holdout_commitment: holdout_commitment(recs), + holdout_size: HOLDOUT_SIZE, + status: TopicStatus::Open, + ..TopicDocument::default() + } + } + fn seal_topic( pin: &ProofPin, mut topic: TopicDocument, @@ -881,6 +919,7 @@ mod tests { json_req(app("op"), "GET", "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/v1/status", serde_json::json!({}), None).await; assert_eq!(st, StatusCode::OK); assert_eq!(body["eval_backend"], "sim"); + assert_eq!(body["sim_stub_win"], true, "{body}"); assert_eq!(body["can_score"], true, "{body}"); assert_eq!(body["baseline_sealed"], true, "{body}"); assert_eq!(body["open_topics"][0], "dt-no-ib-v0"); @@ -1406,6 +1445,279 @@ mod tests { "{body}" ); } + + fn app_staging_sim() -> Router { + let p = pin(""); + let store = MemoryStore::new(); + for draft in [unsigned_topic(&[]), unsigned_muon_topic(&[])] { + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, meas) = seal_topic(&p, draft); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, meas.into_sealed()) + .expect("baseline"); + } + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Sim, + live_scorer: None, + offer: Some(staging_offer()), + judge_api_key: None, + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }) + } + + fn assert_scored_row(created: &serde_json::Value, topic_id: &str) { + assert!( + created["id"] + .as_str() + .is_some_and(|id| id.starts_with("pf_")), + "silent empty id: {created}" + ); + assert_eq!(created["topic_id"], topic_id, "{created}"); + assert_eq!(created["eval_backend"], "sim", "{created}"); + let state = created["state"].as_str().unwrap_or_default(); + assert!( + state == "awaiting_admin" || state == "rejected", + "non-terminal or empty state: {created}" + ); + assert!(created["eligible"].is_boolean(), "{created}"); + assert!( + created["submission_digest"] + .as_str() + .is_some_and(|d| d.len() == 64), + "{created}" + ); + } + + #[tokio::test] + async fn sim_submit_scores_claim_artifact_and_flops() { + let app = app("op"); + let body = submit_body( + "staging-e2e-artifact", + &serde_json::json!({ + "claim": "beats the sealed reference under the cap", + "declared_flops": 1_000_000_000_000u64, + }), + ); + let (st, created) = json_req(app.clone(), "POST", "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/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_scored_row(&created, "dt-no-ib-v0"); + + let id = created["id"].as_str().expect("id"); + let (st, row) = json_req( + app, + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{row}"); + assert_eq!(row["id"], id); + assert_eq!(row["topic_id"], "dt-no-ib-v0"); + assert_eq!(row["declared_flops"], 1_000_000_000_000u64); + assert_eq!(row["claim"], "beats the sealed reference under the cap"); + assert!(row["verdict"].is_object(), "judge path missing: {row}"); + assert!(row["verdict"]["agent"].is_object(), "{row}"); + assert!( + row["verdict"]["harness"]["holdout_nll"].is_number(), + "{row}" + ); + assert!(row["verdict"]["lattice"].is_number(), "{row}"); + assert!( + row["verdict"]["agent"]["rationale"] + .as_str() + .is_some_and(|s| !s.is_empty()), + "notation missing: {row}" + ); + let receipt = row["receipt_json"].as_str().unwrap_or_default(); + assert!(receipt.contains("sim"), "sim receipt missing: {row}"); + let dump = row.to_string(); + assert!(!dump.contains("content_sha256"), "{dump}"); + assert!(!dump.contains("api_key"), "{dump}"); + } + + #[tokio::test] + async fn sim_submit_accepts_staging_topic_ids() { + let app = app_staging_sim(); + let (st, status) = json_req( + app.clone(), + "GET", + "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/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["can_score"], true, "{status}"); + assert_eq!(status["eval_backend"], "sim", "{status}"); + assert_eq!(status["baseline_sealed"], true, "{status}"); + assert_eq!( + status["inference_offer"]["offer_id"], + "openrouter-glm53flash-v0" + ); + let open = status["open_topics"].as_array().expect("open_topics"); + let ids: Vec<&str> = open.iter().filter_map(|v| v.as_str()).collect(); + assert!(ids.contains(&"dt-no-ib-v0"), "{status}"); + assert!(ids.contains(&"muon-vs-adamw-10m-v0"), "{status}"); + + let (st, list) = json_req( + app.clone(), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + let dump = list.to_string(); + assert!(!dump.contains("content_sha256"), "{dump}"); + assert!(!dump.contains("synthetic-dev"), "{dump}"); + + for topic_id in ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"] { + let (st, created) = json_req( + app.clone(), + "POST", + "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/v1/submissions", + submit_body( + topic_id, + &serde_json::json!({ + "topic_id": topic_id, + "declared_flops": 42u64, + }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{topic_id}: {created}"); + assert_scored_row(&created, topic_id); + } + } + + #[tokio::test] + async fn sim_submit_fail_closed_reasons_are_explicit() { + let app = app_staging_sim(); + let (st, body) = json_req( + app.clone(), + "POST", + "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/v1/submissions", + submit_body("x", &serde_json::json!({ "topic_id": "" })), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(body["error"], "topic_id is required"); + + let (st, body) = json_req( + app.clone(), + "POST", + "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/v1/submissions", + submit_body("x", &serde_json::json!({ "topic_id": "not-a-live-topic" })), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(body["error"], "unknown topic"); + + let (st, body) = json_req( + app, + "POST", + "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/v1/submissions", + submit_body("x", &serde_json::json!({ "declared_flops": u64::MAX })), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("declared_flops"), + "{body}" + ); + } + + fn tight_sealed() -> proof_score::SealedBaseline { + let mut split = std::collections::BTreeMap::new(); + for s in HoldoutSplit::SCORED { + split.insert(s.as_str().to_owned(), 0.29); + } + proof_score::SealedBaseline { + holdout_nll: 0.29, + split_nll: split, + tokens_per_sec: Some(80.0), + step_latency_ms: None, + custom_value: None, + } + } + + fn app_tight_sim() -> Router { + let p = pin(""); + let store = MemoryStore::new(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, _) = seal_topic(&p, unsigned_topic(&recs)); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, tight_sealed()) + .expect("baseline"); + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Sim, + live_scorer: None, + offer: Some(staging_offer()), + judge_api_key: None, + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + epoch: 0, + }) + } + + #[tokio::test] + async fn sim_stub_win_submit_reaches_awaiting_admin() { + let app = app_tight_sim(); + let (st, status) = json_req( + app.clone(), + "GET", + "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/v1/status", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["eval_backend"], "sim"); + assert_eq!(status["sim_stub_win"], true, "{status}"); + + let (st, created) = json_req( + app.clone(), + "POST", + "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/v1/submissions", + submit_body("tight-win", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "awaiting_admin", "{created}"); + assert_eq!(created["eligible"], true, "{created}"); + assert_eq!(created["eval_backend"], "sim"); + let id = created["id"].as_str().expect("id"); + let (st, row) = json_req( + app, + "GET", + &format!("/v1/submissions/{id}"), + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{row}"); + assert_eq!(row["verdict"]["pass"], true, "{row}"); + assert_eq!(row["verdict"]["agent"]["rationale"], "sim stub win"); + assert_eq!(row["verdict"]["failed"].as_array().map(Vec::len), Some(0)); + } } #[cfg(test)] diff --git a/crates/proof-http/tests/live_submit_e2e.rs b/crates/proof-http/tests/live_submit_e2e.rs new file mode 100644 index 000000000..218938cb7 --- /dev/null +++ b/crates/proof-http/tests/live_submit_e2e.rs @@ -0,0 +1,170 @@ +//! Optional live probe of a running Proof host (`PROOF_E2E_BASE`). +//! +//! Set `PROOF_E2E_BASE` to the challenge origin (no trailing slash), e.g. +//! `http://127.0.0.1:28100` or `http://staging.api.joinbase.ai/challenge/proof`. +//! +//! Never POSTs when the host is live Lium and `can_score` (would rent). +//! Never talks to production. Staging sim (`eval_backend=sim`) is the intended +//! target. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use serde_json::Value; + +const STAGING_TOPICS: [&str; 2] = ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"]; + +fn base_url() -> Option { + std::env::var("PROOF_E2E_BASE") + .ok() + .map(|s| s.trim().trim_end_matches('/').to_owned()) + .filter(|s| !s.is_empty()) +} + +fn is_prod_host(base: &str) -> bool { + base.contains("network.cortex.foundation") || base.contains("chain.joinbase.ai") +} + +fn hex64(label: &str) -> String { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) +} + +async fn get(client: &reqwest::Client, url: &str) -> (u16, Value) { + let resp = client.get(url).send().await.expect("GET"); + let status = resp.status().as_u16(); + let body = resp.json::().await.unwrap_or(Value::Null); + (status, body) +} + +async fn post(client: &reqwest::Client, url: &str, body: &Value) -> (u16, Value) { + let resp = client.post(url).json(body).send().await.expect("POST"); + let status = resp.status().as_u16(); + let body = resp.json::().await.unwrap_or(Value::Null); + (status, body) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn live_host_submit_scores_or_fails_closed() { + let Some(base) = base_url() else { + eprintln!("skip live_submit_e2e: set PROOF_E2E_BASE to probe a running host"); + return; + }; + assert!( + !is_prod_host(&base), + "refusing production host {base} (staging/local only)" + ); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("client"); + + let (st, health) = get(&client, &format!("{base}/health")).await; + assert_eq!(st, 200, "{health}"); + assert_eq!(health["challenge_id"], "proof"); + + let (st, status) = get(&client, &format!("{base}/v1/status")).await; + assert_eq!(st, 200, "{status}"); + assert_eq!(status["challenge_id"], "proof"); + assert!(status["can_score"].is_boolean(), "{status}"); + assert!(status["eval_backend"].is_string(), "{status}"); + assert!( + status.get("error").is_none(), + "status must not be an error object: {status}" + ); + let dump = status.to_string(); + assert!(!dump.contains("api_key"), "{dump}"); + assert!(!dump.contains("content_sha256"), "{dump}"); + + let (st, topics) = get(&client, &format!("{base}/v1/proof/topics")).await; + assert_eq!(st, 200, "{topics}"); + let items = topics["items"].as_array().cloned().unwrap_or_default(); + let listed: Vec = items + .iter() + .filter_map(|t| t.get("id").and_then(Value::as_str).map(ToOwned::to_owned)) + .collect(); + assert!( + !topics.to_string().contains("content_sha256"), + "holdout leak: {topics}" + ); + + let (st, missing) = post( + &client, + &format!("{base}/v1/submissions"), + &serde_json::json!({ + "miner_hotkey": hex64("e2e-hotkey"), + "artifact_digest": hex64("e2e-artifact"), + "claim": "e2e probe", + "declared_flops": 1, + "topic_id": "", + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] } + }), + ) + .await; + assert_eq!(st, 400, "empty topic_id must 400, got {st} {missing}"); + assert!( + missing["error"].as_str().is_some_and(|e| !e.is_empty()), + "silent empty 400: {missing}" + ); + + let can_score = status["can_score"].as_bool().unwrap_or(false); + let backend = status["eval_backend"].as_str().unwrap_or_default(); + if can_score && backend == "lium" { + eprintln!("skip live POST: host is Lium + can_score (would rent)"); + return; + } + + let mut topic_ids: Vec<&str> = STAGING_TOPICS + .iter() + .copied() + .filter(|id| listed.iter().any(|got| got == *id)) + .collect(); + if topic_ids.is_empty() { + topic_ids.push(listed.first().map_or("dt-no-ib-v0", String::as_str)); + } + + for topic_id in topic_ids { + let (st, created) = post( + &client, + &format!("{base}/v1/submissions"), + &serde_json::json!({ + "miner_hotkey": hex64("e2e-hotkey"), + "artifact_digest": hex64(&format!("e2e-artifact-{topic_id}")), + "claim": "e2e sim submit against an open topic", + "declared_flops": 1, + "topic_id": topic_id, + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] } + }), + ) + .await; + assert!( + st == 201 || st == 400 || st == 503, + "unexpected {st} {created}" + ); + if st == 201 { + assert!( + created["id"] + .as_str() + .is_some_and(|id| id.starts_with("pf_")), + "silent empty create: {created}" + ); + assert_eq!(created["topic_id"], topic_id); + assert!(created["eval_backend"].is_string(), "{created}"); + let id = created["id"].as_str().expect("id"); + let (gst, row) = get(&client, &format!("{base}/v1/submissions/{id}")).await; + assert_eq!(gst, 200, "{row}"); + assert!( + row["verdict"].is_object() || row["state"] == "rejected", + "{row}" + ); + } else { + assert!( + created["error"].as_str().is_some_and(|e| !e.is_empty()), + "silent empty fail-closed: HTTP {st} {created}" + ); + } + } +} diff --git a/deploy/compose/env-local.yml b/deploy/compose/env-local.yml index 7ced428d1..1f00e7655 100644 --- a/deploy/compose/env-local.yml +++ b/deploy/compose/env-local.yml @@ -68,6 +68,7 @@ services: BASE_DATABASE_URL: ${LOCAL_DATABASE_URL:-postgres://base:base_dev_only_change_me@postgres:5432/base} # No Lium spend on a laptop unless operator opts in. PROOF_FORCE_SIM: "${LOCAL_PROOF_FORCE_SIM:-true}" + PROOF_SIM_STUB_WIN: "${LOCAL_PROOF_SIM_STUB_WIN:-true}" bounty-challenge: ports: diff --git a/deploy/env/proof-challenge.env.example b/deploy/env/proof-challenge.env.example index 62beab457..f6bad040a 100644 --- a/deploy/env/proof-challenge.env.example +++ b/deploy/env/proof-challenge.env.example @@ -14,6 +14,11 @@ BASE_NETUID=541 # live_harvest_wired, baseline_sealed. PROOF_FORCE_SIM=false +# Leftover no-op. Under PROOF_FORCE_SIM a sealed topic already emits +# harness numbers relative to the seal. Do not set this in +# deploy/compose/env-staging.yml or env-prod.yml. +PROOF_SIM_STUB_WIN=false + # The miner pays for the eval pod; this key is the master's Lium account used # to provision and terminate it. Never logged, never echoed on /v1/status. # LIUM_API_KEY= diff --git a/deploy/scripts/assert-compose-matrix.sh b/deploy/scripts/assert-compose-matrix.sh index 414e6e6da..4cdd13f79 100755 --- a/deploy/scripts/assert-compose-matrix.sh +++ b/deploy/scripts/assert-compose-matrix.sh @@ -132,7 +132,7 @@ for env_file in deploy/compose/env-staging.yml deploy/compose/env-prod.yml; do if echo "$rendered" | grep -qE 'DESIGN_FORCE_SIM:[[:space:]]*["'\'']?(1|true|TRUE|yes)["'\'']?'; then fail "$env_file enables DESIGN_FORCE_SIM (retired; must not ship)" fi - for sim_var in RELEARN_FORCE_SIM RELEARN_T2I_FORCE_SIM RELEARN_AGENT_FORCE_SIM RELEARN_MM_FORCE_SIM PROOF_FORCE_SIM; do + for sim_var in RELEARN_FORCE_SIM RELEARN_T2I_FORCE_SIM RELEARN_AGENT_FORCE_SIM RELEARN_MM_FORCE_SIM PROOF_FORCE_SIM PROOF_SIM_STUB_WIN; do if echo "$rendered" | grep -qE "${sim_var}:[[:space:]]*[\"']?(1|true|TRUE|yes)[\"']?"; then fail "$env_file enables $sim_var (sim is local-only; must not ship on droplets)" fi diff --git a/deploy/scripts/local-e2e.sh b/deploy/scripts/local-e2e.sh index 5f5bee66f..2786992bb 100755 --- a/deploy/scripts/local-e2e.sh +++ b/deploy/scripts/local-e2e.sh @@ -664,6 +664,18 @@ probe_bounty_fail_closed() { fi } +# Proof submit → score (or explicit 400/503). Never rents Lium. +# Full matrix + curl/ctx contract: docs/runbooks/proof-submit-e2e.md +probe_proof_submit() { + local base="http://127.0.0.1:${PROOF_HOST_PORT}" + if ! curl -fsS -m 5 "${base}/health" >/dev/null 2>&1; then + log "warning: proof /health unavailable (skipping submit probe)" + return 0 + fi + log "proof submit e2e probe against ${base}" + PROOF_E2E_BASE="$base" "$ROOT/deploy/scripts/proof-submit-e2e.sh" --probe "$base" +} + print_summary() { local pub="" if [[ -f "$TUNNEL_ENV" ]]; then @@ -678,6 +690,7 @@ Internal (compose network): bounty: http://127.0.0.1:${BOUNTY_HOST_PORT}/health (scorer: GET /v1/status → scoring_backend, can_score) proof: http://127.0.0.1:${PROOF_HOST_PORT}/health + (submit: docs/runbooks/proof-submit-e2e.md) EOF if [[ -n "$pub" ]]; then @@ -773,6 +786,7 @@ wait_all_health || die "health checks failed — see logs above" # tunnel flake cannot mask a weights regression. probe_weights_latest || die "weights seal smoke failed" probe_bounty_fail_closed +probe_proof_submit || die "proof submit e2e probe failed" if [[ "$DO_TUNNEL" -eq 1 ]]; then start_tunnel diff --git a/deploy/scripts/proof-submit-e2e.sh b/deploy/scripts/proof-submit-e2e.sh new file mode 100755 index 000000000..8ab1ead42 --- /dev/null +++ b/deploy/scripts/proof-submit-e2e.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Proof submit → score E2E (staging / local sim only). +# +# Never: production hosts, set_weights, master Lium rent. +# POST /v1/submissions is skipped when eval_backend=lium and can_score=true. +# +# Usage: +# ./deploy/scripts/proof-submit-e2e.sh --http-tests +# ./deploy/scripts/proof-submit-e2e.sh --local-sim +# ./deploy/scripts/proof-submit-e2e.sh --probe [BASE] +# ./deploy/scripts/proof-submit-e2e.sh --bounty [BASE] +# ./deploy/scripts/proof-submit-e2e.sh --all +# +# Optional env: +# PROOF_E2E_BASE challenge origin (no trailing slash) +# BOUNTY_E2E_BASE bounty origin +# PROOF_E2E_TOPIC override topic id (default: first of dt-no-ib-v0 / muon-vs-adamw-10m-v0) +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +RED() { printf '\033[31m%s\033[0m\n' "$*"; } +GRN() { printf '\033[32m%s\033[0m\n' "$*"; } +LOG() { printf '[proof-e2e] %s\n' "$*"; } + +PROD_HOSTS='network.cortex.foundation|chain.joinbase.ai' +STAGING_TOPICS=(dt-no-ib-v0 muon-vs-adamw-10m-v0) + +refuse_prod() { + local url="${1:-}" + if echo "$url" | grep -Eq "$PROD_HOSTS"; then + RED "refusing production host: $url" + exit 2 + fi +} + +http_tests() { + LOG "cargo test -p proof-http (in-process submit→score + live skip unless PROOF_E2E_BASE)" + cargo test -p proof-http -- --nocapture + LOG "cargo test -p ctx topic list items wrapper" + cargo test -p ctx -- topic_list_reads_the_items_wrapper + GRN "PASS --http-tests" +} + +local_sim() { + LOG "cargo test -p proof-challenge-bin --test submit_e2e (force_sim binary + both staging topic ids)" + cargo test -p proof-challenge-bin --test submit_e2e -- --nocapture + GRN "PASS --local-sim" +} + +# Probe a running Proof origin. Prints status / topics / submit result shapes. +# Exit 0 on scored 201 or explicit 400/503. Exit 1 on silent empty / unexpected. +probe_proof() { + local base="${1:-${PROOF_E2E_BASE:-}}" + if [[ -z "$base" ]]; then + for cand in \ + http://127.0.0.1:28100 \ + http://127.0.0.1:8100 \ + http://159.223.159.205/challenge/proof \ + http://159.223.159.205:8080/challenge/proof \ + http://159.223.159.205:8100 \ + http://staging.api.joinbase.ai/challenge/proof + do + if curl -fsS -m 3 "$cand/health" >/dev/null 2>&1; then + base="$cand" + break + fi + done + fi + if [[ -z "$base" ]]; then + LOG "no reachable Proof origin (set PROOF_E2E_BASE); skip --probe" + return 0 + fi + base="${base%/}" + refuse_prod "$base" + LOG "probing $base" + + local health status topics code body + health="$(curl -fsS -m 8 "$base/health")" + echo "$health" | grep -q '"challenge_id":"proof"' || { RED "health is not proof: $health"; return 1; } + LOG "GET /health → $health" + + status="$(curl -fsS -m 8 "$base/v1/status")" + LOG "GET /v1/status → $status" + echo "$status" | grep -q '"challenge_id":"proof"' || { RED "status missing challenge_id"; return 1; } + if echo "$status" | grep -q 'api_key\|content_sha256'; then + RED "status leaked a secret or holdout fingerprint" + return 1 + fi + + topics="$(curl -fsS -m 8 "$base/v1/proof/topics")" + LOG "GET /v1/proof/topics → $(echo "$topics" | head -c 400)…" + echo "$topics" | grep -q 'content_sha256' && { RED "topics leaked holdout records"; return 1; } + + code="$(curl -sS -m 8 -o /tmp/proof-e2e-empty.json -w '%{http_code}' \ + -X POST -H 'content-type: application/json' \ + -d '{"miner_hotkey":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","artifact_digest":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","claim":"probe","declared_flops":1,"topic_id":"","manifest":{"train_dataset_ids":["e2e-mix-v0"]}}' \ + "$base/v1/submissions")" + body="$(cat /tmp/proof-e2e-empty.json)" + LOG "POST /v1/submissions empty topic_id → HTTP $code $body" + [[ "$code" == "400" ]] || { RED "empty topic_id expected 400, got $code"; return 1; } + echo "$body" | grep -q '"error"' || { RED "400 had no error field"; return 1; } + + local backend can_score + backend="$(echo "$status" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("eval_backend",""))')" + can_score="$(echo "$status" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("can_score", False))')" + + if [[ "$can_score" == "True" && "$backend" == "lium" ]]; then + LOG "skip POST: host is Lium + can_score (would rent). Fail-closed probe already passed." + GRN "PASS --probe $base (status+400 only; no Lium rent)" + return 0 + fi + + local topics_to_hit=() + if [[ -n "${PROOF_E2E_TOPIC:-}" ]]; then + topics_to_hit=("$PROOF_E2E_TOPIC") + else + for id in "${STAGING_TOPICS[@]}"; do + echo "$topics" | grep -q "\"$id\"" && topics_to_hit+=("$id") + done + if [[ ${#topics_to_hit[@]} -eq 0 ]]; then + topics_to_hit=(dt-no-ib-v0) + fi + fi + + local topic hex sid row any_scored=0 + for topic in "${topics_to_hit[@]}"; do + hex="$(printf '%s' "e2e-$topic-$RANDOM-$$" | sha256sum | awk '{print $1}')" + code="$(curl -sS -m 20 -o /tmp/proof-e2e-submit.json -w '%{http_code}' \ + -X POST -H 'content-type: application/json' \ + -d "{\"miner_hotkey\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"artifact_digest\":\"$hex\",\"claim\":\"e2e sim submit against $topic\",\"declared_flops\":1,\"topic_id\":\"$topic\",\"manifest\":{\"train_dataset_ids\":[\"e2e-mix-v0\"]}}" \ + "$base/v1/submissions")" + body="$(cat /tmp/proof-e2e-submit.json)" + LOG "POST /v1/submissions topic_id=$topic → HTTP $code $body" + case "$code" in + 201) + echo "$body" | grep -q '"id":"pf_' || { RED "201 missing pf_ id"; return 1; } + echo "$body" | grep -q "\"topic_id\":\"$topic\"" || { RED "201 topic_id mismatch"; return 1; } + sid="$(echo "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))')" + row="$(curl -fsS -m 8 "$base/v1/submissions/$sid")" + LOG "GET /v1/submissions/$sid → $row" + echo "$row" | grep -q '"verdict"' || { RED "scored row missing verdict"; return 1; } + any_scored=1 + GRN "PASS --probe $base submit→score HTTP 201 topic=$topic" + ;; + 400|503) + echo "$body" | grep -q '"error"' || { RED "HTTP $code silent empty"; return 1; } + GRN "PASS --probe $base fail-closed HTTP $code topic=$topic (explicit error)" + ;; + *) + RED "unexpected HTTP $code topic=$topic (want 201/400/503, never silent empty)" + return 1 + ;; + esac + done + if [[ "$any_scored" == "1" ]]; then + GRN "PASS --probe $base scored ${#topics_to_hit[@]} topic(s)" + fi +} + +probe_bounty() { + local base="${1:-${BOUNTY_E2E_BASE:-}}" + if [[ -z "$base" ]]; then + for cand in \ + http://127.0.0.1:28096 \ + http://127.0.0.1:8096 \ + http://159.223.159.205:8096 \ + http://159.223.159.205:8080/challenge/bounty \ + http://staging.api.joinbase.ai/challenge/bounty + do + if curl -fsS -m 3 "$cand/health" >/dev/null 2>&1; then + base="$cand" + break + fi + done + fi + if [[ -z "$base" ]]; then + LOG "no reachable Bounty origin (set BOUNTY_E2E_BASE); skip --bounty" + return 0 + fi + base="${base%/}" + refuse_prod "$base" + LOG "probing bounty $base" + local status + status="$(curl -fsS -m 8 "$base/v1/status")" + LOG "GET /v1/status → $status" + echo "$status" | grep -q '"challenge_id":"bounty"' || { RED "not bounty"; return 1; } + + local code + code="$(curl -sS -m 8 -o /tmp/bounty-e2e-report.json -w '%{http_code}' \ + -X POST -H 'content-type: application/json' \ + -d '{"session":"not-a-session","title":"e2e","body":"e2e","repro_steps":"e2e"}' \ + "$base/v1/reports")" + LOG "POST /v1/reports (thin) → HTTP $code $(cat /tmp/bounty-e2e-report.json)" + if echo "$status" | grep -q '"scoring_backend":"unconfigured"'; then + [[ "$code" == "503" ]] || { RED "unconfigured bounty must 503, got $code"; return 1; } + GRN "PASS --bounty $base fail-closed 503" + else + # Feed configured: do not file a real report. Session gate is enough. + [[ "$code" == "401" || "$code" == "400" || "$code" == "503" ]] \ + || { RED "configured bounty unexpected $code"; return 1; } + GRN "PASS --bounty $base ingest reached an explicit gate HTTP $code (no prod write)" + fi +} + +usage() { + sed -n '2,20p' "$0" +} + +DO_HTTP=0 +DO_LOCAL=0 +DO_PROBE=0 +DO_BOUNTY=0 +PROBE_BASE="" +BOUNTY_BASE="" + +if [[ $# -eq 0 ]]; then + usage + exit 1 +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + --http-tests) DO_HTTP=1; shift ;; + --local-sim) DO_LOCAL=1; shift ;; + --probe) + DO_PROBE=1 + if [[ "${2:-}" != --* && -n "${2:-}" ]]; then PROBE_BASE="$2"; shift; fi + shift + ;; + --bounty) + DO_BOUNTY=1 + if [[ "${2:-}" != --* && -n "${2:-}" ]]; then BOUNTY_BASE="$2"; shift; fi + shift + ;; + --all) DO_HTTP=1; DO_LOCAL=1; DO_PROBE=1; DO_BOUNTY=1; shift ;; + -h|--help) usage; exit 0 ;; + *) RED "unknown arg: $1"; usage; exit 1 ;; + esac +done + +[[ "$DO_HTTP" -eq 1 ]] && http_tests +[[ "$DO_LOCAL" -eq 1 ]] && local_sim +[[ "$DO_PROBE" -eq 1 ]] && probe_proof "$PROBE_BASE" +[[ "$DO_BOUNTY" -eq 1 ]] && probe_bounty "$BOUNTY_BASE" +GRN "proof-submit-e2e done" diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 7df8ac766..4cd381c61 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -18,6 +18,7 @@ services: environment: BASE_CHALLENGE_BIND: 0.0.0.0:8100 PROOF_FORCE_SIM: "true" + PROOF_SIM_STUB_WIN: "true" bounty-challenge: ports: - "8096:8096" diff --git a/docker-compose.yml b/docker-compose.yml index 017a18deb..ed18f2f30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -253,6 +253,8 @@ services: BASE_CHALLENGE_SK_FILE: /run/base/challenge_sk # Sim eval unless the operator opts in. Never log LIUM_API_KEY. PROOF_FORCE_SIM: "${PROOF_FORCE_SIM:-false}" + # Staging/dev only. Ignored unless PROOF_FORCE_SIM is on. Never a Lium path. + PROOF_SIM_STUB_WIN: "${PROOF_SIM_STUB_WIN:-false}" PROOF_PIN_FILE: /etc/base/config/proof-pin.toml PROOF_TOPICS_FILE: /run/base/proof/topics.json PROOF_HOLDOUT_FILE: /run/base/proof/holdouts.json diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 7bd57759a..d1a435a59 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -19,6 +19,7 @@ When a spike or evidence report conflicts with a frozen spec or runbook, the nor | [`runbooks/promote-rollback-restore.md`](runbooks/promote-rollback-restore.md) | Digest promote, rollback, Postgres backup/restore | | [`runbooks/local-testnet-e2e.md`](runbooks/local-testnet-e2e.md) | Local laptop/VM full subnet stack on testnet 541 + ephemeral gateway tunnel | | [`runbooks/staging-testnet-e2e.md`](runbooks/staging-testnet-e2e.md) | Staging droplet testnet end-to-end validation | +| [`runbooks/proof-submit-e2e.md`](runbooks/proof-submit-e2e.md) | Proof (and Bounty) submit → score: cargo tests, local `--force-sim`, staging curl/ctx | | [`runbooks/trust-root-rotation.md`](runbooks/trust-root-rotation.md) | Trust-root key rotation | | [`runbooks/gateway-failover.md`](runbooks/gateway-failover.md) | Gateway kill/restart / failover checks | | [`runbooks/measurement-repin-socket-proxy.md`](runbooks/measurement-repin-socket-proxy.md) | Socket-proxy measurement re-pin | diff --git a/docs/PROOF.md b/docs/PROOF.md index c3abb60f7..26960cbe3 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -150,7 +150,11 @@ baseline + an open topic are on the host. publish that topic; scoring fail-closes until the real harness fills `custom_value`. - `PROOF_FORCE_SIM` is CI/local opt-in only. Never a fallback. Forbidden on - droplet overlays. + droplet overlays. Under sim, a sealed topic scores with harness numbers + relative to the seal (`sim_win_document`); skill-only `sim_document` + cannot beat a real ~0.29 NLL baseline. `PROOF_SIM_STUB_WIN` is a leftover + no-op. Resealing staging to `BASELINE_SKILL=0.40` (NLL ≈ 2.94) is an + operator lane, not this binary. - No Modal. No secrets, hosts, holdout records, or teacher endpoints in git. ## Publish a research topic @@ -268,6 +272,12 @@ takes the topic. } ``` +### `muon-vs-adamw-10m-v0` — NLL **wta** + +Operator **example**, not in the pin. Beat sealed AdamW holdout NLL with Muon +at ~10M params under the same FLOP budget. Staging may publish this id next +to `dt-no-ib-v0`; miners still discover it from `GET /v1/proof/topics`. + ### `agent-harness-improve-v0` — custom **discovery** Operator POST, not in git. `custom_id = harness_success_rate` is listed so diff --git a/docs/runbooks/local-testnet-e2e.md b/docs/runbooks/local-testnet-e2e.md index f8076ad73..2c5768293 100644 --- a/docs/runbooks/local-testnet-e2e.md +++ b/docs/runbooks/local-testnet-e2e.md @@ -79,7 +79,14 @@ cargo run -q --release -p weights-smoke -- \ ./deploy/scripts/local-e2e.sh --down ``` -Challenge verification must **simulate a submission** (harness/intake) and probe failures (bad harness, sanitize, quota, routes) in addition to the weights seal smoke above — see root [`AGENTS.md`](../../AGENTS.md). +Challenge verification must **simulate a submission** (harness/intake) and probe failures (bad harness, sanitize, quota, routes) in addition to the weights seal smoke above — see root [`AGENTS.md`](../../AGENTS.md). Proof submit → score (Sim or explicit 400/503) is [`proof-submit-e2e.md`](proof-submit-e2e.md); `--smoke` now probes it when `proof-challenge` is healthy. + +```bash +# In-process + disposable --force-sim binary (no Docker, no Lium): +./deploy/scripts/proof-submit-e2e.sh --http-tests --local-sim +# Against the compose Proof port: +PROOF_E2E_BASE=http://127.0.0.1:28100 ./deploy/scripts/proof-submit-e2e.sh --probe +``` Compose matrix equivalent (what the script runs): diff --git a/docs/runbooks/proof-submit-e2e.md b/docs/runbooks/proof-submit-e2e.md new file mode 100644 index 000000000..e4fac7b66 --- /dev/null +++ b/docs/runbooks/proof-submit-e2e.md @@ -0,0 +1,273 @@ +# Proof submit → score E2E (staging / local sim) + +Operator check that `POST /v1/submissions` returns a **score** or an **explicit +fail-closed reason**. Healthz alone is not enough. + +**Scope:** staging + disposable local sim. **Not** production. +**Never:** `set_weights`, master Lium rent, commit secrets. + +Staging (operator-ready at time of writing): `can_score=true`, +`PROOF_FORCE_SIM=true`, `baseline_sealed=true`, offer +`openrouter-glm53flash-v0`, open topics `dt-no-ib-v0` and +`muon-vs-adamw-10m-v0`. + +### Ownership: StubWin (A) vs reseal (B) + +Skill-only `sim_document` uses `nll = (3.10 - 0.40 * skill).max(1.0)`. +Even skill=1.0 (and `StubScorer::win` skill=0.95) stays at NLL ≥ 1.0, so +a CPU-sealed ~0.29 baseline always trips `quality_floor`. Prefer **A**. + +| Option | Owner | What | +|--------|--------|------| +| **A (lasting)** | this PR / code | Under `PROOF_FORCE_SIM`, a sealed topic emits harness numbers relative to the seal: holdout ≤ baseline+floor, splits ≤ baseline+`epsilon_topic_max_regress`, `tokens_per_sec` ≥ ref×(1+`epsilon_rel`). No extra host env. Lium never takes this path. | +| **B (ops, paused)** | Développeur | Reseal staging to `BASELINE_SKILL=0.40` (NLL ≈ 2.94), resign topics, retest. **Paused** — Mathis redirected that lane to prod RLM E2E (1× GPU). | + +Do **not** reseal or edit staging host files from the code lane. Deploy A. + +Local compose (`env-local.yml`) defaults `LOCAL_PROOF_FORCE_SIM=true`. +`PROOF_SIM_STUB_WIN` is a leftover no-op. Droplet overlays stay sim-off +(`assert-compose-matrix.sh`). + +## Commands (in-repo, CI-safe) + +```bash +# In-process HTTP contract (Sim + fail-closed 400/503). Always run. +./deploy/scripts/proof-submit-e2e.sh --http-tests + +# Spawn proof-challenge --force-sim with synthetic topic/holdout/baseline/offer. +# Scores both staging topic ids. No Docker, no Lium. +./deploy/scripts/proof-submit-e2e.sh --local-sim + +# Equivalent cargo invocations: +cargo test -p proof-http +cargo test -p proof-challenge-bin --test submit_e2e +cargo test -p ctx -- topic_list_reads_the_items_wrapper +``` + +Pass: every test above is green. Fail: any assertion on silent empty body, +missing `error`, missing `verdict` after 201, or holdout leak. + +## Probe a running host (local compose or staging) + +Do **not** point this at `https://network.cortex.foundation` or +`https://chain.joinbase.ai`. + +```bash +# Auto-detect first healthy origin among loopback + documented staging URLs: +./deploy/scripts/proof-submit-e2e.sh --probe + +# Or pin the origin (no trailing slash). Prefer the droplet IP on :80 — +# staging.api.joinbase.ai has historically answered a stale Lium/fail-closed +# instance while 159.223.159.205/challenge/proof is the ready sim host. +PROOF_E2E_BASE=http://127.0.0.1:28100 ./deploy/scripts/proof-submit-e2e.sh --probe +PROOF_E2E_BASE=http://159.223.159.205/challenge/proof \ + ./deploy/scripts/proof-submit-e2e.sh --probe + +# Same contract as a Rust test (skip if unset): +PROOF_E2E_BASE=http://127.0.0.1:28100 cargo test -p proof-http --test live_submit_e2e +``` + +The probe **skips POST** when `eval_backend=lium` and `can_score=true` +(would rent a miner-paid pod). Staging sim is the intended POST target. + +### Exact curl (Proof) + +Gateway prefix on staging is `/challenge/proof` (reachable on the droplet +at `http://159.223.159.205/challenge/proof`; host-local +`http://127.0.0.1:8080/challenge/proof/...`). Direct service is `:8100` +(local overlay `:28100`). Do not POST to `staging.api.joinbase.ai` while +it still reports `eval_backend=lium`. + +```bash +BASE="${PROOF_E2E_BASE:-http://127.0.0.1:28100}" # or …/challenge/proof + +curl -sS "$BASE/health" +# {"ok":true,"challenge_id":"proof","scoring_version":1} + +curl -sS "$BASE/v1/status" +# { +# "challenge_id": "proof", +# "eval_backend": "sim", +# "force_sim": true, +# "sim_stub_win": true, +# "can_score": true, +# "baseline_sealed": true, +# "open_topics": ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"], +# "inference_offer": { "offer_id": "openrouter-glm53flash-v0", "status": "open", ... } +# } +# Never contains api_key, base_url, or holdout records. + +curl -sS "$BASE/v1/proof/topics" +# { "items": [ { "id": "dt-no-ib-v0", ... }, { "id": "muon-vs-adamw-10m-v0", ... } ] } +# Never contains content_sha256. + +curl -sS -X POST "$BASE/v1/submissions" \ + -H 'content-type: application/json' \ + -d '{ + "miner_hotkey": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "topic_id": "dt-no-ib-v0", + "artifact_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "claim": "beats the sealed reference under the cap", + "declared_flops": 1000000000000, + "manifest": { "train_dataset_ids": ["e2e-mix-v0"] } + }' +``` + +### Exact ctx (Proof) + +`ctx` talks to a **gateway** (`/challenge/proof/...`). For local compose: + +```bash +ctx --gateway http://127.0.0.1:8080 proof status +ctx --gateway http://127.0.0.1:8080 proof topics +ctx --gateway http://127.0.0.1:8080 proof submit \ + --hotkey aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + --topic-id dt-no-ib-v0 \ + --artifact-digest bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb \ + --claim "beats the sealed reference under the cap" \ + --declared-flops 1000000000000 \ + --train-dataset e2e-mix-v0 +ctx --gateway http://127.0.0.1:8080 proof show +``` + +On staging, talk to the droplet gateway +(`http://159.223.159.205`, host-local `http://127.0.0.1:8080`). Do not +use `http://staging.api.joinbase.ai` while it still reports +`eval_backend=lium`. Do not send `X-Lium-Api-Key` over `http://` — +`ctx` refuses keyed cleartext. + +Second topic (same body, different id): + +```bash +# topic_id: muon-vs-adamw-10m-v0 +``` + +### Expected response shapes (no secrets) + +**201 scored** (`SubmitResp` then full row on GET): + +```json +{ + "id": "pf_<16 hex>", + "submission_digest": "<64 hex>", + "topic_id": "dt-no-ib-v0", + "state": "awaiting_admin", + "eval_backend": "sim", + "eligible": true +} +``` + +`state` may be `rejected` (gates failed) — that is still a **score**, not +silence. GET `/v1/submissions/{id}` then includes: + +```json +{ + "id": "pf_…", + "topic_id": "dt-no-ib-v0", + "claim": "…", + "declared_flops": 1000000000000, + "state": "awaiting_admin", + "verdict": { + "pass": true, + "agent": { "verdict": "clean", "reproduced": true, "rationale": "sim reproduced", "topic_id": "dt-no-ib-v0" }, + "harness": { "holdout_nll": 0.29, "tokens_per_sec": 84.8 }, + "failed": [], + "lattice": 65535 + }, + "receipt_json": "{\"provider\":\"sim\",…}" +} +``` + +**400 / 503 fail-closed** (no row, no rent): + +```json +{ "error": "topic_id is required" } +{ "error": "unknown topic" } +{ "error": "declared_flops exceeds the topic budget" } +{ "error": "inference offer missing; refuse scoring" } +``` + +A 2xx/4xx/5xx with an empty `{}` and no `id` / no `error` is a **fail**. + +| HTTP | When | Stored? | +|------|------|---------| +| 201 | Sim (or stub) finished judge+harness | yes | +| 400 | bad/missing/unknown/not-open `topic_id`, bad hex, FLOP over budget | no | +| 503 | host cannot score (no open sealed topic, no offer, unpinned Lium, …) | no | + +## Local compose (disposable) + +`env-local.yml` defaults `LOCAL_PROOF_FORCE_SIM=true`. Droplet overlays +(`env-staging.yml` / `env-prod.yml`) must keep `PROOF_FORCE_SIM` (and the +leftover `PROOF_SIM_STUB_WIN`) false; `assert-compose-matrix.sh` fails if +they do not. A staging **host** may already have `PROOF_FORCE_SIM=true` +in `deploy/env/proof-challenge.env` — that is operator state, not a git +overlay. Do not reseal from this lane. + +```bash +./deploy/scripts/materialize-env.sh +./deploy/scripts/local-e2e.sh --smoke --no-tunnel +# soft-probes Proof health + this submit contract when the service is up + +# Minimal cleartext (no testnet): +docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d proof-challenge +# still needs operator files under deploy/secrets/proof/ or submits 503 +# Prefer --local-sim (self-contained fixtures) when those files are absent. +``` + +`local-e2e.sh` will not invent a sha256 digest and will not POST if the +running host is Lium + `can_score`. + +## Bounty smoke (optional) + +```bash +./deploy/scripts/proof-submit-e2e.sh --bounty +# GET /v1/status → scoring_backend, can_score, backend_public_configured +# unconfigured feed: POST /v1/reports → 503 + error (no offline scorer) +# configured feed: thin POST must 401/400/503 — do not file a real report +``` + +## Pass / fail log (fill when you run) + +| Step | Command | Result | +|------|---------|--------| +| In-process Sim submit | `cargo test -p proof-http sim_submit` | PASS (in-repo) | +| StubWin → awaiting_admin | `cargo test -p proof-http sim_stub_win_submit_reaches_awaiting_admin` | PASS (in-repo) | +| Sealed-relative win (0.29 NLL) | `cargo test -p proof-eval stub_win_clears_quality_floor` | PASS (in-repo) | +| Both staging topic ids | `cargo test -p proof-http sim_submit_accepts_staging_topic_ids` | PASS (in-repo) | +| Process-level `--force-sim` | `cargo test -p proof-challenge-bin --test submit_e2e` | PASS (in-repo) | +| Live probe | `PROOF_E2E_BASE=http://159.223.159.205/challenge/proof ./deploy/scripts/proof-submit-e2e.sh --probe` | PASS 201×2 `rejected` (see below) | +| Live Rust | `PROOF_E2E_BASE=http://159.223.159.205/challenge/proof cargo test -p proof-http --test live_submit_e2e` | PASS | + +### Live staging 2026-09-07 (sim, no Lium, no merge) + +Origin: `http://159.223.159.205/challenge/proof` (`eval_backend=sim`, +`force_sim=true`, `can_score=true`, `baseline_sealed=true`, offer +`openrouter-glm53flash-v0` open). Status has **no** `sim_stub_win` field +— host binary predates this PR / env is unset. + +| topic | HTTP | id | state | eligible | gates | +|-------|------|----|-------|----------|-------| +| (empty) | 400 | — | — | — | `topic_id is required` | +| `not-a-real-topic` | 400 | — | — | — | `unknown topic` | +| `dt-no-ib-v0` | 201 | `pf_0000000000000002` | `rejected` | false | QualityFloor holdout 2.827 vs baseline 0.291 floor 0.02; split_regress; ThroughputMiss 113.9 vs 213.4 | +| `muon-vs-adamw-10m-v0` | 201 | `pf_0000000000000003` | `rejected` | false | NllMiss holdout 3.042 vs baseline 0.344 ε 0.02; split_regress | + +Receipts: `"provider":"sim"`. Agent: `clean` / `reproduced`. No rent. +`staging.api.joinbase.ai` still answers `eval_backend=lium` / +`can_score=false` / empty topics — do not POST there. + +To reach `awaiting_admin` on this host (**option A**): deploy this branch +(no host-file edit, no reseal). Status then reports `sim_stub_win=true` +whenever `eval_backend=sim`. Re-run `--probe`. **Option B** (reseal to +`BASELINE_SKILL=0.40` / NLL ≈ 2.94 and resign topics) is Développeur-only +— do not reseal from this lane. Admin adjudicate needs the host bearer at +`/opt/base/deploy/secrets/proof/admin_tokens` (do not log). No +`set_weights`. + +## Related + +- Miner HTTP: [`../external-miner/proof.md`](../external-miner/proof.md) +- Operator Proof: [`../PROOF.md`](../PROOF.md) +- Local stack: [`local-testnet-e2e.md`](local-testnet-e2e.md) +- Staging droplets: [`staging-testnet-e2e.md`](staging-testnet-e2e.md) diff --git a/docs/runbooks/staging-testnet-e2e.md b/docs/runbooks/staging-testnet-e2e.md index 2fc3a2721..7a0335f40 100644 --- a/docs/runbooks/staging-testnet-e2e.md +++ b/docs/runbooks/staging-testnet-e2e.md @@ -117,4 +117,15 @@ git checkout - `FakeChain` is the default backend; `BASE_CHAIN_BACKEND=live` switches to `chain-live`. - CRV4 tlock encryption is implemented (`tle` / Drand Quicknet); when commit-reveal is off, `set_weights` is used instead. -- Proof live submits stay **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. +- Proof live (Lium) submits stay **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. +- When staging is opted into **sim** (`PROOF_FORCE_SIM=true` on the host, not in `env-staging.yml`), submit → score is the contract in [`proof-submit-e2e.md`](proof-submit-e2e.md). Topic ids in that window: `dt-no-ib-v0`, `muon-vs-adamw-10m-v0`. **Option A (code):** a sealed topic under `force_sim` emits a sealed-relative harness (deploy the binary; do not edit host files). **Option B (Développeur, paused):** reseal to `BASELINE_SKILL=0.40` (NLL ≈ 2.94); Mathis redirected that lane to prod RLM E2E. Do not reseal from the code lane. Do not `set_weights`. Do not POST if `eval_backend` is `lium` and `can_score` is true. + +```bash +# From a machine that can reach the staging gateway (no SSH required). +# Prefer the droplet IP: staging.api.joinbase.ai has answered a stale +# Lium/fail-closed instance while 159.223.159.205 is the ready sim host. +PROOF_E2E_BASE=http://159.223.159.205/challenge/proof \ + ./deploy/scripts/proof-submit-e2e.sh --probe +# Host-local: http://127.0.0.1:8080/challenge/proof +# Direct service if published: http://:8100 +```