From 217f1a9e89ded6727454162e81ea9b53f5d5c936 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:01:18 +0800 Subject: [PATCH 01/15] Archive v0.25: derive temporal execution and evidence context changes --- .../src/archive-context-transitions.ts | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 packages/persistence/src/archive-context-transitions.ts diff --git a/packages/persistence/src/archive-context-transitions.ts b/packages/persistence/src/archive-context-transitions.ts new file mode 100644 index 0000000..b3bfb3b --- /dev/null +++ b/packages/persistence/src/archive-context-transitions.ts @@ -0,0 +1,96 @@ +import type { ArchiveRunView } from "./archive-repository.js"; + +export interface ArchiveContextDifference { + readonly field: string; + readonly previous: string; + readonly current: string; + readonly kind: "execution" | "evidence"; +} + +export interface ArchiveContextTransition { + readonly runId: string; + readonly previousRunId: string | null; + readonly status: "baseline" | "unchanged" | "changed" | "evidence_changed" | "unknown"; + readonly changes: readonly ArchiveContextDifference[]; + readonly unknownFields: readonly string[]; + readonly caveats: readonly string[]; +} + +/** + * Compare archived snapshots, never mutable current Provider/Runner state. + * Unknown values must not silently compare as equal or count as a change. + */ +export function archiveContextTransitions( + runs: readonly ArchiveRunView[], +): readonly ArchiveContextTransition[] { + const projection = (run: ArchiveRunView) => { + const q = run.executionQualification; + const effectiveTier = q?.returnedServiceTier ?? q?.serviceTier ?? null; + return [ + ["execution_path", run.executionPath, "execution"], + ["environment", q?.executionEnvironment?.id ?? null, "execution"], + ["region", q?.executionRegion ?? null, "execution"], + ["account_tier", q?.accountTier ?? null, "execution"], + ["service_tier", q?.serviceTier ?? null, "execution"], + ["effective_service_tier", effectiveTier, "execution"], + ["service_assurance", q?.serviceAssurance ?? null, "execution"], + ["provider_policy_observation", q?.providerPolicyObservation?.id ?? null, "evidence"], + ["runner_access_observation", q?.runnerAccessObservation?.id ?? null, "evidence"], + ["fleet_capability_event", q?.executionEnvironment?.capabilityEventId ?? null, "evidence"], + ] as const; + }; + + return runs.map((run, index) => { + const prior = runs[index - 1]; + const caveats = [ + ...(run.executionQualification?.caveats ?? ["execution_qualification_missing"]), + ...(run.cost?.caveats ?? []), + ]; + if (!run.executionQualification?.executionEnvironment) { + caveats.push("execution_environment_unassigned"); + } + + if (!prior) { + return { + runId: run.id, + previousRunId: null, + status: "baseline" as const, + changes: [], + unknownFields: [], + caveats: [...new Set(caveats)], + }; + } + + const before = projection(prior); + const after = projection(run); + const changes: ArchiveContextDifference[] = []; + const unknownFields: string[] = []; + for (let i = 0; i < before.length; i += 1) { + const [field, previous, kind] = before[i]!; + const current = after[i]![1]; + if (previous === null || current === null) { + unknownFields.push(field); + } else if (previous !== current) { + changes.push({ field, previous, current, kind }); + } + } + + const executionChanged = changes.some((change) => change.kind === "execution"); + const status: ArchiveContextTransition["status"] = executionChanged + ? "changed" + : unknownFields.length > 0 + ? "unknown" + : changes.length > 0 + ? "evidence_changed" + : "unchanged"; + + return { + runId: run.id, + previousRunId: prior.id, + status, + changes, + unknownFields, + caveats: [...new Set(caveats)], + }; + }); +} From 2a6222574d339218757aca003a55d930a65980bb Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:01:27 +0800 Subject: [PATCH 02/15] Archive v0.25: include explicit context transitions in history --- packages/persistence/src/archive-repository.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index 0655579..cddce59 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { Pool } from "pg"; import type { ComparabilityPolicyView } from "./comparability-repository.js"; +import { archiveContextTransitions, type ArchiveContextTransition } from "./archive-context-transitions.js"; export interface ArchiveModelView { readonly id: string; @@ -457,6 +458,7 @@ export interface ArchiveRunHistoryView { readonly model: ArchiveModelView; readonly test: ArchiveTestView; readonly runs: readonly ArchiveRunView[]; + readonly contextTransitions: readonly ArchiveContextTransition[]; readonly relations: readonly ArchiveRunRelationEdgeView[]; } @@ -1382,6 +1384,7 @@ export class PgArchiveRepository { model, test, runs, + contextTransitions: archiveContextTransitions(runs), relations, }; } From 51389a050f6e693905646309806c34497b20a158 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:01:45 +0800 Subject: [PATCH 03/15] Archive v0.25: type public context transitions and expose policy catalog --- apps/web/src/modelapse.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index d6fe48f..cc71086 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -550,10 +550,25 @@ interface ArchiveRunDetailWire extends ArchiveRun { readonly relations: readonly ArchiveRunRelation[]; } +export interface ArchiveContextTransition { + readonly runId: string; + readonly previousRunId: string | null; + readonly status: "baseline" | "unchanged" | "changed" | "evidence_changed" | "unknown"; + readonly changes: readonly { + readonly field: string; + readonly previous: string; + readonly current: string; + readonly kind: "execution" | "evidence"; + }[]; + readonly unknownFields: readonly string[]; + readonly caveats: readonly string[]; +} + export interface ArchiveRunHistory { readonly model: ArchiveModel; readonly test: ArchiveTest; readonly runs: readonly ArchiveRun[]; + readonly contextTransitions: readonly ArchiveContextTransition[]; readonly relations: readonly ArchiveRunRelationEdge[]; } @@ -562,6 +577,7 @@ export interface ArchiveTemporalComparison { readonly rows: readonly { readonly model: ArchiveModel; readonly runs: readonly ArchiveRun[]; + readonly contextTransitions: readonly ArchiveContextTransition[]; readonly relations: readonly ArchiveRunRelationEdge[]; }[]; } @@ -2242,6 +2258,15 @@ export const getArchiveCatalogChanges = createServerFn({ method: "GET" }).handle }, ); +export const getArchiveComparabilityPolicies = createServerFn({ method: "GET" }).handler( + async (): Promise => { + const result = await requestJson<{ policies: readonly ArchiveComparabilityPolicy[] }>( + "/v1/archive/comparability/policies", + ); + return result.policies; + }, +); + export const getArchiveCatalog = createServerFn({ method: "GET" }).handler( async (): Promise => { const [models, tests] = await Promise.all([ @@ -2347,6 +2372,7 @@ export const compareArchiveHistory = createServerFn({ method: "POST" }) rows: complete.map((history) => ({ model: history.model, runs: history.runs, + contextTransitions: history.contextTransitions, relations: history.relations, })), }; From d7b68fff5d6733430974abd7fd0aa3ad42401bd4 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:01:59 +0800 Subject: [PATCH 04/15] Archive v0.25: render field-level longitudinal context and caveats --- .../routes/history.$modelId.$testCaseId.tsx | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/apps/web/src/routes/history.$modelId.$testCaseId.tsx b/apps/web/src/routes/history.$modelId.$testCaseId.tsx index 511863a..12ce718 100644 --- a/apps/web/src/routes/history.$modelId.$testCaseId.tsx +++ b/apps/web/src/routes/history.$modelId.$testCaseId.tsx @@ -131,13 +131,16 @@ function ArchiveRunHistoryPage() {

Same model × same Test

- Runs are ordered by capture time. Changes are shown as records; no causal - explanation or quality trend is inferred. + Every transition compares frozen execution snapshots and evidence references. + A changed region, account tier, service tier, or Fleet identity is not proof + that the model itself changed. Missing evidence remains unknown.

- {history.runs.map((run, index) => ( + {history.runs.map((run, index) => { + const transition = history.contextTransitions[index]; + return (
{String(index + 1).padStart(2, "0")}
@@ -148,20 +151,43 @@ function ArchiveRunHistoryPage() { {evaluationLabel(run)} evidence {run.evidenceLevel ?? "—"} - {run.executionQualification?.contextKey - ? "context captured" - : "context unknown"} + context {transition?.status ?? "unknown"}
{run.returnedModel ?? run.requestedModel} {run.executionPath} + + {run.executionQualification + ? [ + run.executionQualification.executionRegion ?? "region unknown", + run.executionQualification.accountTier ?? "account unknown", + run.executionQualification.returnedServiceTier ?? + run.executionQualification.serviceTier ?? + "service unknown", + run.executionQualification.serviceAssurance, + ].join(" · ") + : "execution qualification unavailable"} + + {transition?.changes.map((change) => ( + + {change.kind} · {change.field.replaceAll("_", " ")}:{" "} + {change.previous} → {change.current} + + ))} + {transition?.unknownFields.length ? ( + Unresolved: {transition.unknownFields.join(", ")} + ) : null} + {transition?.caveats.length ? ( + Caveats: {transition.caveats.join(" · ")} + ) : null}
Run {run.id.slice(0, 8)} →
- ))} + ); + })} {history.runs.length === 0 ? (
No sealed public Runs exist for this pair yet.
) : null} From 6fad100c65025a8eb3e511ade3f611f381de2e36 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:02:21 +0800 Subject: [PATCH 05/15] Archive v0.25: public policy switch, cross-provider defaults and visible run caveats --- apps/web/src/routes/compare.tsx | 99 ++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 8 deletions(-) diff --git a/apps/web/src/routes/compare.tsx b/apps/web/src/routes/compare.tsx index 628ed36..542a4c3 100644 --- a/apps/web/src/routes/compare.tsx +++ b/apps/web/src/routes/compare.tsx @@ -4,13 +4,20 @@ import { compareArchive, compareArchiveHistory, getArchiveCatalog, + getArchiveComparabilityPolicies, type ArchiveComparison, type ArchiveRun, type ArchiveTemporalComparison, } from "../modelapse"; export const Route = createFileRoute("/compare")({ - loader: () => getArchiveCatalog(), + loader: async () => { + const [catalog, policies] = await Promise.all([ + getArchiveCatalog(), + getArchiveComparabilityPolicies(), + ]); + return { catalog, policies }; + }, component: ArchiveComparePage, }); @@ -44,8 +51,15 @@ function contextRelation( } function ArchiveComparePage() { - const catalog = Route.useLoaderData(); - const defaultModelIds = catalog.models.slice(0, 2).map((model) => model.id); + const { catalog, policies } = Route.useLoaderData(); + const firstModel = catalog.models[0]; + const secondModel = + catalog.models.find( + (model) => firstModel && model.provider.id !== firstModel.provider.id, + ) ?? catalog.models[1]; + const defaultModelIds = [firstModel?.id, secondModel?.id].filter( + (id): id is string => Boolean(id), + ); const comparableTests = catalog.tests.filter( (test) => test.category !== "calibration", ); @@ -53,6 +67,7 @@ function ArchiveComparePage() { const [testCaseId, setTestCaseId] = useState( comparableTests[0]?.testCaseId ?? "", ); + const [policyVersion, setPolicyVersion] = useState(policies[0]?.version ?? ""); const [comparison, setComparison] = useState(null); const [temporal, setTemporal] = useState(null); const [busy, setBusy] = useState(false); @@ -90,6 +105,7 @@ function ArchiveComparePage() { data: { modelIds, testCaseId, + ...(policyVersion ? { policyVersion } : {}), }, }), compareArchiveHistory({ @@ -171,6 +187,28 @@ function ArchiveComparePage() { Latest comparison and temporal lanes use this exact public Test Case. +
02 / Models @@ -253,7 +291,7 @@ function ArchiveComparePage() { {comparison.comparabilitySet.reasons.length > 0 ? comparison.comparabilitySet.reasons.join(" · ") - : "All selected latest Runs satisfy the current set policy."} + : "All selected latest Runs satisfy this policy. This is not a quality ranking."}
@@ -308,6 +346,25 @@ function ArchiveComparePage() {
Execution
{run.executionPath}
+
+
Region / account
+
+ {run.executionQualification?.executionRegion ?? "unknown"} + {" · "} + {run.executionQualification?.accountTier ?? "unknown"} +
+
+
+
Service / assurance
+
+ {run.executionQualification?.returnedServiceTier ?? + run.executionQualification?.serviceTier ?? + "unknown"} + {" · "} + {run.executionQualification?.serviceAssurance ?? + "unknown"} +
+
Evaluator
@@ -335,9 +392,21 @@ function ArchiveComparePage() { {row.comparability?.reasons.length ? (

- {row.comparability.reasons.join(" · ")} + Comparability: {row.comparability.reasons.join(" · ")}

) : null} +

+ Execution caveats:{" "} + {run.executionQualification + ? run.executionQualification.caveats.join(" · ") || "none reported" + : "qualification unavailable"} +

+

+ Cost caveats:{" "} + {run.cost + ? run.cost.caveats.join(" · ") || "none reported" + : "cost evidence unavailable"} +

) : (
@@ -408,10 +477,24 @@ function ArchiveComparePage() { {run.evidenceLevel ?? "—"} · {run.returnedModel ?? run.requestedModel} {" · "} - {run.executionQualification?.contextKey - ? "context captured" - : "context unknown"} + context {row.contextTransitions[index]?.status ?? "unknown"} + {row.contextTransitions[index]?.changes.map((change) => ( + + {change.kind} · {change.field.replaceAll("_", " ")}: + {" "}{change.previous} → {change.current} + + ))} + {row.contextTransitions[index]?.unknownFields.length ? ( + + Unknown: {row.contextTransitions[index]?.unknownFields.join(", ")} + + ) : null} + {row.contextTransitions[index]?.caveats.length ? ( + + Caveats: {row.contextTransitions[index]?.caveats.join(" · ")} + + ) : null} ))} {row.runs.length === 0 ? ( From 8edab76933ec922936ea9b48ab65afadd4d9509a Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:02:54 +0800 Subject: [PATCH 06/15] Archive v0.25: extend public history API fixture with context transitions --- apps/api/test/archive-api.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index 6eb23b8..eefb261 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -214,6 +214,24 @@ const archiveRunHistory = { }, archiveRun, ], + contextTransitions: [ + { + runId: PREVIOUS_RUN_ID, + previousRunId: null, + status: "baseline", + changes: [], + unknownFields: [], + caveats: [], + }, + { + runId: RUN_ID, + previousRunId: PREVIOUS_RUN_ID, + status: "unchanged", + changes: [], + unknownFields: [], + caveats: [], + }, + ], relations: [ { fromRunId: RUN_ID, From c080354bcf36a222b535cd31dc6a3d9b5a9414f0 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:03:10 +0800 Subject: [PATCH 07/15] Test frozen context changes, evidence-only refreshes and unknown history --- .../test/archive-context-transitions.test.ts | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 packages/persistence/test/archive-context-transitions.test.ts diff --git a/packages/persistence/test/archive-context-transitions.test.ts b/packages/persistence/test/archive-context-transitions.test.ts new file mode 100644 index 0000000..f601d3a --- /dev/null +++ b/packages/persistence/test/archive-context-transitions.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; +import { archiveContextTransitions } from "../src/archive-context-transitions.js"; +import type { ArchiveRunView } from "../src/archive-repository.js"; + +function run(id: string, options: { + region?: string | null; + account?: string | null; + policyId?: string | null; + caveats?: string[]; +} = {}): ArchiveRunView { + return { + id, + executionPath: "first_party_direct", + executionQualification: { + selectedAt: "2026-10-08T00:00:00.000Z", + executionEnvironment: { + id: "fleet-primary", + slug: "us-paid", + capabilityEventId: "capability-v1", + }, + executionRegion: options.region === undefined ? "US" : options.region, + accountTier: options.account === undefined ? "paid" : options.account, + serviceTier: "default", + requestedServiceTier: "default", + returnedServiceTier: "default", + serviceAssurance: "documented_default", + providerPolicyObservation: options.policyId === null + ? null + : { + id: options.policyId ?? "policy-v1", + sourceId: "source", + accessState: "available", + }, + runnerAccessObservation: { + id: "runner-v1", + sourceId: "source", + accessState: "available", + }, + caveats: options.caveats ?? [], + contextKey: "frozen", + }, + cost: { + selectedAt: "2026-10-08T00:00:00.000Z", + pricingObservation: null, + pricing: null, + usage: null, + estimatedNativeCost: null, + caveats: ["native_cost_unavailable"], + }, + } as ArchiveRunView; +} + +describe("public Archive longitudinal context transitions", () => { + it("differentiates identical snapshots from evidence refreshes and execution changes", () => { + const transitions = archiveContextTransitions([ + run("a"), + run("b"), + run("c", { policyId: "policy-v2" }), + run("d", { policyId: "policy-v2", region: "JP" }), + ]); + expect(transitions.map((entry) => entry.status)).toEqual([ + "baseline", + "unchanged", + "evidence_changed", + "changed", + ]); + expect(transitions[2]?.changes).toEqual([ + { + field: "provider_policy_observation", + previous: "policy-v1", + current: "policy-v2", + kind: "evidence", + }, + ]); + expect(transitions[3]?.changes).toEqual([ + { + field: "region", + previous: "US", + current: "JP", + kind: "execution", + }, + ]); + expect(transitions[3]?.previousRunId).toBe("c"); + }); + + it("preserves unknown provenance and displays historical caveats", () => { + const [baseline, unknown] = archiveContextTransitions([ + run("a"), + run("b", { + region: null, + account: null, + policyId: null, + caveats: ["execution_region_unknown"], + }), + ]); + expect(baseline?.status).toBe("baseline"); + expect(unknown?.status).toBe("unknown"); + expect(unknown?.changes).toEqual([]); + expect(unknown?.unknownFields).toEqual( + expect.arrayContaining(["region", "account_tier", "provider_policy_observation"]), + ); + expect(unknown?.caveats).toEqual( + expect.arrayContaining(["execution_region_unknown", "native_cost_unavailable"]), + ); + }); + + it("does not invent transitions or infer model degradation from a single Run", () => { + const transitions = archiveContextTransitions([run("only")]); + expect(transitions).toMatchObject([{ + runId: "only", + previousRunId: null, + status: "baseline", + changes: [], + }]); + expect(archiveContextTransitions([])).toEqual([]); + }); +}); From c842d3e2b9bdb4111362f9566b4966639b384e2f Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:03:18 +0800 Subject: [PATCH 08/15] Run Archive context transition unit tests in CI --- packages/persistence/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/persistence/package.json b/packages/persistence/package.json index 157e17f..8bcb3ef 100644 --- a/packages/persistence/package.json +++ b/packages/persistence/package.json @@ -10,7 +10,7 @@ "scripts": { "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "vitest run test/record-provider-run.test.ts", + "test": "vitest run test/record-provider-run.test.ts test/archive-context-transitions.test.ts", "test:integration": "vitest run test/integration.test.ts test/comparability.integration.test.ts" }, "dependencies": { From 396ff82eaf6a142a6b864b16538a2b88b3269905 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:03:30 +0800 Subject: [PATCH 09/15] Validate longitudinal region drift against persisted Run qualifications --- .../test/comparability.integration.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/persistence/test/comparability.integration.test.ts b/packages/persistence/test/comparability.integration.test.ts index ef02ecc..e4bf6a6 100644 --- a/packages/persistence/test/comparability.integration.test.ts +++ b/packages/persistence/test/comparability.integration.test.ts @@ -470,6 +470,25 @@ describe("Archive comparability policy", () => { expect(regionMismatch?.comparabilitySet.status).toBe("mismatched"); expect(regionMismatch?.comparabilitySet.reasons).toContain("region_mismatch"); + const modelHistory = await archive.getRunHistory({ + modelId: models[1]!, + testCaseId: benchmarkCase.rows[0]!.id, + }); + expect(modelHistory?.runs).toHaveLength(2); + expect(modelHistory?.contextTransitions).toHaveLength(2); + expect(modelHistory?.contextTransitions[0]?.status).toBe("baseline"); + expect(modelHistory?.contextTransitions[1]).toMatchObject({ + status: "changed", + changes: expect.arrayContaining([ + { + field: "region", + previous: "US", + current: "JP", + kind: "execution", + }, + ]), + }); + const custom = await policies.recordPolicy({ version: "comparability-integration-" + suffix, minimumEvidenceLevel: "E4", From 432d35ff9762144d8d4ae7dcdebc59d21cb44706 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:03:51 +0800 Subject: [PATCH 10/15] Avoid cross-provider false mismatches from incomparable evidence IDs --- apps/web/src/routes/compare.tsx | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/web/src/routes/compare.tsx b/apps/web/src/routes/compare.tsx index 542a4c3..3850941 100644 --- a/apps/web/src/routes/compare.tsx +++ b/apps/web/src/routes/compare.tsx @@ -44,10 +44,23 @@ function contextRelation( run: ArchiveRun, reference: ArchiveRun | null, ): "matched" | "mismatched" | "unknown" { - const context = run.executionQualification?.contextKey; - const referenceContext = reference?.executionQualification?.contextKey; - if (!context || !referenceContext) return "unknown"; - return context === referenceContext ? "matched" : "mismatched"; + if (!reference) return "unknown"; + const execution = (value: ArchiveRun): readonly (string | null)[] => [ + value.executionPath, + value.executionQualification?.executionRegion ?? null, + value.executionQualification?.accountTier ?? null, + value.executionQualification?.returnedServiceTier ?? + value.executionQualification?.serviceTier ?? null, + value.executionQualification?.serviceAssurance ?? null, + ]; + const current = execution(run); + const baseline = execution(reference); + if (current.some((value) => value === null) || baseline.some((value) => value === null)) { + return "unknown"; + } + return current.every((value, index) => value === baseline[index]) + ? "matched" + : "mismatched"; } function ArchiveComparePage() { From 1ac4aec81705d56d2be20c238d253ab3ccb963d1 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:04:12 +0800 Subject: [PATCH 11/15] Document v0.25 public evidence-first comparisons and temporal context --- docs/public-comparability-longitudinal.md | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/public-comparability-longitudinal.md diff --git a/docs/public-comparability-longitudinal.md b/docs/public-comparability-longitudinal.md new file mode 100644 index 0000000..0656089 --- /dev/null +++ b/docs/public-comparability-longitudinal.md @@ -0,0 +1,62 @@ +# Public Comparability & Longitudinal Context — Archive v0.25 + +Archive v0.25 builds a **public, evidence-first** comparison surface on top of Archive v0.19–v0.24. It does not introduce aggregate model scores, rankings, or unsupported quality claims. + +## Public comparison + +`/compare` selects 2–4 canonical Models and an **exact public non-calibration Test Case**. The initial selection prefers Models from two different Providers, when present. + +The public API `GET /v1/archive/compare?modelIds=…&testCaseId=…&policyVersion=…` already evaluates Runs under v0.23 versioned comparability policies. The UI now surfaces policy selection, so visitors can inspect the same immutable evidence under different **explicit policy versions**, without rewriting Run or Evaluation. + +Beside each latest result the public UI shows: + +- comparability outcome and machine-readable reasons; +- replication requirement and achieved count; +- same-context calibration status and anomaly streak; +- execution region and account tier; +- effective service tier and documented assurance; +- frozen execution qualification caveats; +- frozen cost-evidence caveats, including unavailable pricing or usage. + +The comparison result may be `matched`, `mismatched`, or `unknown`. These describe whether the Runs are eligible to be compared *under the chosen policy*, not which model performs better. + +A **cross-provider execution-context badge** checks execution path, region, account, effective service tier, and assurance. It intentionally excludes Provider-specific policy/runner observation IDs from equality checks. Those references are independent evidence and must not create false environment mismatches. + +## Public longitudinal history + +`GET /v1/archive/history?modelId=…&testCaseId=…&limit=…` now includes `contextTransitions`, aligned one-to-one with the returned `runs`, in capture-time order. + +Each derived transition contains: + +- `runId`, `previousRunId`; +- status: `baseline`, `unchanged`, `changed`, `evidence_changed`, or `unknown`; +- field-level changes with previous/current values and `execution` or `evidence` category; +- unresolved field names, instead of treating missing values as equal; +- the Run's frozen qualification/cost caveats. + +Execution changes cover execution path, Fleet environment ID, region, account tier, service tier, effective returned service tier, and service assurance. + +Evidence changes cover Provider policy observation ID, runner access observation ID, and Fleet capability declaration event ID. A new evidence observation can therefore appear as `evidence_changed` without implying the model or execution environment changed. + +A missing value on either side is listed in `unknownFields`. Known execution differences take precedence over unknown fields; if only evidence changes are known but required execution/evidence fields are missing, status remains `unknown` with the evidence differences still visible. + +The first returned Run is always `baseline`: there is no claim about Runs earlier than the selected window. History is bounded by the existing Archive limit (1–100), and the default temporal cross-model view uses the latest 20 Runs per Model. + +## Non-causal interpretation + +A model response changing between Runs **does not prove model degradation**. In particular: + +- region, service tier, account, or assurance changes may explain why historical points are not directly comparable; +- an evidence-source refresh is not an execution-environment change; +- a single calibration anomaly cannot be reported as degradation; +- missing qualification remains unknown; +- historical Run, qualification, pricing, Evidence, and Evaluation stay immutable. + +This phase is a presentation and derived-read extension. It adds no schema migration and does not change Planner, Queue, or Runner truth. + +## Verification + +- unit tests cover stable context, evidence-only changes, execution changes, missing context, and single-Run baseline; +- database integration checks a real pair of sealed Run qualification snapshots with a region change; +- public Archive API fixture covers history transitions; +- standard CI checks TypeScript, unit tests, migrations, persistence/runner integrations, and image builds. From a79c41586a2a5424da3e4afe79e6d81cef9d5e59 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:04:15 +0800 Subject: [PATCH 12/15] ADR 0021: distinguish public comparability, execution context and evidence --- docs/adr/0021-public-comparability-context.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/adr/0021-public-comparability-context.md diff --git a/docs/adr/0021-public-comparability-context.md b/docs/adr/0021-public-comparability-context.md new file mode 100644 index 0000000..c3c7e51 --- /dev/null +++ b/docs/adr/0021-public-comparability-context.md @@ -0,0 +1,28 @@ +# ADR 0021 — Keep public comparability distinct from context changes + +## Status + +Accepted for Archive v0.25. + +## Context + +Archive v0.23 provided policy-based eligibility, but public comparison and history surfaces could still confuse the separate questions: “were Runs executed in compatible contexts?” and “did the sourced observations or execution environment change over time?” + +Existing history badges displayed only whether a context key existed. That key included Provider-specific evidence IDs and therefore was inappropriate for direct cross-provider equality checks. + +## Decision + +1. Public comparisons select a named, append-only comparability policy version and show its eligibility reasons beside results. +2. No aggregate winner or quality score is added. +3. Cross-provider context equality compares execution characteristics, not provider-specific evidence IDs. +4. Longitudinal histories expose derived, field-level transitions between frozen Run qualification snapshots. +5. Execution field changes and evidence-reference changes are classified separately. +6. Unknown/missing fields remain visibly unknown; two missing values are never treated as proof of equality. +7. The first Run in a bounded history window is marked baseline. +8. Context transition caveats reuse frozen qualification and cost evidence. +9. Changes are descriptive records; they do not establish a causal explanation for output changes. +10. No historical Run, Evaluation, calibration or pricing evidence is mutated. + +## Consequences + +The public Archive can explain why a later result should not automatically be compared to an earlier result without equating provider-specific evidence refreshes with execution changes, or mistaking execution changes for model degradation. From 34c4c6c06e5c25d722b1250b2bf89bce125f8cfe Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:04:29 +0800 Subject: [PATCH 13/15] Mark v0.25 implemented and retain future public research roadmap --- docs/archive-roadmap.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/archive-roadmap.md b/docs/archive-roadmap.md index c911da0..9a9d4e4 100644 --- a/docs/archive-roadmap.md +++ b/docs/archive-roadmap.md @@ -293,13 +293,17 @@ Provider onboarding should require: - calibration coverage; - explicit unsupported capabilities. -### Archive v0.25+ — Public comparability and longitudinal views +### Archive v0.25 — Public comparability and longitudinal context -Only after the access/cost/qualification layers exist should public cross-provider views grow into broader benchmark-style surfaces. +**Implemented:** see `docs/public-comparability-longitudinal.md` and ADR 0021. -Public UI should show caveats next to results rather than hide them behind one aggregate score. +With the access/cost/qualification layers in place, the public cross-provider comparison now exposes policy-version selection, latest Run eligibility, replication, calibration and qualification/cost caveats **next to results**, without inventing an aggregate score. -Longitudinal views should also expose when execution context changed between Runs (region, account/service tier, assurance, evidence references), so an environment change is not misread as a model change. +Longitudinal views explicitly derive field-level changes from frozen Run qualification snapshots (Fleet, region, account/service tier and assurance), and separately label Provider policy/runner/capability evidence refreshes. Unknown context remains unknown; an environment change is not attributed to the Model. + +### Archive v0.26+ — Broader public research surfaces + +Grow benchmark-style exploration carefully from the existing per-Test comparability and evidence-first temporal views, without treating heterogeneous Test results as one ranking. Continue to make cost, availability, registration and calibration limitations explicit alongside the records. ## Scheduling principle From 93b0cc8bf15aca14aac07a63b5065d16d47f87b9 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:05:44 +0800 Subject: [PATCH 14/15] Treat uncertain service assurance as unknown in cross-provider context badges --- apps/web/src/routes/compare.tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/web/src/routes/compare.tsx b/apps/web/src/routes/compare.tsx index 3850941..1af65d4 100644 --- a/apps/web/src/routes/compare.tsx +++ b/apps/web/src/routes/compare.tsx @@ -55,7 +55,16 @@ function contextRelation( ]; const current = execution(run); const baseline = execution(reference); - if (current.some((value) => value === null) || baseline.some((value) => value === null)) { + const assurance = run.executionQualification?.serviceAssurance; + const baselineAssurance = reference.executionQualification?.serviceAssurance; + if ( + current.some((value) => value === null) || + baseline.some((value) => value === null) || + !["documented_default", "documented_variant"].includes(assurance ?? "") || + !["documented_default", "documented_variant"].includes( + baselineAssurance ?? "", + ) + ) { return "unknown"; } return current.every((value, index) => value === baseline[index]) From dc841389b46ddf7b2f44bd539bc485ddca402409 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:06:11 +0800 Subject: [PATCH 15/15] Use complete typed Run fixture for context transition unit tests --- .../test/archive-context-transitions.test.ts | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/persistence/test/archive-context-transitions.test.ts b/packages/persistence/test/archive-context-transitions.test.ts index f601d3a..42d425e 100644 --- a/packages/persistence/test/archive-context-transitions.test.ts +++ b/packages/persistence/test/archive-context-transitions.test.ts @@ -10,6 +10,29 @@ function run(id: string, options: { } = {}): ArchiveRunView { return { id, + status: "completed", + model: { + id: "model-1", + canonicalSlug: "model-test", + marketingName: "Model Test", + }, + provider: { + id: "provider-1", + slug: "fixture", + name: "Fixture Provider", + }, + test: { + testCaseId: "test-case-1", + familySlug: "context-fixture", + familyName: "Context Fixture", + variantSlug: "benchmark", + variantName: "Benchmark", + version: "1.0.0", + caseSlug: "same-test", + }, + requestedModel: "model-test", + returnedModel: "model-test", + evidenceLevel: "E4", executionPath: "first_party_direct", executionQualification: { selectedAt: "2026-10-08T00:00:00.000Z", @@ -47,7 +70,12 @@ function run(id: string, options: { estimatedNativeCost: null, caveats: ["native_cost_unavailable"], }, - } as ArchiveRunView; + evaluation: null, + runnerBuild: "context-fixture", + createdAt: "2026-10-08T00:00:00.000Z", + completedAt: "2026-10-08T00:00:01.000Z", + sealedAt: "2026-10-08T00:00:02.000Z", + }; } describe("public Archive longitudinal context transitions", () => {