diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index 6eb23b8..eefb261 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -214,6 +214,24 @@ const archiveRunHistory = { }, archiveRun, ], + contextTransitions: [ + { + runId: PREVIOUS_RUN_ID, + previousRunId: null, + status: "baseline", + changes: [], + unknownFields: [], + caveats: [], + }, + { + runId: RUN_ID, + previousRunId: PREVIOUS_RUN_ID, + status: "unchanged", + changes: [], + unknownFields: [], + caveats: [], + }, + ], relations: [ { fromRunId: RUN_ID, diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index d6fe48f..cc71086 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -550,10 +550,25 @@ interface ArchiveRunDetailWire extends ArchiveRun { readonly relations: readonly ArchiveRunRelation[]; } +export interface ArchiveContextTransition { + readonly runId: string; + readonly previousRunId: string | null; + readonly status: "baseline" | "unchanged" | "changed" | "evidence_changed" | "unknown"; + readonly changes: readonly { + readonly field: string; + readonly previous: string; + readonly current: string; + readonly kind: "execution" | "evidence"; + }[]; + readonly unknownFields: readonly string[]; + readonly caveats: readonly string[]; +} + export interface ArchiveRunHistory { readonly model: ArchiveModel; readonly test: ArchiveTest; readonly runs: readonly ArchiveRun[]; + readonly contextTransitions: readonly ArchiveContextTransition[]; readonly relations: readonly ArchiveRunRelationEdge[]; } @@ -562,6 +577,7 @@ export interface ArchiveTemporalComparison { readonly rows: readonly { readonly model: ArchiveModel; readonly runs: readonly ArchiveRun[]; + readonly contextTransitions: readonly ArchiveContextTransition[]; readonly relations: readonly ArchiveRunRelationEdge[]; }[]; } @@ -2242,6 +2258,15 @@ export const getArchiveCatalogChanges = createServerFn({ method: "GET" }).handle }, ); +export const getArchiveComparabilityPolicies = createServerFn({ method: "GET" }).handler( + async (): Promise => { + const result = await requestJson<{ policies: readonly ArchiveComparabilityPolicy[] }>( + "/v1/archive/comparability/policies", + ); + return result.policies; + }, +); + export const getArchiveCatalog = createServerFn({ method: "GET" }).handler( async (): Promise => { const [models, tests] = await Promise.all([ @@ -2347,6 +2372,7 @@ export const compareArchiveHistory = createServerFn({ method: "POST" }) rows: complete.map((history) => ({ model: history.model, runs: history.runs, + contextTransitions: history.contextTransitions, relations: history.relations, })), }; diff --git a/apps/web/src/routes/compare.tsx b/apps/web/src/routes/compare.tsx index 628ed36..1af65d4 100644 --- a/apps/web/src/routes/compare.tsx +++ b/apps/web/src/routes/compare.tsx @@ -4,13 +4,20 @@ import { compareArchive, compareArchiveHistory, getArchiveCatalog, + getArchiveComparabilityPolicies, type ArchiveComparison, type ArchiveRun, type ArchiveTemporalComparison, } from "../modelapse"; export const Route = createFileRoute("/compare")({ - loader: () => getArchiveCatalog(), + loader: async () => { + const [catalog, policies] = await Promise.all([ + getArchiveCatalog(), + getArchiveComparabilityPolicies(), + ]); + return { catalog, policies }; + }, component: ArchiveComparePage, }); @@ -37,15 +44,44 @@ function contextRelation( run: ArchiveRun, reference: ArchiveRun | null, ): "matched" | "mismatched" | "unknown" { - const context = run.executionQualification?.contextKey; - const referenceContext = reference?.executionQualification?.contextKey; - if (!context || !referenceContext) return "unknown"; - return context === referenceContext ? "matched" : "mismatched"; + if (!reference) return "unknown"; + const execution = (value: ArchiveRun): readonly (string | null)[] => [ + value.executionPath, + value.executionQualification?.executionRegion ?? null, + value.executionQualification?.accountTier ?? null, + value.executionQualification?.returnedServiceTier ?? + value.executionQualification?.serviceTier ?? null, + value.executionQualification?.serviceAssurance ?? null, + ]; + const current = execution(run); + const baseline = execution(reference); + const assurance = run.executionQualification?.serviceAssurance; + const baselineAssurance = reference.executionQualification?.serviceAssurance; + if ( + current.some((value) => value === null) || + baseline.some((value) => value === null) || + !["documented_default", "documented_variant"].includes(assurance ?? "") || + !["documented_default", "documented_variant"].includes( + baselineAssurance ?? "", + ) + ) { + return "unknown"; + } + return current.every((value, index) => value === baseline[index]) + ? "matched" + : "mismatched"; } function ArchiveComparePage() { - const catalog = Route.useLoaderData(); - const defaultModelIds = catalog.models.slice(0, 2).map((model) => model.id); + const { catalog, policies } = Route.useLoaderData(); + const firstModel = catalog.models[0]; + const secondModel = + catalog.models.find( + (model) => firstModel && model.provider.id !== firstModel.provider.id, + ) ?? catalog.models[1]; + const defaultModelIds = [firstModel?.id, secondModel?.id].filter( + (id): id is string => Boolean(id), + ); const comparableTests = catalog.tests.filter( (test) => test.category !== "calibration", ); @@ -53,6 +89,7 @@ function ArchiveComparePage() { const [testCaseId, setTestCaseId] = useState( comparableTests[0]?.testCaseId ?? "", ); + const [policyVersion, setPolicyVersion] = useState(policies[0]?.version ?? ""); const [comparison, setComparison] = useState(null); const [temporal, setTemporal] = useState(null); const [busy, setBusy] = useState(false); @@ -90,6 +127,7 @@ function ArchiveComparePage() { data: { modelIds, testCaseId, + ...(policyVersion ? { policyVersion } : {}), }, }), compareArchiveHistory({ @@ -171,6 +209,28 @@ function ArchiveComparePage() { Latest comparison and temporal lanes use this exact public Test Case. +
02 / Models @@ -253,7 +313,7 @@ function ArchiveComparePage() { {comparison.comparabilitySet.reasons.length > 0 ? comparison.comparabilitySet.reasons.join(" · ") - : "All selected latest Runs satisfy the current set policy."} + : "All selected latest Runs satisfy this policy. This is not a quality ranking."}
@@ -308,6 +368,25 @@ function ArchiveComparePage() {
Execution
{run.executionPath}
+
+
Region / account
+
+ {run.executionQualification?.executionRegion ?? "unknown"} + {" · "} + {run.executionQualification?.accountTier ?? "unknown"} +
+
+
+
Service / assurance
+
+ {run.executionQualification?.returnedServiceTier ?? + run.executionQualification?.serviceTier ?? + "unknown"} + {" · "} + {run.executionQualification?.serviceAssurance ?? + "unknown"} +
+
Evaluator
@@ -335,9 +414,21 @@ function ArchiveComparePage() { {row.comparability?.reasons.length ? (

- {row.comparability.reasons.join(" · ")} + Comparability: {row.comparability.reasons.join(" · ")}

) : null} +

+ Execution caveats:{" "} + {run.executionQualification + ? run.executionQualification.caveats.join(" · ") || "none reported" + : "qualification unavailable"} +

+

+ Cost caveats:{" "} + {run.cost + ? run.cost.caveats.join(" · ") || "none reported" + : "cost evidence unavailable"} +

) : (
@@ -408,10 +499,24 @@ function ArchiveComparePage() { {run.evidenceLevel ?? "—"} · {run.returnedModel ?? run.requestedModel} {" · "} - {run.executionQualification?.contextKey - ? "context captured" - : "context unknown"} + context {row.contextTransitions[index]?.status ?? "unknown"} + {row.contextTransitions[index]?.changes.map((change) => ( + + {change.kind} · {change.field.replaceAll("_", " ")}: + {" "}{change.previous} → {change.current} + + ))} + {row.contextTransitions[index]?.unknownFields.length ? ( + + Unknown: {row.contextTransitions[index]?.unknownFields.join(", ")} + + ) : null} + {row.contextTransitions[index]?.caveats.length ? ( + + Caveats: {row.contextTransitions[index]?.caveats.join(" · ")} + + ) : null} ))} {row.runs.length === 0 ? ( diff --git a/apps/web/src/routes/history.$modelId.$testCaseId.tsx b/apps/web/src/routes/history.$modelId.$testCaseId.tsx index 511863a..12ce718 100644 --- a/apps/web/src/routes/history.$modelId.$testCaseId.tsx +++ b/apps/web/src/routes/history.$modelId.$testCaseId.tsx @@ -131,13 +131,16 @@ function ArchiveRunHistoryPage() {

Same model × same Test

- Runs are ordered by capture time. Changes are shown as records; no causal - explanation or quality trend is inferred. + Every transition compares frozen execution snapshots and evidence references. + A changed region, account tier, service tier, or Fleet identity is not proof + that the model itself changed. Missing evidence remains unknown.

- {history.runs.map((run, index) => ( + {history.runs.map((run, index) => { + const transition = history.contextTransitions[index]; + return (
{String(index + 1).padStart(2, "0")}
@@ -148,20 +151,43 @@ function ArchiveRunHistoryPage() { {evaluationLabel(run)} evidence {run.evidenceLevel ?? "—"} - {run.executionQualification?.contextKey - ? "context captured" - : "context unknown"} + context {transition?.status ?? "unknown"}
{run.returnedModel ?? run.requestedModel} {run.executionPath} + + {run.executionQualification + ? [ + run.executionQualification.executionRegion ?? "region unknown", + run.executionQualification.accountTier ?? "account unknown", + run.executionQualification.returnedServiceTier ?? + run.executionQualification.serviceTier ?? + "service unknown", + run.executionQualification.serviceAssurance, + ].join(" · ") + : "execution qualification unavailable"} + + {transition?.changes.map((change) => ( + + {change.kind} · {change.field.replaceAll("_", " ")}:{" "} + {change.previous} → {change.current} + + ))} + {transition?.unknownFields.length ? ( + Unresolved: {transition.unknownFields.join(", ")} + ) : null} + {transition?.caveats.length ? ( + Caveats: {transition.caveats.join(" · ")} + ) : null}
Run {run.id.slice(0, 8)} →
- ))} + ); + })} {history.runs.length === 0 ? (
No sealed public Runs exist for this pair yet.
) : null} diff --git a/docs/adr/0021-public-comparability-context.md b/docs/adr/0021-public-comparability-context.md new file mode 100644 index 0000000..c3c7e51 --- /dev/null +++ b/docs/adr/0021-public-comparability-context.md @@ -0,0 +1,28 @@ +# ADR 0021 — Keep public comparability distinct from context changes + +## Status + +Accepted for Archive v0.25. + +## Context + +Archive v0.23 provided policy-based eligibility, but public comparison and history surfaces could still confuse the separate questions: “were Runs executed in compatible contexts?” and “did the sourced observations or execution environment change over time?” + +Existing history badges displayed only whether a context key existed. That key included Provider-specific evidence IDs and therefore was inappropriate for direct cross-provider equality checks. + +## Decision + +1. Public comparisons select a named, append-only comparability policy version and show its eligibility reasons beside results. +2. No aggregate winner or quality score is added. +3. Cross-provider context equality compares execution characteristics, not provider-specific evidence IDs. +4. Longitudinal histories expose derived, field-level transitions between frozen Run qualification snapshots. +5. Execution field changes and evidence-reference changes are classified separately. +6. Unknown/missing fields remain visibly unknown; two missing values are never treated as proof of equality. +7. The first Run in a bounded history window is marked baseline. +8. Context transition caveats reuse frozen qualification and cost evidence. +9. Changes are descriptive records; they do not establish a causal explanation for output changes. +10. No historical Run, Evaluation, calibration or pricing evidence is mutated. + +## Consequences + +The public Archive can explain why a later result should not automatically be compared to an earlier result without equating provider-specific evidence refreshes with execution changes, or mistaking execution changes for model degradation. diff --git a/docs/archive-roadmap.md b/docs/archive-roadmap.md index c911da0..9a9d4e4 100644 --- a/docs/archive-roadmap.md +++ b/docs/archive-roadmap.md @@ -293,13 +293,17 @@ Provider onboarding should require: - calibration coverage; - explicit unsupported capabilities. -### Archive v0.25+ — Public comparability and longitudinal views +### Archive v0.25 — Public comparability and longitudinal context -Only after the access/cost/qualification layers exist should public cross-provider views grow into broader benchmark-style surfaces. +**Implemented:** see `docs/public-comparability-longitudinal.md` and ADR 0021. -Public UI should show caveats next to results rather than hide them behind one aggregate score. +With the access/cost/qualification layers in place, the public cross-provider comparison now exposes policy-version selection, latest Run eligibility, replication, calibration and qualification/cost caveats **next to results**, without inventing an aggregate score. -Longitudinal views should also expose when execution context changed between Runs (region, account/service tier, assurance, evidence references), so an environment change is not misread as a model change. +Longitudinal views explicitly derive field-level changes from frozen Run qualification snapshots (Fleet, region, account/service tier and assurance), and separately label Provider policy/runner/capability evidence refreshes. Unknown context remains unknown; an environment change is not attributed to the Model. + +### Archive v0.26+ — Broader public research surfaces + +Grow benchmark-style exploration carefully from the existing per-Test comparability and evidence-first temporal views, without treating heterogeneous Test results as one ranking. Continue to make cost, availability, registration and calibration limitations explicit alongside the records. ## Scheduling principle diff --git a/docs/public-comparability-longitudinal.md b/docs/public-comparability-longitudinal.md new file mode 100644 index 0000000..0656089 --- /dev/null +++ b/docs/public-comparability-longitudinal.md @@ -0,0 +1,62 @@ +# Public Comparability & Longitudinal Context — Archive v0.25 + +Archive v0.25 builds a **public, evidence-first** comparison surface on top of Archive v0.19–v0.24. It does not introduce aggregate model scores, rankings, or unsupported quality claims. + +## Public comparison + +`/compare` selects 2–4 canonical Models and an **exact public non-calibration Test Case**. The initial selection prefers Models from two different Providers, when present. + +The public API `GET /v1/archive/compare?modelIds=…&testCaseId=…&policyVersion=…` already evaluates Runs under v0.23 versioned comparability policies. The UI now surfaces policy selection, so visitors can inspect the same immutable evidence under different **explicit policy versions**, without rewriting Run or Evaluation. + +Beside each latest result the public UI shows: + +- comparability outcome and machine-readable reasons; +- replication requirement and achieved count; +- same-context calibration status and anomaly streak; +- execution region and account tier; +- effective service tier and documented assurance; +- frozen execution qualification caveats; +- frozen cost-evidence caveats, including unavailable pricing or usage. + +The comparison result may be `matched`, `mismatched`, or `unknown`. These describe whether the Runs are eligible to be compared *under the chosen policy*, not which model performs better. + +A **cross-provider execution-context badge** checks execution path, region, account, effective service tier, and assurance. It intentionally excludes Provider-specific policy/runner observation IDs from equality checks. Those references are independent evidence and must not create false environment mismatches. + +## Public longitudinal history + +`GET /v1/archive/history?modelId=…&testCaseId=…&limit=…` now includes `contextTransitions`, aligned one-to-one with the returned `runs`, in capture-time order. + +Each derived transition contains: + +- `runId`, `previousRunId`; +- status: `baseline`, `unchanged`, `changed`, `evidence_changed`, or `unknown`; +- field-level changes with previous/current values and `execution` or `evidence` category; +- unresolved field names, instead of treating missing values as equal; +- the Run's frozen qualification/cost caveats. + +Execution changes cover execution path, Fleet environment ID, region, account tier, service tier, effective returned service tier, and service assurance. + +Evidence changes cover Provider policy observation ID, runner access observation ID, and Fleet capability declaration event ID. A new evidence observation can therefore appear as `evidence_changed` without implying the model or execution environment changed. + +A missing value on either side is listed in `unknownFields`. Known execution differences take precedence over unknown fields; if only evidence changes are known but required execution/evidence fields are missing, status remains `unknown` with the evidence differences still visible. + +The first returned Run is always `baseline`: there is no claim about Runs earlier than the selected window. History is bounded by the existing Archive limit (1–100), and the default temporal cross-model view uses the latest 20 Runs per Model. + +## Non-causal interpretation + +A model response changing between Runs **does not prove model degradation**. In particular: + +- region, service tier, account, or assurance changes may explain why historical points are not directly comparable; +- an evidence-source refresh is not an execution-environment change; +- a single calibration anomaly cannot be reported as degradation; +- missing qualification remains unknown; +- historical Run, qualification, pricing, Evidence, and Evaluation stay immutable. + +This phase is a presentation and derived-read extension. It adds no schema migration and does not change Planner, Queue, or Runner truth. + +## Verification + +- unit tests cover stable context, evidence-only changes, execution changes, missing context, and single-Run baseline; +- database integration checks a real pair of sealed Run qualification snapshots with a region change; +- public Archive API fixture covers history transitions; +- standard CI checks TypeScript, unit tests, migrations, persistence/runner integrations, and image builds. diff --git a/packages/persistence/package.json b/packages/persistence/package.json index 157e17f..8bcb3ef 100644 --- a/packages/persistence/package.json +++ b/packages/persistence/package.json @@ -10,7 +10,7 @@ "scripts": { "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "vitest run test/record-provider-run.test.ts", + "test": "vitest run test/record-provider-run.test.ts test/archive-context-transitions.test.ts", "test:integration": "vitest run test/integration.test.ts test/comparability.integration.test.ts" }, "dependencies": { diff --git a/packages/persistence/src/archive-context-transitions.ts b/packages/persistence/src/archive-context-transitions.ts new file mode 100644 index 0000000..b3bfb3b --- /dev/null +++ b/packages/persistence/src/archive-context-transitions.ts @@ -0,0 +1,96 @@ +import type { ArchiveRunView } from "./archive-repository.js"; + +export interface ArchiveContextDifference { + readonly field: string; + readonly previous: string; + readonly current: string; + readonly kind: "execution" | "evidence"; +} + +export interface ArchiveContextTransition { + readonly runId: string; + readonly previousRunId: string | null; + readonly status: "baseline" | "unchanged" | "changed" | "evidence_changed" | "unknown"; + readonly changes: readonly ArchiveContextDifference[]; + readonly unknownFields: readonly string[]; + readonly caveats: readonly string[]; +} + +/** + * Compare archived snapshots, never mutable current Provider/Runner state. + * Unknown values must not silently compare as equal or count as a change. + */ +export function archiveContextTransitions( + runs: readonly ArchiveRunView[], +): readonly ArchiveContextTransition[] { + const projection = (run: ArchiveRunView) => { + const q = run.executionQualification; + const effectiveTier = q?.returnedServiceTier ?? q?.serviceTier ?? null; + return [ + ["execution_path", run.executionPath, "execution"], + ["environment", q?.executionEnvironment?.id ?? null, "execution"], + ["region", q?.executionRegion ?? null, "execution"], + ["account_tier", q?.accountTier ?? null, "execution"], + ["service_tier", q?.serviceTier ?? null, "execution"], + ["effective_service_tier", effectiveTier, "execution"], + ["service_assurance", q?.serviceAssurance ?? null, "execution"], + ["provider_policy_observation", q?.providerPolicyObservation?.id ?? null, "evidence"], + ["runner_access_observation", q?.runnerAccessObservation?.id ?? null, "evidence"], + ["fleet_capability_event", q?.executionEnvironment?.capabilityEventId ?? null, "evidence"], + ] as const; + }; + + return runs.map((run, index) => { + const prior = runs[index - 1]; + const caveats = [ + ...(run.executionQualification?.caveats ?? ["execution_qualification_missing"]), + ...(run.cost?.caveats ?? []), + ]; + if (!run.executionQualification?.executionEnvironment) { + caveats.push("execution_environment_unassigned"); + } + + if (!prior) { + return { + runId: run.id, + previousRunId: null, + status: "baseline" as const, + changes: [], + unknownFields: [], + caveats: [...new Set(caveats)], + }; + } + + const before = projection(prior); + const after = projection(run); + const changes: ArchiveContextDifference[] = []; + const unknownFields: string[] = []; + for (let i = 0; i < before.length; i += 1) { + const [field, previous, kind] = before[i]!; + const current = after[i]![1]; + if (previous === null || current === null) { + unknownFields.push(field); + } else if (previous !== current) { + changes.push({ field, previous, current, kind }); + } + } + + const executionChanged = changes.some((change) => change.kind === "execution"); + const status: ArchiveContextTransition["status"] = executionChanged + ? "changed" + : unknownFields.length > 0 + ? "unknown" + : changes.length > 0 + ? "evidence_changed" + : "unchanged"; + + return { + runId: run.id, + previousRunId: prior.id, + status, + changes, + unknownFields, + caveats: [...new Set(caveats)], + }; + }); +} diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index 0655579..cddce59 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { Pool } from "pg"; import type { ComparabilityPolicyView } from "./comparability-repository.js"; +import { archiveContextTransitions, type ArchiveContextTransition } from "./archive-context-transitions.js"; export interface ArchiveModelView { readonly id: string; @@ -457,6 +458,7 @@ export interface ArchiveRunHistoryView { readonly model: ArchiveModelView; readonly test: ArchiveTestView; readonly runs: readonly ArchiveRunView[]; + readonly contextTransitions: readonly ArchiveContextTransition[]; readonly relations: readonly ArchiveRunRelationEdgeView[]; } @@ -1382,6 +1384,7 @@ export class PgArchiveRepository { model, test, runs, + contextTransitions: archiveContextTransitions(runs), relations, }; } diff --git a/packages/persistence/test/archive-context-transitions.test.ts b/packages/persistence/test/archive-context-transitions.test.ts new file mode 100644 index 0000000..42d425e --- /dev/null +++ b/packages/persistence/test/archive-context-transitions.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; +import { archiveContextTransitions } from "../src/archive-context-transitions.js"; +import type { ArchiveRunView } from "../src/archive-repository.js"; + +function run(id: string, options: { + region?: string | null; + account?: string | null; + policyId?: string | null; + caveats?: string[]; +} = {}): ArchiveRunView { + return { + id, + status: "completed", + model: { + id: "model-1", + canonicalSlug: "model-test", + marketingName: "Model Test", + }, + provider: { + id: "provider-1", + slug: "fixture", + name: "Fixture Provider", + }, + test: { + testCaseId: "test-case-1", + familySlug: "context-fixture", + familyName: "Context Fixture", + variantSlug: "benchmark", + variantName: "Benchmark", + version: "1.0.0", + caseSlug: "same-test", + }, + requestedModel: "model-test", + returnedModel: "model-test", + evidenceLevel: "E4", + executionPath: "first_party_direct", + executionQualification: { + selectedAt: "2026-10-08T00:00:00.000Z", + executionEnvironment: { + id: "fleet-primary", + slug: "us-paid", + capabilityEventId: "capability-v1", + }, + executionRegion: options.region === undefined ? "US" : options.region, + accountTier: options.account === undefined ? "paid" : options.account, + serviceTier: "default", + requestedServiceTier: "default", + returnedServiceTier: "default", + serviceAssurance: "documented_default", + providerPolicyObservation: options.policyId === null + ? null + : { + id: options.policyId ?? "policy-v1", + sourceId: "source", + accessState: "available", + }, + runnerAccessObservation: { + id: "runner-v1", + sourceId: "source", + accessState: "available", + }, + caveats: options.caveats ?? [], + contextKey: "frozen", + }, + cost: { + selectedAt: "2026-10-08T00:00:00.000Z", + pricingObservation: null, + pricing: null, + usage: null, + estimatedNativeCost: null, + caveats: ["native_cost_unavailable"], + }, + evaluation: null, + runnerBuild: "context-fixture", + createdAt: "2026-10-08T00:00:00.000Z", + completedAt: "2026-10-08T00:00:01.000Z", + sealedAt: "2026-10-08T00:00:02.000Z", + }; +} + +describe("public Archive longitudinal context transitions", () => { + it("differentiates identical snapshots from evidence refreshes and execution changes", () => { + const transitions = archiveContextTransitions([ + run("a"), + run("b"), + run("c", { policyId: "policy-v2" }), + run("d", { policyId: "policy-v2", region: "JP" }), + ]); + expect(transitions.map((entry) => entry.status)).toEqual([ + "baseline", + "unchanged", + "evidence_changed", + "changed", + ]); + expect(transitions[2]?.changes).toEqual([ + { + field: "provider_policy_observation", + previous: "policy-v1", + current: "policy-v2", + kind: "evidence", + }, + ]); + expect(transitions[3]?.changes).toEqual([ + { + field: "region", + previous: "US", + current: "JP", + kind: "execution", + }, + ]); + expect(transitions[3]?.previousRunId).toBe("c"); + }); + + it("preserves unknown provenance and displays historical caveats", () => { + const [baseline, unknown] = archiveContextTransitions([ + run("a"), + run("b", { + region: null, + account: null, + policyId: null, + caveats: ["execution_region_unknown"], + }), + ]); + expect(baseline?.status).toBe("baseline"); + expect(unknown?.status).toBe("unknown"); + expect(unknown?.changes).toEqual([]); + expect(unknown?.unknownFields).toEqual( + expect.arrayContaining(["region", "account_tier", "provider_policy_observation"]), + ); + expect(unknown?.caveats).toEqual( + expect.arrayContaining(["execution_region_unknown", "native_cost_unavailable"]), + ); + }); + + it("does not invent transitions or infer model degradation from a single Run", () => { + const transitions = archiveContextTransitions([run("only")]); + expect(transitions).toMatchObject([{ + runId: "only", + previousRunId: null, + status: "baseline", + changes: [], + }]); + expect(archiveContextTransitions([])).toEqual([]); + }); +}); diff --git a/packages/persistence/test/comparability.integration.test.ts b/packages/persistence/test/comparability.integration.test.ts index ef02ecc..e4bf6a6 100644 --- a/packages/persistence/test/comparability.integration.test.ts +++ b/packages/persistence/test/comparability.integration.test.ts @@ -470,6 +470,25 @@ describe("Archive comparability policy", () => { expect(regionMismatch?.comparabilitySet.status).toBe("mismatched"); expect(regionMismatch?.comparabilitySet.reasons).toContain("region_mismatch"); + const modelHistory = await archive.getRunHistory({ + modelId: models[1]!, + testCaseId: benchmarkCase.rows[0]!.id, + }); + expect(modelHistory?.runs).toHaveLength(2); + expect(modelHistory?.contextTransitions).toHaveLength(2); + expect(modelHistory?.contextTransitions[0]?.status).toBe("baseline"); + expect(modelHistory?.contextTransitions[1]).toMatchObject({ + status: "changed", + changes: expect.arrayContaining([ + { + field: "region", + previous: "US", + current: "JP", + kind: "execution", + }, + ]), + }); + const custom = await policies.recordPolicy({ version: "comparability-integration-" + suffix, minimumEvidenceLevel: "E4",