Archive v0.25 — Public Comparability & Longitudinal Context

apps/api/test/archive-api.test.ts

@@ -214,6 +214,24 @@ const archiveRunHistory = {
     },
     archiveRun,
   ],
+  contextTransitions: [
+    {
+      runId: PREVIOUS_RUN_ID,
+      previousRunId: null,
+      status: "baseline",
+      changes: [],
+      unknownFields: [],
+      caveats: [],
+    },
+    {
+      runId: RUN_ID,
+      previousRunId: PREVIOUS_RUN_ID,
+      status: "unchanged",
+      changes: [],
+      unknownFields: [],
+      caveats: [],
+    },
+  ],
   relations: [
     {
       fromRunId: RUN_ID,

apps/web/src/modelapse.ts

@@ -550,10 +550,25 @@ interface ArchiveRunDetailWire extends ArchiveRun {
   readonly relations: readonly ArchiveRunRelation[];
 }
 
+export interface ArchiveContextTransition {
+  readonly runId: string;
+  readonly previousRunId: string | null;
+  readonly status: "baseline" | "unchanged" | "changed" | "evidence_changed" | "unknown";
+  readonly changes: readonly {
+    readonly field: string;
+    readonly previous: string;
+    readonly current: string;
+    readonly kind: "execution" | "evidence";
+  }[];
+  readonly unknownFields: readonly string[];
+  readonly caveats: readonly string[];
+}
+
 export interface ArchiveRunHistory {
   readonly model: ArchiveModel;
   readonly test: ArchiveTest;
   readonly runs: readonly ArchiveRun[];
+  readonly contextTransitions: readonly ArchiveContextTransition[];
   readonly relations: readonly ArchiveRunRelationEdge[];
 }
 
@@ -562,6 +577,7 @@ export interface ArchiveTemporalComparison {
   readonly rows: readonly {
     readonly model: ArchiveModel;
     readonly runs: readonly ArchiveRun[];
+    readonly contextTransitions: readonly ArchiveContextTransition[];
     readonly relations: readonly ArchiveRunRelationEdge[];
   }[];
 }
@@ -2242,6 +2258,15 @@ export const getArchiveCatalogChanges = createServerFn({ method: "GET" }).handle
   },
 );
 
+export const getArchiveComparabilityPolicies = createServerFn({ method: "GET" }).handler(
+  async (): Promise<readonly ArchiveComparabilityPolicy[]> => {
+    const result = await requestJson<{ policies: readonly ArchiveComparabilityPolicy[] }>(
+      "/v1/archive/comparability/policies",
+    );
+    return result.policies;
+  },
+);
+
 export const getArchiveCatalog = createServerFn({ method: "GET" }).handler(
   async (): Promise<ArchiveCatalog> => {
     const [models, tests] = await Promise.all([
@@ -2347,6 +2372,7 @@ export const compareArchiveHistory = createServerFn({ method: "POST" })
       rows: complete.map((history) => ({
         model: history.model,
         runs: history.runs,
+        contextTransitions: history.contextTransitions,
         relations: history.relations,
       })),
     };

apps/web/src/routes/compare.tsx

@@ -4,13 +4,20 @@ import {
   compareArchive,
   compareArchiveHistory,
   getArchiveCatalog,
+  getArchiveComparabilityPolicies,
   type ArchiveComparison,
   type ArchiveRun,
   type ArchiveTemporalComparison,
 } from "../modelapse";
 
 export const Route = createFileRoute("/compare")({
-  loader: () => getArchiveCatalog(),
+  loader: async () => {
+    const [catalog, policies] = await Promise.all([
+      getArchiveCatalog(),
+      getArchiveComparabilityPolicies(),
+    ]);
+    return { catalog, policies };
+  },
   component: ArchiveComparePage,
 });
 
@@ -37,22 +44,52 @@ function contextRelation(
   run: ArchiveRun,
   reference: ArchiveRun | null,
 ): "matched" | "mismatched" | "unknown" {
-  const context = run.executionQualification?.contextKey;
-  const referenceContext = reference?.executionQualification?.contextKey;
-  if (!context || !referenceContext) return "unknown";
-  return context === referenceContext ? "matched" : "mismatched";
+  if (!reference) return "unknown";
+  const execution = (value: ArchiveRun): readonly (string | null)[] => [
+    value.executionPath,
+    value.executionQualification?.executionRegion ?? null,
+    value.executionQualification?.accountTier ?? null,
+    value.executionQualification?.returnedServiceTier ??
+      value.executionQualification?.serviceTier ?? null,
+    value.executionQualification?.serviceAssurance ?? null,
+  ];
+  const current = execution(run);
+  const baseline = execution(reference);
+  const assurance = run.executionQualification?.serviceAssurance;
+  const baselineAssurance = reference.executionQualification?.serviceAssurance;
+  if (
+    current.some((value) => value === null) ||
+    baseline.some((value) => value === null) ||
+    !["documented_default", "documented_variant"].includes(assurance ?? "") ||
+    !["documented_default", "documented_variant"].includes(
+      baselineAssurance ?? "",
+    )
+  ) {
+    return "unknown";
+  }
+  return current.every((value, index) => value === baseline[index])
+    ? "matched"
+    : "mismatched";
 }
 
 function ArchiveComparePage() {
-  const catalog = Route.useLoaderData();
-  const defaultModelIds = catalog.models.slice(0, 2).map((model) => model.id);
+  const { catalog, policies } = Route.useLoaderData();
+  const firstModel = catalog.models[0];
+  const secondModel =
+    catalog.models.find(
+      (model) => firstModel && model.provider.id !== firstModel.provider.id,
+    ) ?? catalog.models[1];
+  const defaultModelIds = [firstModel?.id, secondModel?.id].filter(
+    (id): id is string => Boolean(id),
+  );
   const comparableTests = catalog.tests.filter(
     (test) => test.category !== "calibration",
   );
   const [modelIds, setModelIds] = useState<readonly string[]>(defaultModelIds);
   const [testCaseId, setTestCaseId] = useState(
     comparableTests[0]?.testCaseId ?? "",
   );
+  const [policyVersion, setPolicyVersion] = useState(policies[0]?.version ?? "");
   const [comparison, setComparison] = useState<ArchiveComparison | null>(null);
   const [temporal, setTemporal] = useState<ArchiveTemporalComparison | null>(null);
   const [busy, setBusy] = useState(false);
@@ -90,6 +127,7 @@ function ArchiveComparePage() {
           data: {
             modelIds,
             testCaseId,
+            ...(policyVersion ? { policyVersion } : {}),
           },
         }),
         compareArchiveHistory({
@@ -171,6 +209,28 @@ function ArchiveComparePage() {
               Latest comparison and temporal lanes use this exact public Test Case.
             </small>
           </label>
+          <label>
+            <span>Comparability policy</span>
+            <select
+              value={policyVersion}
+              onChange={(event) => {
+                setPolicyVersion(event.target.value);
+                setComparison(null);
+                setTemporal(null);
+                setError(null);
+              }}
+            >
+              {policies.map((policy) => (
+                <option key={policy.id} value={policy.version}>
+                  {policy.version} · E{policy.minimumEvidenceLevel.slice(1)}+
+                </option>
+              ))}
+            </select>
+            <small>
+              Versioned eligibility rules, not a model score. Earlier policy versions
+              remain selectable for re-evaluating historical Runs.
+            </small>
+          </label>
 
           <div className="model-picker">
             <span>02 / Models</span>
@@ -253,7 +313,7 @@ function ArchiveComparePage() {
               <span>
                 {comparison.comparabilitySet.reasons.length > 0
                   ? comparison.comparabilitySet.reasons.join(" · ")
-                  : "All selected latest Runs satisfy the current set policy."}
+                  : "All selected latest Runs satisfy this policy. This is not a quality ranking."}
               </span>
             </div>
             <div className="comparison-grid">
@@ -308,6 +368,25 @@ function ArchiveComparePage() {
                           <dt>Execution</dt>
                           <dd>{run.executionPath}</dd>
                         </div>
+                        <div>
+                          <dt>Region / account</dt>
+                          <dd>
+                            {run.executionQualification?.executionRegion ?? "unknown"}
+                            {" · "}
+                            {run.executionQualification?.accountTier ?? "unknown"}
+                          </dd>
+                        </div>
+                        <div>
+                          <dt>Service / assurance</dt>
+                          <dd>
+                            {run.executionQualification?.returnedServiceTier ??
+                              run.executionQualification?.serviceTier ??
+                              "unknown"}
+                            {" · "}
+                            {run.executionQualification?.serviceAssurance ??
+                              "unknown"}
+                          </dd>
+                        </div>
                         <div>
                           <dt>Evaluator</dt>
                           <dd>
@@ -335,9 +414,21 @@ function ArchiveComparePage() {
                       </dl>
                       {row.comparability?.reasons.length ? (
                         <p className="section-note">
-                          {row.comparability.reasons.join(" · ")}
+                          Comparability: {row.comparability.reasons.join(" · ")}
                         </p>
                       ) : null}
+                      <p className="section-note">
+                        Execution caveats:{" "}
+                        {run.executionQualification
+                          ? run.executionQualification.caveats.join(" · ") || "none reported"
+                          : "qualification unavailable"}
+                      </p>
+                      <p className="section-note">
+                        Cost caveats:{" "}
+                        {run.cost
+                          ? run.cost.caveats.join(" · ") || "none reported"
+                          : "cost evidence unavailable"}
+                      </p>
                     </>
                   ) : (
                     <div className="comparison-empty">
@@ -408,10 +499,24 @@ function ArchiveComparePage() {
                       <small>
                         {run.evidenceLevel ?? "—"} · {run.returnedModel ?? run.requestedModel}
                         {" · "}
-                        {run.executionQualification?.contextKey
-                          ? "context captured"
-                          : "context unknown"}
+                        context {row.contextTransitions[index]?.status ?? "unknown"}
                       </small>
+                      {row.contextTransitions[index]?.changes.map((change) => (
+                        <small key={change.field}>
+                          {change.kind} · {change.field.replaceAll("_", " ")}:
+                          {" "}{change.previous} → {change.current}
+                        </small>
+                      ))}
+                      {row.contextTransitions[index]?.unknownFields.length ? (
+                        <small>
+                          Unknown: {row.contextTransitions[index]?.unknownFields.join(", ")}
+                        </small>
+                      ) : null}
+                      {row.contextTransitions[index]?.caveats.length ? (
+                        <small>
+                          Caveats: {row.contextTransitions[index]?.caveats.join(" · ")}
+                        </small>
+                      ) : null}
                     </a>
                   ))}
                   {row.runs.length === 0 ? (

apps/web/src/routes/history.$modelId.$testCaseId.tsx

@@ -131,13 +131,16 @@ function ArchiveRunHistoryPage() {
             <h2>Same model × same Test</h2>
           </div>
           <p className="section-note">
-            Runs are ordered by capture time. Changes are shown as records; no causal
-            explanation or quality trend is inferred.
+            Every transition compares frozen execution snapshots and evidence references.
+            A changed region, account tier, service tier, or Fleet identity is not proof
+            that the model itself changed. Missing evidence remains unknown.
           </p>
         </div>
 
         <div className="history-rail">
-          {history.runs.map((run, index) => (
+          {history.runs.map((run, index) => {
+            const transition = history.contextTransitions[index];
+            return (
             <article className="history-run" key={run.id}>
               <div className="history-index">{String(index + 1).padStart(2, "0")}</div>
               <div className="history-run-time">
@@ -148,20 +151,43 @@ function ArchiveRunHistoryPage() {
                 <span className={evaluationClass(run)}>{evaluationLabel(run)}</span>
                 <span className="badge">evidence {run.evidenceLevel ?? "—"}</span>
                 <span className="badge">
-                  {run.executionQualification?.contextKey
-                    ? "context captured"
-                    : "context unknown"}
+                  context {transition?.status ?? "unknown"}
                 </span>
               </div>
               <div className="history-run-model">
                 <strong>{run.returnedModel ?? run.requestedModel}</strong>
                 <small>{run.executionPath}</small>
+                <small>
+                  {run.executionQualification
+                    ? [
+                        run.executionQualification.executionRegion ?? "region unknown",
+                        run.executionQualification.accountTier ?? "account unknown",
+                        run.executionQualification.returnedServiceTier ??
+                          run.executionQualification.serviceTier ??
+                          "service unknown",
+                        run.executionQualification.serviceAssurance,
+                      ].join(" · ")
+                    : "execution qualification unavailable"}
+                </small>
+                {transition?.changes.map((change) => (
+                  <small key={change.field}>
+                    {change.kind} · {change.field.replaceAll("_", " ")}:{" "}
+                    {change.previous} → {change.current}
+                  </small>
+                ))}
+                {transition?.unknownFields.length ? (
+                  <small>Unresolved: {transition.unknownFields.join(", ")}</small>
+                ) : null}
+                {transition?.caveats.length ? (
+                  <small>Caveats: {transition.caveats.join(" · ")}</small>
+                ) : null}
               </div>
               <a className="text-link" href={`/runs/${run.id}`}>
                 Run {run.id.slice(0, 8)} →
               </a>
             </article>
-          ))}
+            );
+          })}
           {history.runs.length === 0 ? (
             <div className="empty-state">No sealed public Runs exist for this pair yet.</div>
           ) : null}

docs/adr/0021-public-comparability-context.md

@@ -0,0 +1,28 @@
+# ADR 0021 — Keep public comparability distinct from context changes
+
+## Status
+
+Accepted for Archive v0.25.
+
+## Context
+
+Archive v0.23 provided policy-based eligibility, but public comparison and history surfaces could still confuse the separate questions: “were Runs executed in compatible contexts?” and “did the sourced observations or execution environment change over time?”
+
+Existing history badges displayed only whether a context key existed. That key included Provider-specific evidence IDs and therefore was inappropriate for direct cross-provider equality checks.
+
+## Decision
+
+1. Public comparisons select a named, append-only comparability policy version and show its eligibility reasons beside results.
+2. No aggregate winner or quality score is added.
+3. Cross-provider context equality compares execution characteristics, not provider-specific evidence IDs.
+4. Longitudinal histories expose derived, field-level transitions between frozen Run qualification snapshots.
+5. Execution field changes and evidence-reference changes are classified separately.
+6. Unknown/missing fields remain visibly unknown; two missing values are never treated as proof of equality.
+7. The first Run in a bounded history window is marked baseline.
+8. Context transition caveats reuse frozen qualification and cost evidence.
+9. Changes are descriptive records; they do not establish a causal explanation for output changes.
+10. No historical Run, Evaluation, calibration or pricing evidence is mutated.
+
+## Consequences
+
+The public Archive can explain why a later result should not automatically be compared to an earlier result without equating provider-specific evidence refreshes with execution changes, or mistaking execution changes for model degradation.