From 92573ba37929f214b3526432fdb9357709bd94c9 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:58:00 +0800 Subject: [PATCH 01/13] Type Archive v0.28 observed facet counts with explicit all-pages scope --- packages/persistence/src/archive-research.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/persistence/src/archive-research.ts b/packages/persistence/src/archive-research.ts index 7de9108..d8072e1 100644 --- a/packages/persistence/src/archive-research.ts +++ b/packages/persistence/src/archive-research.ts @@ -60,6 +60,26 @@ export function encodeArchiveResearchCursor( ).toString("base64url"); } +export type ArchiveResearchFacetKey = + | "provider" | "model" | "test" | "evidence" | "region" + | "accountTier" | "serviceTier" | "cost" | "serviceAssurance"; + +export interface ArchiveResearchFacetValue { + readonly value: string | null; + readonly count: number; +} + +export interface ArchiveResearchFacetsView { + readonly scope: "sealed_public_non_calibration"; + readonly counting: "matching_runs_all_pages"; + readonly totalMatches: number; + readonly perFacetLimit: 25; + readonly facets: Record; +} + export type ArchiveResearchEvidence = "any" | "E4+" | "missing"; export type ArchiveResearchCost = "any" | "estimated" | "unknown"; From 7bac3c9f765c89bfd5c4460721d7ae61b5b51e04 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:58:18 +0800 Subject: [PATCH 02/13] Aggregate real PostgreSQL facet counts across all matching sealed public Runs --- .../persistence/src/archive-repository.ts | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index 1d47a8d..0eef61e 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -7,6 +7,8 @@ import { encodeArchiveResearchCursor, validateArchiveResearchFilters, type ArchiveResearchFilters, + type ArchiveResearchFacetsView, + type ArchiveResearchFacetKey, } from "./archive-research.js"; export interface ArchiveModelView { @@ -1239,6 +1241,148 @@ export class PgArchiveRepository { return result.rows.map(runView); } + async researchFacets( + input: ArchiveResearchFilters = {}, + ): Promise { + const filters = validateArchiveResearchFilters(input); + // Facets cover the entire filtered archive, never merely the current page. + // Cursor/limit are not applicable to a facet count. + if (filters.cursor !== undefined) { + throw new Error("invalid_research_facet_cursor"); + } + const result = await this.pool.query<{ + total_matches: string; + facet: ArchiveResearchFacetKey | null; + value: string | null; + match_count: string | null; + position: string | null; + }>( + `WITH scope AS MATERIALIZED ( + SELECT + p.slug AS provider, + r.model_id::text AS model, + r.test_case_id::text AS test, + CASE + WHEN res.level IN ('E4', 'E5') THEN 'E4+' + WHEN res.level IS NULL THEN 'missing' + ELSE 'below_E4' + END AS evidence, + qualification.execution_region AS region, + qualification.account_tier AS account_tier, + COALESCE( + qualification.returned_service_tier, + qualification.service_tier + ) AS service_tier, + qualification.service_assurance AS service_assurance, + CASE + WHEN cost.estimated_native_cost IS NULL THEN 'unknown' + ELSE 'estimated' + END AS cost + FROM modelapse.runs r + JOIN modelapse.test_cases tc ON tc.id = r.test_case_id + JOIN modelapse.models m ON m.id = r.model_id + JOIN modelapse.providers p ON p.id = r.provider_id + LEFT JOIN modelapse.run_evidence_summary res ON res.run_id = r.id + LEFT JOIN modelapse.run_execution_qualification qualification + ON qualification.run_id = r.id + LEFT JOIN modelapse.run_cost_ledger cost ON cost.run_id = r.id + WHERE tc.visibility = 'public' + AND tc.case_type <> 'calibration' + AND r.status = 'completed' + AND r.sealed_at IS NOT NULL + AND r.completed_at IS NOT NULL + AND ($1::text IS NULL OR p.slug = $1) + AND ($2::uuid IS NULL OR r.model_id = $2) + AND ($3::uuid IS NULL OR r.test_case_id = $3) + AND ( + $4::text = 'any' OR + ($4 = 'E4+' AND res.level IN ('E4', 'E5')) OR + ($4 = 'missing' AND res.level IS NULL) + ) + AND ($5::text IS NULL OR qualification.execution_region = $5) + AND ($6::text IS NULL OR qualification.account_tier = $6) + AND ( + $7::text IS NULL OR + COALESCE( + qualification.returned_service_tier, + qualification.service_tier + ) = $7 + ) + AND ( + $8::text = 'any' OR + ($8 = 'estimated' AND cost.estimated_native_cost IS NOT NULL) OR + ($8 = 'unknown' AND cost.estimated_native_cost IS NULL) + ) + ), + counts AS ( + SELECT facet.key, facet.value, COUNT(*)::text AS match_count + FROM scope + CROSS JOIN LATERAL ( + VALUES + ('provider', scope.provider), + ('model', scope.model), + ('test', scope.test), + ('evidence', scope.evidence), + ('region', scope.region), + ('accountTier', scope.account_tier), + ('serviceTier', scope.service_tier), + ('serviceAssurance', scope.service_assurance), + ('cost', scope.cost) + ) AS facet(key, value) + GROUP BY facet.key, facet.value + ), + ranked AS ( + SELECT + key AS facet, + value, + match_count, + ROW_NUMBER() OVER ( + PARTITION BY key + ORDER BY match_count::bigint DESC, value ASC NULLS LAST + )::text AS position + FROM counts + ) + SELECT + (SELECT COUNT(*)::text FROM scope) AS total_matches, + ranked.facet, ranked.value, ranked.match_count, ranked.position + FROM (SELECT 1) AS singleton + LEFT JOIN ranked ON ranked.position::integer <= 26 + ORDER BY ranked.facet ASC, ranked.position::integer ASC`, + [ + filters.providerSlug ?? null, + filters.modelId ?? null, + filters.testCaseId ?? null, + filters.evidence, + filters.region ?? null, + filters.accountTier ?? null, + filters.serviceTier ?? null, + filters.cost, + ], + ); + const keys: readonly ArchiveResearchFacetKey[] = [ + "provider", "model", "test", "evidence", "region", + "accountTier", "serviceTier", "serviceAssurance", "cost", + ]; + const facets = {} as ArchiveResearchFacetsView["facets"]; + for (const key of keys) { + const matches = result.rows.filter((row) => row.facet === key); + facets[key] = { + values: matches.slice(0, 25).map((row) => ({ + value: row.value, + count: Number(row.match_count ?? "0"), + })), + truncated: matches.length > 25, + }; + } + return { + scope: "sealed_public_non_calibration", + counting: "matching_runs_all_pages", + totalMatches: Number(result.rows[0]?.total_matches ?? "0"), + perFacetLimit: 25, + facets, + }; + } + async researchRuns( input: ArchiveResearchFilters = {}, ): Promise { From f7adeea81e7d83ec7d0422bf89ea813a62671fee Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:58:42 +0800 Subject: [PATCH 03/13] Reuse versioned per-Run comparability engine for immutable research collection annotations --- .../persistence/src/archive-repository.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index 0eef61e..d40a8a0 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -462,6 +462,16 @@ export interface ArchiveComparisonView { }[]; } +export interface ArchiveResearchPolicyAssessmentView { + readonly policy: ComparabilityPolicyView; + readonly scope: "per_run_only_not_cross_provider_match"; + readonly collectionRunCount: number; + readonly rows: readonly { + readonly runId: string; + readonly comparability: ArchiveRunComparabilityView; + }[]; +} + export interface ArchiveResearchPageView { readonly runs: readonly ArchiveRunView[]; readonly nextCursor: string | null; @@ -2905,6 +2915,65 @@ export class PgArchiveRepository { }; } + async assessResearchRuns( + runIds: readonly string[], + policyVersion?: string, + ): Promise { + if (runIds.length < 1 || runIds.length > 50 || + new Set(runIds).size !== runIds.length) { + throw new Error("invalid_research_run_count"); + } + if (policyVersion !== undefined && + !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion)) { + throw new Error("invalid_policy_version"); + } + + const [policy, runs] = await Promise.all([ + this.comparabilityPolicy(policyVersion), + this.getRunsByIds(runIds), + ]); + if (!policy) return null; + if (runs.length !== runIds.length) { + throw new Error("research_collection_snapshot_incomplete"); + } + + const tests = [...new Set(runs.map((run) => run.test.testCaseId))]; + const metadata = await this.pool.query<{ + id: string; + unstable: boolean; + explicit_min_repeats: string | null; + }>( + `SELECT id, + COALESCE((metadata->>'unstable')::boolean, false) AS unstable, + metadata->>'comparabilityMinRepeats' AS explicit_min_repeats + FROM modelapse.test_cases + WHERE id = ANY($1::uuid[])`, + [tests], + ); + const byTest = new Map(metadata.rows.map((row) => [row.id, row])); + + const rows = await Promise.all(runs.map(async (run) => { + const meta = byTest.get(run.test.testCaseId); + const raw = meta?.explicit_min_repeats; + const num = raw && /^\d+$/.test(raw) ? Number(raw) : null; + const explicitMinRepeats = num !== null && num >= 1 && num <= 20 + ? num : null; + return { + runId: run.id, + comparability: await this.assessRunComparability(run, policy, { + unstable: meta?.unstable ?? false, + explicitMinRepeats, + }), + }; + })); + return { + policy, + scope: "per_run_only_not_cross_provider_match", + collectionRunCount: runs.length, + rows, + }; + } + async compareLatest(input: { readonly modelIds: readonly string[]; readonly testCaseId: string; From ffbc695f2fb57d4e898cb5560d0b4300493926b6 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:59:01 +0800 Subject: [PATCH 04/13] Expose public all-pages Research facets and policy-anchored immutable collection assessment --- apps/api/src/app.ts | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index b5c60df..f59687a 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -120,6 +120,8 @@ type ArchiveRepository = Pick< | "listTests" | "listRuns" | "researchRuns" + | "researchFacets" + | "assessResearchRuns" | "listCatalogChanges" | "getRun" | "getModel" @@ -417,6 +419,64 @@ export function createApp(deps: AppDependencies) { } }); + app.get("/v1/archive/research/facets", async (c) => { + if (!deps.archive) return c.json({ error: "archive_unavailable" }, 503); + const query = c.req.query(); + const filters: ArchiveResearchFilters = { + ...(query.provider !== undefined ? { providerSlug: query.provider } : {}), + ...(query.modelId !== undefined ? { modelId: query.modelId } : {}), + ...(query.testCaseId !== undefined ? { testCaseId: query.testCaseId } : {}), + ...(query.evidence !== undefined + ? { evidence: query.evidence as NonNullable } + : {}), + ...(query.region !== undefined ? { region: query.region } : {}), + ...(query.accountTier !== undefined ? { accountTier: query.accountTier } : {}), + ...(query.serviceTier !== undefined ? { serviceTier: query.serviceTier } : {}), + ...(query.cost !== undefined + ? { cost: query.cost as NonNullable } + : {}), + ...(query.cursor !== undefined ? { cursor: query.cursor } : {}), + }; + try { + const validated = validateArchiveResearchFilters(filters); + if (validated.cursor !== undefined) { + return c.json({ error: "invalid_research_facet_cursor" }, 400); + } + return c.json({ facets: await deps.archive.researchFacets(validated) }); + } catch (error) { + if (error instanceof Error && error.message.startsWith("invalid_")) { + return c.json({ error: error.message }, 400); + } + throw error; + } + }); + + app.get("/v1/archive/research/collections/:collectionId/assessment", async (c) => { + if (!deps.researchCollections || !deps.archive) { + return c.json({ error: "research_collections_unavailable" }, 503); + } + const id = c.req.param("collectionId"); + if (!UUID_RE.test(id)) { + return c.json({ error: "invalid_research_collection_id" }, 400); + } + const policyVersion = c.req.query("policyVersion"); + if (policyVersion !== undefined && !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion)) { + return c.json({ error: "invalid_policy_version" }, 400); + } + const collection = await deps.researchCollections.get(id); + if (!collection) return c.json({ error: "research_collection_not_found" }, 404); + const assessment = await deps.archive.assessResearchRuns( + collection.runIds, + policyVersion, + ); + if (!assessment) return c.json({ error: "comparability_policy_not_found" }, 404); + return c.json({ + collectionId: collection.id, + manifestSha256: collection.contentSha256, + assessment, + }); + }); + app.get("/v1/archive/changes", async (c) => { if (!deps.archive) { return c.json({ error: "archive_unavailable" }, 503); From 1e1a03de8a732f4f68d87d58c1b9222aed87986d Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:59:20 +0800 Subject: [PATCH 05/13] Add public typed facet discovery and explicit policy-version collection assessment clients --- apps/web/src/modelapse.ts | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index cf7ba13..baf5e89 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -579,6 +579,31 @@ export interface ResearchCollectionDetail extends ResearchCollectionSummary { readonly runs: readonly ArchiveRun[]; } +export interface ArchiveResearchFacets { + readonly scope: "sealed_public_non_calibration"; + readonly counting: "matching_runs_all_pages"; + readonly totalMatches: number; + readonly perFacetLimit: 25; + readonly facets: Record< + "provider" | "model" | "test" | "evidence" | "region" | + "accountTier" | "serviceTier" | "serviceAssurance" | "cost", + { + readonly values: readonly { readonly value: string | null; readonly count: number }[]; + readonly truncated: boolean; + } + >; +} + +export interface ArchiveResearchPolicyAssessment { + readonly policy: ArchiveComparabilityPolicy; + readonly scope: "per_run_only_not_cross_provider_match"; + readonly collectionRunCount: number; + readonly rows: readonly { + readonly runId: string; + readonly comparability: ArchiveRunComparability; + }[]; +} + export interface ArchiveResearchPage { readonly runs: readonly ArchiveRun[]; readonly nextCursor: string | null; @@ -1564,6 +1589,10 @@ interface ResearchCollectionIdInput { readonly collectionId: string; } +interface ResearchCollectionAssessmentInput extends ResearchCollectionIdInput { + readonly policyVersion: string; +} + interface ResearchCollectionExportInput extends ResearchCollectionIdInput { readonly format: "json" | "csv"; } @@ -2264,6 +2293,18 @@ function parseResearchCollectionId(value: unknown): ResearchCollectionIdInput { return { collectionId: value.collectionId }; } +function parseResearchCollectionAssessment( + value: unknown, +): ResearchCollectionAssessmentInput { + const { collectionId } = parseResearchCollectionId(value); + if (!isRecord(value) || + typeof value.policyVersion !== "string" || + !/^[a-z0-9][a-z0-9._-]*$/.test(value.policyVersion)) { + throw new Error("Research assessment requires a valid policy version"); + } + return { collectionId, policyVersion: value.policyVersion }; +} + function parseResearchCollectionExport(value: unknown): ResearchCollectionExportInput { const { collectionId } = parseResearchCollectionId(value); if (!isRecord(value) || @@ -2424,6 +2465,39 @@ export const searchArchiveResearch = createServerFn({ method: "POST" }) return result.research; }); +export const getArchiveResearchFacets = createServerFn({ method: "POST" }) + .validator(parseArchiveResearchInput) + .handler(async ({ data }): Promise => { + if (data.cursor) { + throw new Error("Research facets cannot use a pagination cursor"); + } + const params = new URLSearchParams(); + if (data.providerSlug) params.set("provider", data.providerSlug); + if (data.modelId) params.set("modelId", data.modelId); + if (data.testCaseId) params.set("testCaseId", data.testCaseId); + if (data.evidence && data.evidence !== "any") params.set("evidence", data.evidence); + if (data.region) params.set("region", data.region); + if (data.accountTier) params.set("accountTier", data.accountTier); + if (data.serviceTier) params.set("serviceTier", data.serviceTier); + if (data.cost && data.cost !== "any") params.set("cost", data.cost); + const result = await requestJson<{ facets: ArchiveResearchFacets }>( + `/v1/archive/research/facets?${params.toString()}`, + ); + return result.facets; + }); + +export const getResearchCollectionAssessment = createServerFn({ method: "POST" }) + .validator(parseResearchCollectionAssessment) + .handler(async ({ data }): Promise<{ + readonly collectionId: string; + readonly manifestSha256: string; + readonly assessment: ArchiveResearchPolicyAssessment; + }> => { + return requestJson( + `/v1/archive/research/collections/${data.collectionId}/assessment?policyVersion=${encodeURIComponent(data.policyVersion)}`, + ); + }); + export const getResearchCollections = createServerFn({ method: "GET" }).handler( async (): Promise => { const result = await requestJson<{ From 8fba5352054a423edaf65d1976e3014fa0863abc Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:59:40 +0800 Subject: [PATCH 06/13] Display count-scoped typed research facet discovery with filter links and no quality ranking --- apps/web/src/routes/research.tsx | 91 ++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/apps/web/src/routes/research.tsx b/apps/web/src/routes/research.tsx index 1338ae8..bd8e547 100644 --- a/apps/web/src/routes/research.tsx +++ b/apps/web/src/routes/research.tsx @@ -2,6 +2,7 @@ import { createFileRoute } from "@tanstack/react-router"; import { useState } from "react"; import { getArchiveCatalog, + getArchiveResearchFacets, searchArchiveResearch, captureResearchCollection, type ArchiveRun, @@ -61,11 +62,13 @@ export const Route = createFileRoute("/research")({ ...(deps.cursor ? { cursor: deps.cursor } : {}), limit: 20, }; - const [catalog, research] = await Promise.all([ + const { cursor: _cursor, limit: _limit, ...faceted } = request; + const [catalog, research, facets] = await Promise.all([ getArchiveCatalog(), searchArchiveResearch({ data: request }), + getArchiveResearchFacets({ data: faceted }), ]); - return { catalog, research }; + return { catalog, research, facets }; }, component: ArchiveResearchPage, }); @@ -90,7 +93,7 @@ function evaluation(run: ArchiveRun): string { } function ArchiveResearchPage() { - const { catalog, research } = Route.useLoaderData(); + const { catalog, research, facets } = Route.useLoaderData(); const filters = Route.useSearch(); const [operatorToken, setOperatorToken] = useState(""); const [collectionTitle, setCollectionTitle] = useState(""); @@ -143,6 +146,36 @@ function ArchiveResearchPage() { if (filters.serviceTier) withoutCursor.set("serviceTier", filters.serviceTier); if (filters.cost) withoutCursor.set("cost", filters.cost); + function facetHref(field: string, value: string): string { + const params = new URLSearchParams(withoutCursor); + params.set(field, value); + return "/research?" + params.toString(); + } + + const facetGroups = [ + { key: "provider", label: "Provider", query: "provider" }, + { key: "model", label: "Model", query: "modelId" }, + { key: "test", label: "Exact Test", query: "testCaseId" }, + { key: "evidence", label: "Evidence", query: "evidence" }, + { key: "region", label: "Region", query: "region" }, + { key: "accountTier", label: "Account tier", query: "accountTier" }, + { key: "serviceTier", label: "Service tier", query: "serviceTier" }, + { key: "cost", label: "Native cost", query: "cost" }, + { key: "serviceAssurance", label: "Assurance (read-only)", query: null }, + ] as const; + + function facetLabel(key: string, value: string | null): string { + if (value === null) return "unknown / not recorded"; + if (key === "model") { + return catalog.models.find((model) => model.id === value)?.marketingName ?? value; + } + if (key === "test") { + const test = catalog.tests.find((entry) => entry.testCaseId === value); + return test ? test.familySlug + " / " + test.caseSlug : value; + } + return value; + } + const nextParams = new URLSearchParams(withoutCursor); if (research.nextCursor) nextParams.set("cursor", research.nextCursor); @@ -316,6 +349,58 @@ function ArchiveResearchPage() { +
+
+
+

ARCHIVE v0.28 · OBSERVED FACETS

+

Evidence-backed filter counts

+
+ {facets.totalMatches} matching Runs · all pages +
+

+ Each count represents sealed public non-calibration Runs satisfying + all current filters, not just this page. Facets count + archival observations, not unique models or a quality score. + Up to {facets.perFacetLimit} values are displayed per facet, with + overflow explicitly marked. Unknown context is never inferred. +

+
+ {facetGroups.map(({ key, label, query }) => { + const facet = facets.facets[key]; + return ( +
+
+ {label} + + {facet.values.length} visible value(s) + {facet.truncated ? " · truncated" : ""} + +
+
+ {facet.values.slice(0, 8).map((entry) => { + const selectable = query !== null && entry.value !== null && + (key !== "evidence" || entry.value === "E4+" || + entry.value === "missing"); + return selectable ? ( + + {facetLabel(key, entry.value)} ({entry.count}) + + ) : ( + + {facetLabel(key, entry.value)} ({entry.count}) + + ); + })} + {!facet.values.length ? No matching observations : null} + {facet.values.length > 8 ? Showing first 8 · more in API : null} +
+
+ ); + })} +
+
+
From a69ab536ba3012069d2f1d04278a6b3b7027dc6c Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:00:02 +0800 Subject: [PATCH 07/13] Enable explicit policy-version annotations for immutable collection Runs without set-ranking claims --- .../research.collections.$collectionId.tsx | 111 +++++++++++++++++- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/apps/web/src/routes/research.collections.$collectionId.tsx b/apps/web/src/routes/research.collections.$collectionId.tsx index 40b0905..4ac98d4 100644 --- a/apps/web/src/routes/research.collections.$collectionId.tsx +++ b/apps/web/src/routes/research.collections.$collectionId.tsx @@ -2,18 +2,52 @@ import { createFileRoute } from "@tanstack/react-router"; import { useState } from "react"; import { downloadResearchCollection, + getArchiveComparabilityPolicies, getResearchCollection, + getResearchCollectionAssessment, + type ArchiveResearchPolicyAssessment, } from "../modelapse"; export const Route = createFileRoute("/research/collections/$collectionId")({ - loader: ({ params }) => getResearchCollection({ - data: { collectionId: params.collectionId }, - }), + loader: async ({ params }) => { + const [collection, policies] = await Promise.all([ + getResearchCollection({ data: { collectionId: params.collectionId } }), + getArchiveComparabilityPolicies(), + ]); + return { collection, policies }; + }, component: ResearchCollectionDetailPage, }); function ResearchCollectionDetailPage() { - const collection = Route.useLoaderData(); + const { collection, policies } = Route.useLoaderData(); + const [policyVersion, setPolicyVersion] = useState(policies[0]?.version ?? ""); + const [assessment, setAssessment] = + useState(null); + const [assessing, setAssessing] = useState(false); + const [assessmentError, setAssessmentError] = useState(null); + + async function assessPolicy(): Promise { + if (!collection || !policyVersion) return; + setAssessing(true); + setAssessmentError(null); + setAssessment(null); + try { + const result = await getResearchCollectionAssessment({ + data: { collectionId: collection.id, policyVersion }, + }); + if (result.manifestSha256 !== collection.contentSha256) { + throw new Error("Research manifest changed during assessment"); + } + setAssessment(result.assessment); + } catch (caught) { + setAssessmentError( + caught instanceof Error ? caught.message : "Assessment unavailable", + ); + } finally { + setAssessing(false); + } + } const [exporting, setExporting] = useState<"csv" | "json" | null>(null); const [error, setError] = useState(null); @@ -102,6 +136,75 @@ function ResearchCollectionDetailPage() {
{error ?
{error}
: null}
+
+
+
+

ARCHIVE v0.28 · POLICY-AWARE ANNOTATIONS

+

Reassess frozen Runs with a versioned policy

+
+ derived · not stored in manifest +
+

+ This applies the chosen immutable comparability-policy version + to every captured Run individually, using its historical execution + context, prior calibration evidence and replication available at the + time of that Run. It does not declare this collection a matched + cross-Provider set. A later policy interpretation never changes + captured membership or the SHA-256 manifest. +

+
+ +
+ +
+
+ {assessmentError ?
{assessmentError}
: null} + {assessment ? ( +
+

+ Policy {assessment.policy.version} · {assessment.collectionRunCount} Run(s) + · scope: individual eligibility only, not matched cross-Provider comparison. + Results are derived at request time. +

+ {assessment.rows.map((entry) => ( +
+
+ + Run {entry.runId.slice(0, 8)} → + + {entry.comparability.status} +
+

+ Replication {entry.comparability.repeatCount} / + {" "}{entry.comparability.requiredRepeatCount} + {" · "}Prior calibration{" "} + {entry.comparability.calibration?.status ?? "unknown"} +

+

+ {entry.comparability.reasons.length + ? entry.comparability.reasons.join(" · ") + : "No per-Run blockers under this policy; collection-level matching not checked."} +

+
+ ))} +
+ ) : null} +

CAPTURED SOURCE RECORDS

Run evidence

From b7a15e6ac9cfe928d5e984950b5e70baa1d39d0c Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:00:27 +0800 Subject: [PATCH 08/13] Keep orphan-model archived Runs in research facet counts consistently with main query --- packages/persistence/src/archive-repository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index d40a8a0..4df1591 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -1290,7 +1290,7 @@ export class PgArchiveRepository { END AS cost FROM modelapse.runs r JOIN modelapse.test_cases tc ON tc.id = r.test_case_id - JOIN modelapse.models m ON m.id = r.model_id + LEFT JOIN modelapse.models m ON m.id = r.model_id JOIN modelapse.providers p ON p.id = r.provider_id LEFT JOIN modelapse.run_evidence_summary res ON res.run_id = r.id LEFT JOIN modelapse.run_execution_qualification qualification From ebde69fbc3530cef4fdb8fd46ac42acf58ba2b5b Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:00:42 +0800 Subject: [PATCH 09/13] Test observed facet population/counts and versioned per-Run snapshot annotation against PostgreSQL --- .../test/comparability.integration.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/packages/persistence/test/comparability.integration.test.ts b/packages/persistence/test/comparability.integration.test.ts index 000cfe0..937e8e5 100644 --- a/packages/persistence/test/comparability.integration.test.ts +++ b/packages/persistence/test/comparability.integration.test.ts @@ -594,6 +594,47 @@ describe("Archive comparability policy", () => { ), ).rejects.toThrow(/sealed public non-calibration Runs/i); + const facets = await archive.researchFacets({ + providerSlug: "comparability-provider-" + suffix + "-2", + testCaseId: benchmarkCase.rows[0]!.id, + }); + expect(facets).toMatchObject({ + scope: "sealed_public_non_calibration", + counting: "matching_runs_all_pages", + totalMatches: 2, + perFacetLimit: 25, + }); + expect(facets.facets.provider.values).toEqual([{ + value: "comparability-provider-" + suffix + "-2", + count: 2, + }]); + expect(facets.facets.region.values).toEqual( + expect.arrayContaining([ + { value: "US", count: 1 }, + { value: "JP", count: 1 }, + ]), + ); + expect(facets.facets.evidence.values).toEqual([ + { value: "E4+", count: 2 }, + ]); + const regionalFacets = await archive.researchFacets({ + testCaseId: benchmarkCase.rows[0]!.id, + region: "JP", + evidence: "E4+", + }); + expect(regionalFacets.totalMatches).toBe(1); + expect(regionalFacets.facets.region.values).toEqual([ + { value: "JP", count: 1 }, + ]); + const emptyFacets = await archive.researchFacets({ + providerSlug: "provider-not-present-" + suffix, + }); + expect(emptyFacets.totalMatches).toBe(0); + expect(emptyFacets.facets.provider.values).toEqual([]); + await expect( + archive.researchFacets({ cursor: research.nextCursor! }), + ).rejects.toThrow("invalid_research_facet_cursor"); + const custom = await policies.recordPolicy({ version: "comparability-integration-" + suffix, minimumEvidenceLevel: "E4", @@ -612,6 +653,29 @@ describe("Archive comparability policy", () => { }); expect(custom.version).toBe("comparability-integration-" + suffix); + const annotated = await archive.assessResearchRuns( + [research.runs[0]!.id, older.runs[0]!.id], + custom.version, + ); + expect(annotated).toMatchObject({ + scope: "per_run_only_not_cross_provider_match", + collectionRunCount: 2, + policy: { version: custom.version }, + rows: [ + { runId: research.runs[0]!.id }, + { runId: older.runs[0]!.id }, + ], + }); + expect(annotated!.rows.every((item) => + ["eligible", "unknown", "ineligible"].includes(item.comparability.status), + )).toBe(true); + expect(annotated!.rows.every((item) => + item.comparability.repeatCount >= 1, + )).toBe(true); + expect(await archive.assessResearchRuns( + [research.runs[0]!.id], "unknown-policy-" + suffix, + )).toBeNull(); + await expect( pool.query( `UPDATE modelapse.comparability_policies From f141ce90ec2202e7e9d2ef006f65034063b65019 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:01:14 +0800 Subject: [PATCH 10/13] Cover typed Facet API and policy-anchored snapshot annotations with public guards --- apps/api/test/archive-api.test.ts | 116 ++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index 9098fa5..5648cdd 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -546,6 +546,38 @@ describe("Archive read API", () => { hasMore: false, scope: "sealed_public_non_calibration", }), + researchFacets: async () => ({ + scope: "sealed_public_non_calibration" as const, + counting: "matching_runs_all_pages" as const, + totalMatches: 1, + perFacetLimit: 25 as const, + facets: { + provider: { values: [{ value: "deepseek", count: 1 }], truncated: false }, + model: { values: [{ value: MODEL_ID, count: 1 }], truncated: false }, + test: { values: [{ value: TEST_CASE_ID, count: 1 }], truncated: false }, + evidence: { values: [{ value: "E4+", count: 1 }], truncated: false }, + region: { values: [{ value: "US", count: 1 }], truncated: false }, + accountTier: { values: [], truncated: false }, + serviceTier: { values: [], truncated: false }, + serviceAssurance: { values: [], truncated: false }, + cost: { values: [{ value: "unknown", count: 1 }], truncated: false }, + }, + }), + assessResearchRuns: async () => ({ + policy: archiveComparison.policy, + scope: "per_run_only_not_cross_provider_match" as const, + collectionRunCount: 1, + rows: [{ + runId: RUN_ID, + comparability: { + status: "unknown" as const, + reasons: ["recent_calibration_missing"], + repeatCount: 1, + requiredRepeatCount: 1, + calibration: null, + }, + }], + }), listCatalogChanges: async () => [archiveCatalogChange], getRun: async (runId) => (runId === RUN_ID ? archiveRun : null), getModel: async (modelId) => @@ -574,6 +606,18 @@ describe("Archive read API", () => { ], }); + const facets = await app.request( + `/v1/archive/research/facets?provider=deepseek®ion=US`, + ); + expect(facets.status).toBe(200); + await expect(facets.json()).resolves.toMatchObject({ + facets: { + scope: "sealed_public_non_calibration", + totalMatches: 1, + facets: { provider: { values: [{ value: "deepseek", count: 1 }] } }, + }, + }); + const changes = await app.request( `/v1/archive/changes?modelId=${MODEL_ID}&provider=deepseek&limit=10`, ); @@ -667,6 +711,18 @@ describe("Archive read API", () => { hasMore: false, scope: "sealed_public_non_calibration", }), + researchFacets: async () => ({ + scope: "sealed_public_non_calibration" as const, + counting: "matching_runs_all_pages" as const, + totalMatches: 0, + perFacetLimit: 25 as const, + facets: Object.fromEntries( + ["provider","model","test","evidence","region","accountTier","serviceTier","serviceAssurance","cost"].map(key => + [key, { values: [], truncated: false }], + ), + ) as never, + }), + assessResearchRuns: async () => null, listCatalogChanges: async () => [], getRun: async () => null, getModel: async () => null, @@ -679,6 +735,12 @@ describe("Archive read API", () => { expect( (await app.request("/v1/archive/runs?modelId=nope")).status, ).toBe(400); + expect( + (await app.request("/v1/archive/research/facets?cursor=not-a-cursor!")).status, + ).toBe(400); + expect( + (await app.request("/v1/archive/research/facets?evidence=E7")).status, + ).toBe(400); for (const query of [ "limit=51", "evidence=E6", @@ -754,6 +816,44 @@ describe("Archive v0.27 immutable research collections API", () => { const app = createApp({ runs: baseRuns(), controlToken: "control-secret", + archive: { + ping: async () => undefined, + listModels: async () => [], + listTests: async () => [], + listRuns: async () => [], + researchRuns: async () => ({ + runs: [], nextCursor: null, hasMore: false, + scope: "sealed_public_non_calibration" as const, + }), + researchFacets: async () => ({ + scope: "sealed_public_non_calibration" as const, + counting: "matching_runs_all_pages" as const, + totalMatches: 0, + perFacetLimit: 25 as const, + facets: {} as never, + }), + assessResearchRuns: async () => ({ + policy: archiveComparison.policy, + scope: "per_run_only_not_cross_provider_match" as const, + collectionRunCount: 1, + rows: [{ + runId: RUN_ID, + comparability: { + status: "unknown" as const, + reasons: ["recent_calibration_missing"], + repeatCount: 1, + requiredRepeatCount: 1, + calibration: null, + }, + }], + }), + listCatalogChanges: async () => [], + getRun: async () => null, + getModel: async () => null, + getTest: async () => null, + compareLatest: async () => null, + getRunHistory: async () => null, + }, researchCollections: { ping: async () => undefined, list: async () => [{ ...collection }], @@ -778,6 +878,22 @@ describe("Archive v0.27 immutable research collections API", () => { collection: { id, runs: [{ id: RUN_ID }] }, }); + const annotated = await app.request( + "/v1/archive/research/collections/" + id + + "/assessment?policyVersion=" + archiveComparison.policy.version, + ); + expect(annotated.status).toBe(200); + await expect(annotated.json()).resolves.toMatchObject({ + collectionId: id, + manifestSha256: collection.contentSha256, + assessment: { + scope: "per_run_only_not_cross_provider_match", + rows: [{ runId: RUN_ID, comparability: { status: "unknown" } }], + }, + }); + expect((await app.request( + "/v1/archive/research/collections/" + id + "/assessment?policyVersion=BAD", + )).status).toBe(400); const json = await app.request( "/v1/archive/research/collections/" + id + "/export?format=json", ); From bd6d8330a06836c246c6d61d4e42e7ebf4aae820 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:01:44 +0800 Subject: [PATCH 11/13] Document Archive v0.28 typed facet semantics and policy annotation bounds --- docs/research-facets-policy-annotations.md | 93 ++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docs/research-facets-policy-annotations.md diff --git a/docs/research-facets-policy-annotations.md b/docs/research-facets-policy-annotations.md new file mode 100644 index 0000000..2cb47e3 --- /dev/null +++ b/docs/research-facets-policy-annotations.md @@ -0,0 +1,93 @@ +# Typed Research Facets & Policy-Anchored Annotations — Archive v0.28 + +Archive v0.28 extends public evidence exploration without introducing cross-Test scores, unofficial leaderboards, or retroactive Run mutation. + +## Observed research facets + +Public endpoint: + +```text +GET /v1/archive/research/facets + ?provider= + &modelId= + &testCaseId= + &evidence=any|E4+|missing + ®ion= + &accountTier= + &serviceTier= + &cost=any|estimated|unknown +``` + +All filters use the same validation and historical field semantics as `/v1/archive/research`. No cursor is allowed: counts are computed over **all sealed completed public non-calibration Runs matching the entire active filter set**, not the first 20 results and not the entire unfiltered Archive unless filters are empty. + +Response: + +```json +{ + "facets": { + "scope": "sealed_public_non_calibration", + "counting": "matching_runs_all_pages", + "totalMatches": 0, + "perFacetLimit": 25, + "facets": { + "provider": { "values": [], "truncated": false }, + "model": { "values": [], "truncated": false }, + "test": { "values": [], "truncated": false }, + "evidence": { "values": [], "truncated": false }, + "region": { "values": [], "truncated": false }, + "accountTier": { "values": [], "truncated": false }, + "serviceTier": { "values": [], "truncated": false }, + "serviceAssurance": { "values": [], "truncated": false }, + "cost": { "values": [], "truncated": false } + } + } +} +``` + +Each facet value has a `value` string or `null` (missing/unknown) and an integer `count`. Top values sort by descending observed Run count then value, with at most 25 values per facet, and `truncated=true` when more exist. Unknown region, account or assurance is not treated as a valid tier or hidden. + +Evidence buckets are `E4+`, `below_E4`, and `missing`. These represent evidence availability, **not model performance**. The current filter API permits `any`, `E4+`, and `missing`; below-E4 is displayed for audit but not as a selectable filter. Cost buckets are `estimated` / `unknown`, never `free`. + +The public `/research` page displays observed counts and links to applicable facets. Counts represent **Run records, not unique Models, Tests or users**. Selecting a value applies it on top of all other active filters; a selected facet's counts are likewise already constrained by that filter. + +## Per-Run policy assessment of immutable research collections + +Public endpoint: + +```text +GET /v1/archive/research/collections/:collectionId/assessment?policyVersion= +``` + +The endpoint first verifies the saved collection's immutable Run membership and SHA-256 manifest. It then loads the requested append-only Comparability Policy, and reuses the **v0.23 per-Run evaluator** against each captured Run in stored manifest order. Results include: + +- exact `policy.version` and policy definition; +- `collectionId` and `manifestSha256`; +- `scope="per_run_only_not_cross_provider_match"`; +- each captured `runId`, `eligible/ineligible/unknown` status, machine-readable reasons; +- required/observed replication count in the Run's original execution context; +- only prior same-context calibration evidence and anomaly detail, never a later canary retroactively applied. + +The policy is selected at *interpretation time*; applying a new version to an old selection does not change the original Run, collection membership, or selection digest. The page makes the selected policy version and result scope visible. No aggregate readiness score or collection-wide matched status is calculated; to evaluate a matched cross-provider set, visitors must still use the **same-Test `/compare`** interface. + +The `/research/collections/:id` page has a user-triggered assessment action and a policy-version selector, so it does not assume a newest policy is equivalent to the policy in effect when a Run was executed. Stored JSON/CSV exports from v0.27 retain their *sourced immutable Run representation* and do not silently merge a dynamic policy assessment. + +## Provenance and invariants + +1. Facet counts are historical observations, not current Provider catalog capability or a live access-status check. +2. Facets count Runs across the whole filtered query; there is no accidental current-page sampling. +3. Missing qualification attributes remain unknown, never default or unrestricted. +4. A Run's individual eligibility is not a cross-Provider matched-set verdict. +5. Calibration evidence must predate each evaluated Run and match its frozen context. +6. Historical Run, cost, qualification, calibration and saved-selection manifest rows remain immutable. +7. All queries are bounded by a typed filter vocabulary, or by immutable collection IDs capped at 50 Runs. Distinct facet lists are bounded and explicitly marked if truncated. +8. Display order or observation volume is never a ranking or model performance score. + +## Verification + +- Public API tests assert facet count payload shape, malformed request rejection, versioned assessment metadata and immutable manifest linkage. +- PostgreSQL integration verifies total counts across two pages of historical Runs, scoped region/evidence facets, empty populations, cursor rejection, and policy-version Run assessment without modifying saved Runs. +- Standard CI validates TypeScript, unit tests, migrations, persistence/queue/provider integration and production Docker images. + +### Deferred follow-up (v0.29+) + +A future explicitly transaction-scoped, multi-page frozen collection protocol may lift the 50-Run snapshot limit; it must not silently truncate or treat a saved cursor as a complete research slice. From 871591a4d20ac289157612393bd62cf030481c84 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:01:46 +0800 Subject: [PATCH 12/13] ADR 0024: keep facet scope and per-Run assessment explicit --- docs/adr/0024-research-facets-policy-scope.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/adr/0024-research-facets-policy-scope.md diff --git a/docs/adr/0024-research-facets-policy-scope.md b/docs/adr/0024-research-facets-policy-scope.md new file mode 100644 index 0000000..e011ec8 --- /dev/null +++ b/docs/adr/0024-research-facets-policy-scope.md @@ -0,0 +1,26 @@ +# ADR 0024 — Research facet counts and policy assessments have explicit scopes + +## Status + +Accepted for Archive v0.28. + +## Context + +A limited UI page is not a valid basis for reporting whole-archive facet frequencies. Equally, a saved research selection cannot acquire a collective comparability verdict by applying per-Run eligibility rules; matched sets require same Test and matched contexts. + +## Decision + +1. Expose database-aggregated facets computed over the complete filtered eligible public research population, independent of pagination. +2. Count historical sealed non-calibration **Runs**, not Models or unique Tests; make this unit visible. +3. Type facet keys and response records; cap displayed values and report truncation rather than silently omit the remainder. +4. Represent missing context as null/unknown and maintain existing E4+ and cost availability semantics. +5. Evaluate saved collections' immutable Run memberships with a specifically selected append-only Comparability Policy version. +6. Reuse the existing per-Run evaluator and its historical replication/calibration rules; do not reinvent partial eligibility logic in the Web UI. +7. Label assessment `per_run_only_not_cross_provider_match`; no collective eligibility, winner or numeric ranking is inferred. +8. Keep derived policy assessments separate from collection selection digests and sourced JSON/CSV snapshots. +9. Do not allow paginated cursors to distort facet count scope. +10. Do not mutate original Run, Evaluation, qualification, calibration or research-collection records. + +## Consequences + +Research filters become explainable and auditable without conflating collection size with quality, while published snapshots can be reinterpreted under explicit policy versions without rewriting historical evidence. From dc12b938931bf65689c6620fd08688a1e659b760 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 12:01:59 +0800 Subject: [PATCH 13/13] Mark v0.28 scoped facet discovery and policy-aware research implemented --- docs/archive-roadmap.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/archive-roadmap.md b/docs/archive-roadmap.md index 7abbaa7..34a6cea 100644 --- a/docs/archive-roadmap.md +++ b/docs/archive-roadmap.md @@ -317,9 +317,17 @@ Operators can publish bounded, immutable collections of up to 50 sealed public n Public collection pages support sharing, live filter replay and source-backed JSON/CSV exports with qualification, access, pricing, region/tier and evidence caveats. Empty, oversized and cursor-partial captures fail closed. The export is not a comparability or quality ranking. -### Archive v0.28+ — Typed Research Facets & Policy-Aware Workflows +### Archive v0.28 — Typed Research Facets & Policy-Aware Workflows -Next, consider facet discovery with observed counts, policy-version anchored research annotations and reproducible multi-page collection protocols. Explicitly distinguish current Provider policy from historical Run state; never aggregate heterogeneous Test results into a leaderboard. +**Implemented:** see `docs/research-facets-policy-annotations.md` and ADR 0024. + +Public `/research` now discovers counts from the full filtered sealed non-calibration Run population (not the current page), grouped by Provider, canonical Model, exact Test, evidence, historical region/account/service tier, service assurance and estimated/unknown native cost. Typed facets are bounded, with explicit missing values and overflow markers. + +Published immutable Research Collections can be reassessed Run-by-Run under a selected Comparability Policy version, reusing historical replication and prior same-context calibration. These interpretations never rewrite the saved Run membership or claim that the collection is a cross-Provider matched set. + +### Archive v0.29+ — Reproducible Multi-Page Research Captures + +Consider transaction-consistent snapshot capture beyond the 50-Run selection limit, plus policy-anchored structured annotation export. Keep immutable Run IDs, manifest digest verification and no-composite-ranking invariants. ## Scheduling principle