diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index b5c60df..f59687a 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -120,6 +120,8 @@ type ArchiveRepository = Pick< | "listTests" | "listRuns" | "researchRuns" + | "researchFacets" + | "assessResearchRuns" | "listCatalogChanges" | "getRun" | "getModel" @@ -417,6 +419,64 @@ export function createApp(deps: AppDependencies) { } }); + app.get("/v1/archive/research/facets", async (c) => { + if (!deps.archive) return c.json({ error: "archive_unavailable" }, 503); + const query = c.req.query(); + const filters: ArchiveResearchFilters = { + ...(query.provider !== undefined ? { providerSlug: query.provider } : {}), + ...(query.modelId !== undefined ? { modelId: query.modelId } : {}), + ...(query.testCaseId !== undefined ? { testCaseId: query.testCaseId } : {}), + ...(query.evidence !== undefined + ? { evidence: query.evidence as NonNullable } + : {}), + ...(query.region !== undefined ? { region: query.region } : {}), + ...(query.accountTier !== undefined ? { accountTier: query.accountTier } : {}), + ...(query.serviceTier !== undefined ? { serviceTier: query.serviceTier } : {}), + ...(query.cost !== undefined + ? { cost: query.cost as NonNullable } + : {}), + ...(query.cursor !== undefined ? { cursor: query.cursor } : {}), + }; + try { + const validated = validateArchiveResearchFilters(filters); + if (validated.cursor !== undefined) { + return c.json({ error: "invalid_research_facet_cursor" }, 400); + } + return c.json({ facets: await deps.archive.researchFacets(validated) }); + } catch (error) { + if (error instanceof Error && error.message.startsWith("invalid_")) { + return c.json({ error: error.message }, 400); + } + throw error; + } + }); + + app.get("/v1/archive/research/collections/:collectionId/assessment", async (c) => { + if (!deps.researchCollections || !deps.archive) { + return c.json({ error: "research_collections_unavailable" }, 503); + } + const id = c.req.param("collectionId"); + if (!UUID_RE.test(id)) { + return c.json({ error: "invalid_research_collection_id" }, 400); + } + const policyVersion = c.req.query("policyVersion"); + if (policyVersion !== undefined && !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion)) { + return c.json({ error: "invalid_policy_version" }, 400); + } + const collection = await deps.researchCollections.get(id); + if (!collection) return c.json({ error: "research_collection_not_found" }, 404); + const assessment = await deps.archive.assessResearchRuns( + collection.runIds, + policyVersion, + ); + if (!assessment) return c.json({ error: "comparability_policy_not_found" }, 404); + return c.json({ + collectionId: collection.id, + manifestSha256: collection.contentSha256, + assessment, + }); + }); + app.get("/v1/archive/changes", async (c) => { if (!deps.archive) { return c.json({ error: "archive_unavailable" }, 503); diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index 9098fa5..5648cdd 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -546,6 +546,38 @@ describe("Archive read API", () => { hasMore: false, scope: "sealed_public_non_calibration", }), + researchFacets: async () => ({ + scope: "sealed_public_non_calibration" as const, + counting: "matching_runs_all_pages" as const, + totalMatches: 1, + perFacetLimit: 25 as const, + facets: { + provider: { values: [{ value: "deepseek", count: 1 }], truncated: false }, + model: { values: [{ value: MODEL_ID, count: 1 }], truncated: false }, + test: { values: [{ value: TEST_CASE_ID, count: 1 }], truncated: false }, + evidence: { values: [{ value: "E4+", count: 1 }], truncated: false }, + region: { values: [{ value: "US", count: 1 }], truncated: false }, + accountTier: { values: [], truncated: false }, + serviceTier: { values: [], truncated: false }, + serviceAssurance: { values: [], truncated: false }, + cost: { values: [{ value: "unknown", count: 1 }], truncated: false }, + }, + }), + assessResearchRuns: async () => ({ + policy: archiveComparison.policy, + scope: "per_run_only_not_cross_provider_match" as const, + collectionRunCount: 1, + rows: [{ + runId: RUN_ID, + comparability: { + status: "unknown" as const, + reasons: ["recent_calibration_missing"], + repeatCount: 1, + requiredRepeatCount: 1, + calibration: null, + }, + }], + }), listCatalogChanges: async () => [archiveCatalogChange], getRun: async (runId) => (runId === RUN_ID ? archiveRun : null), getModel: async (modelId) => @@ -574,6 +606,18 @@ describe("Archive read API", () => { ], }); + const facets = await app.request( + `/v1/archive/research/facets?provider=deepseek®ion=US`, + ); + expect(facets.status).toBe(200); + await expect(facets.json()).resolves.toMatchObject({ + facets: { + scope: "sealed_public_non_calibration", + totalMatches: 1, + facets: { provider: { values: [{ value: "deepseek", count: 1 }] } }, + }, + }); + const changes = await app.request( `/v1/archive/changes?modelId=${MODEL_ID}&provider=deepseek&limit=10`, ); @@ -667,6 +711,18 @@ describe("Archive read API", () => { hasMore: false, scope: "sealed_public_non_calibration", }), + researchFacets: async () => ({ + scope: "sealed_public_non_calibration" as const, + counting: "matching_runs_all_pages" as const, + totalMatches: 0, + perFacetLimit: 25 as const, + facets: Object.fromEntries( + ["provider","model","test","evidence","region","accountTier","serviceTier","serviceAssurance","cost"].map(key => + [key, { values: [], truncated: false }], + ), + ) as never, + }), + assessResearchRuns: async () => null, listCatalogChanges: async () => [], getRun: async () => null, getModel: async () => null, @@ -679,6 +735,12 @@ describe("Archive read API", () => { expect( (await app.request("/v1/archive/runs?modelId=nope")).status, ).toBe(400); + expect( + (await app.request("/v1/archive/research/facets?cursor=not-a-cursor!")).status, + ).toBe(400); + expect( + (await app.request("/v1/archive/research/facets?evidence=E7")).status, + ).toBe(400); for (const query of [ "limit=51", "evidence=E6", @@ -754,6 +816,44 @@ describe("Archive v0.27 immutable research collections API", () => { const app = createApp({ runs: baseRuns(), controlToken: "control-secret", + archive: { + ping: async () => undefined, + listModels: async () => [], + listTests: async () => [], + listRuns: async () => [], + researchRuns: async () => ({ + runs: [], nextCursor: null, hasMore: false, + scope: "sealed_public_non_calibration" as const, + }), + researchFacets: async () => ({ + scope: "sealed_public_non_calibration" as const, + counting: "matching_runs_all_pages" as const, + totalMatches: 0, + perFacetLimit: 25 as const, + facets: {} as never, + }), + assessResearchRuns: async () => ({ + policy: archiveComparison.policy, + scope: "per_run_only_not_cross_provider_match" as const, + collectionRunCount: 1, + rows: [{ + runId: RUN_ID, + comparability: { + status: "unknown" as const, + reasons: ["recent_calibration_missing"], + repeatCount: 1, + requiredRepeatCount: 1, + calibration: null, + }, + }], + }), + listCatalogChanges: async () => [], + getRun: async () => null, + getModel: async () => null, + getTest: async () => null, + compareLatest: async () => null, + getRunHistory: async () => null, + }, researchCollections: { ping: async () => undefined, list: async () => [{ ...collection }], @@ -778,6 +878,22 @@ describe("Archive v0.27 immutable research collections API", () => { collection: { id, runs: [{ id: RUN_ID }] }, }); + const annotated = await app.request( + "/v1/archive/research/collections/" + id + + "/assessment?policyVersion=" + archiveComparison.policy.version, + ); + expect(annotated.status).toBe(200); + await expect(annotated.json()).resolves.toMatchObject({ + collectionId: id, + manifestSha256: collection.contentSha256, + assessment: { + scope: "per_run_only_not_cross_provider_match", + rows: [{ runId: RUN_ID, comparability: { status: "unknown" } }], + }, + }); + expect((await app.request( + "/v1/archive/research/collections/" + id + "/assessment?policyVersion=BAD", + )).status).toBe(400); const json = await app.request( "/v1/archive/research/collections/" + id + "/export?format=json", ); diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index cf7ba13..baf5e89 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -579,6 +579,31 @@ export interface ResearchCollectionDetail extends ResearchCollectionSummary { readonly runs: readonly ArchiveRun[]; } +export interface ArchiveResearchFacets { + readonly scope: "sealed_public_non_calibration"; + readonly counting: "matching_runs_all_pages"; + readonly totalMatches: number; + readonly perFacetLimit: 25; + readonly facets: Record< + "provider" | "model" | "test" | "evidence" | "region" | + "accountTier" | "serviceTier" | "serviceAssurance" | "cost", + { + readonly values: readonly { readonly value: string | null; readonly count: number }[]; + readonly truncated: boolean; + } + >; +} + +export interface ArchiveResearchPolicyAssessment { + readonly policy: ArchiveComparabilityPolicy; + readonly scope: "per_run_only_not_cross_provider_match"; + readonly collectionRunCount: number; + readonly rows: readonly { + readonly runId: string; + readonly comparability: ArchiveRunComparability; + }[]; +} + export interface ArchiveResearchPage { readonly runs: readonly ArchiveRun[]; readonly nextCursor: string | null; @@ -1564,6 +1589,10 @@ interface ResearchCollectionIdInput { readonly collectionId: string; } +interface ResearchCollectionAssessmentInput extends ResearchCollectionIdInput { + readonly policyVersion: string; +} + interface ResearchCollectionExportInput extends ResearchCollectionIdInput { readonly format: "json" | "csv"; } @@ -2264,6 +2293,18 @@ function parseResearchCollectionId(value: unknown): ResearchCollectionIdInput { return { collectionId: value.collectionId }; } +function parseResearchCollectionAssessment( + value: unknown, +): ResearchCollectionAssessmentInput { + const { collectionId } = parseResearchCollectionId(value); + if (!isRecord(value) || + typeof value.policyVersion !== "string" || + !/^[a-z0-9][a-z0-9._-]*$/.test(value.policyVersion)) { + throw new Error("Research assessment requires a valid policy version"); + } + return { collectionId, policyVersion: value.policyVersion }; +} + function parseResearchCollectionExport(value: unknown): ResearchCollectionExportInput { const { collectionId } = parseResearchCollectionId(value); if (!isRecord(value) || @@ -2424,6 +2465,39 @@ export const searchArchiveResearch = createServerFn({ method: "POST" }) return result.research; }); +export const getArchiveResearchFacets = createServerFn({ method: "POST" }) + .validator(parseArchiveResearchInput) + .handler(async ({ data }): Promise => { + if (data.cursor) { + throw new Error("Research facets cannot use a pagination cursor"); + } + const params = new URLSearchParams(); + if (data.providerSlug) params.set("provider", data.providerSlug); + if (data.modelId) params.set("modelId", data.modelId); + if (data.testCaseId) params.set("testCaseId", data.testCaseId); + if (data.evidence && data.evidence !== "any") params.set("evidence", data.evidence); + if (data.region) params.set("region", data.region); + if (data.accountTier) params.set("accountTier", data.accountTier); + if (data.serviceTier) params.set("serviceTier", data.serviceTier); + if (data.cost && data.cost !== "any") params.set("cost", data.cost); + const result = await requestJson<{ facets: ArchiveResearchFacets }>( + `/v1/archive/research/facets?${params.toString()}`, + ); + return result.facets; + }); + +export const getResearchCollectionAssessment = createServerFn({ method: "POST" }) + .validator(parseResearchCollectionAssessment) + .handler(async ({ data }): Promise<{ + readonly collectionId: string; + readonly manifestSha256: string; + readonly assessment: ArchiveResearchPolicyAssessment; + }> => { + return requestJson( + `/v1/archive/research/collections/${data.collectionId}/assessment?policyVersion=${encodeURIComponent(data.policyVersion)}`, + ); + }); + export const getResearchCollections = createServerFn({ method: "GET" }).handler( async (): Promise => { const result = await requestJson<{ diff --git a/apps/web/src/routes/research.collections.$collectionId.tsx b/apps/web/src/routes/research.collections.$collectionId.tsx index 40b0905..4ac98d4 100644 --- a/apps/web/src/routes/research.collections.$collectionId.tsx +++ b/apps/web/src/routes/research.collections.$collectionId.tsx @@ -2,18 +2,52 @@ import { createFileRoute } from "@tanstack/react-router"; import { useState } from "react"; import { downloadResearchCollection, + getArchiveComparabilityPolicies, getResearchCollection, + getResearchCollectionAssessment, + type ArchiveResearchPolicyAssessment, } from "../modelapse"; export const Route = createFileRoute("/research/collections/$collectionId")({ - loader: ({ params }) => getResearchCollection({ - data: { collectionId: params.collectionId }, - }), + loader: async ({ params }) => { + const [collection, policies] = await Promise.all([ + getResearchCollection({ data: { collectionId: params.collectionId } }), + getArchiveComparabilityPolicies(), + ]); + return { collection, policies }; + }, component: ResearchCollectionDetailPage, }); function ResearchCollectionDetailPage() { - const collection = Route.useLoaderData(); + const { collection, policies } = Route.useLoaderData(); + const [policyVersion, setPolicyVersion] = useState(policies[0]?.version ?? ""); + const [assessment, setAssessment] = + useState(null); + const [assessing, setAssessing] = useState(false); + const [assessmentError, setAssessmentError] = useState(null); + + async function assessPolicy(): Promise { + if (!collection || !policyVersion) return; + setAssessing(true); + setAssessmentError(null); + setAssessment(null); + try { + const result = await getResearchCollectionAssessment({ + data: { collectionId: collection.id, policyVersion }, + }); + if (result.manifestSha256 !== collection.contentSha256) { + throw new Error("Research manifest changed during assessment"); + } + setAssessment(result.assessment); + } catch (caught) { + setAssessmentError( + caught instanceof Error ? caught.message : "Assessment unavailable", + ); + } finally { + setAssessing(false); + } + } const [exporting, setExporting] = useState<"csv" | "json" | null>(null); const [error, setError] = useState(null); @@ -102,6 +136,75 @@ function ResearchCollectionDetailPage() { {error ?
{error}
: null} +
+
+
+

ARCHIVE v0.28 · POLICY-AWARE ANNOTATIONS

+

Reassess frozen Runs with a versioned policy

+
+ derived · not stored in manifest +
+

+ This applies the chosen immutable comparability-policy version + to every captured Run individually, using its historical execution + context, prior calibration evidence and replication available at the + time of that Run. It does not declare this collection a matched + cross-Provider set. A later policy interpretation never changes + captured membership or the SHA-256 manifest. +

+
+ +
+ +
+
+ {assessmentError ?
{assessmentError}
: null} + {assessment ? ( +
+

+ Policy {assessment.policy.version} · {assessment.collectionRunCount} Run(s) + · scope: individual eligibility only, not matched cross-Provider comparison. + Results are derived at request time. +

+ {assessment.rows.map((entry) => ( +
+
+ + Run {entry.runId.slice(0, 8)} → + + {entry.comparability.status} +
+

+ Replication {entry.comparability.repeatCount} / + {" "}{entry.comparability.requiredRepeatCount} + {" · "}Prior calibration{" "} + {entry.comparability.calibration?.status ?? "unknown"} +

+

+ {entry.comparability.reasons.length + ? entry.comparability.reasons.join(" · ") + : "No per-Run blockers under this policy; collection-level matching not checked."} +

+
+ ))} +
+ ) : null} +

CAPTURED SOURCE RECORDS

Run evidence

diff --git a/apps/web/src/routes/research.tsx b/apps/web/src/routes/research.tsx index 1338ae8..bd8e547 100644 --- a/apps/web/src/routes/research.tsx +++ b/apps/web/src/routes/research.tsx @@ -2,6 +2,7 @@ import { createFileRoute } from "@tanstack/react-router"; import { useState } from "react"; import { getArchiveCatalog, + getArchiveResearchFacets, searchArchiveResearch, captureResearchCollection, type ArchiveRun, @@ -61,11 +62,13 @@ export const Route = createFileRoute("/research")({ ...(deps.cursor ? { cursor: deps.cursor } : {}), limit: 20, }; - const [catalog, research] = await Promise.all([ + const { cursor: _cursor, limit: _limit, ...faceted } = request; + const [catalog, research, facets] = await Promise.all([ getArchiveCatalog(), searchArchiveResearch({ data: request }), + getArchiveResearchFacets({ data: faceted }), ]); - return { catalog, research }; + return { catalog, research, facets }; }, component: ArchiveResearchPage, }); @@ -90,7 +93,7 @@ function evaluation(run: ArchiveRun): string { } function ArchiveResearchPage() { - const { catalog, research } = Route.useLoaderData(); + const { catalog, research, facets } = Route.useLoaderData(); const filters = Route.useSearch(); const [operatorToken, setOperatorToken] = useState(""); const [collectionTitle, setCollectionTitle] = useState(""); @@ -143,6 +146,36 @@ function ArchiveResearchPage() { if (filters.serviceTier) withoutCursor.set("serviceTier", filters.serviceTier); if (filters.cost) withoutCursor.set("cost", filters.cost); + function facetHref(field: string, value: string): string { + const params = new URLSearchParams(withoutCursor); + params.set(field, value); + return "/research?" + params.toString(); + } + + const facetGroups = [ + { key: "provider", label: "Provider", query: "provider" }, + { key: "model", label: "Model", query: "modelId" }, + { key: "test", label: "Exact Test", query: "testCaseId" }, + { key: "evidence", label: "Evidence", query: "evidence" }, + { key: "region", label: "Region", query: "region" }, + { key: "accountTier", label: "Account tier", query: "accountTier" }, + { key: "serviceTier", label: "Service tier", query: "serviceTier" }, + { key: "cost", label: "Native cost", query: "cost" }, + { key: "serviceAssurance", label: "Assurance (read-only)", query: null }, + ] as const; + + function facetLabel(key: string, value: string | null): string { + if (value === null) return "unknown / not recorded"; + if (key === "model") { + return catalog.models.find((model) => model.id === value)?.marketingName ?? value; + } + if (key === "test") { + const test = catalog.tests.find((entry) => entry.testCaseId === value); + return test ? test.familySlug + " / " + test.caseSlug : value; + } + return value; + } + const nextParams = new URLSearchParams(withoutCursor); if (research.nextCursor) nextParams.set("cursor", research.nextCursor); @@ -316,6 +349,58 @@ function ArchiveResearchPage() {
+
+
+
+

ARCHIVE v0.28 · OBSERVED FACETS

+

Evidence-backed filter counts

+
+ {facets.totalMatches} matching Runs · all pages +
+

+ Each count represents sealed public non-calibration Runs satisfying + all current filters, not just this page. Facets count + archival observations, not unique models or a quality score. + Up to {facets.perFacetLimit} values are displayed per facet, with + overflow explicitly marked. Unknown context is never inferred. +

+
+ {facetGroups.map(({ key, label, query }) => { + const facet = facets.facets[key]; + return ( +
+
+ {label} + + {facet.values.length} visible value(s) + {facet.truncated ? " · truncated" : ""} + +
+
+ {facet.values.slice(0, 8).map((entry) => { + const selectable = query !== null && entry.value !== null && + (key !== "evidence" || entry.value === "E4+" || + entry.value === "missing"); + return selectable ? ( + + {facetLabel(key, entry.value)} ({entry.count}) + + ) : ( + + {facetLabel(key, entry.value)} ({entry.count}) + + ); + })} + {!facet.values.length ? No matching observations : null} + {facet.values.length > 8 ? Showing first 8 · more in API : null} +
+
+ ); + })} +
+
+
diff --git a/docs/adr/0024-research-facets-policy-scope.md b/docs/adr/0024-research-facets-policy-scope.md new file mode 100644 index 0000000..e011ec8 --- /dev/null +++ b/docs/adr/0024-research-facets-policy-scope.md @@ -0,0 +1,26 @@ +# ADR 0024 — Research facet counts and policy assessments have explicit scopes + +## Status + +Accepted for Archive v0.28. + +## Context + +A limited UI page is not a valid basis for reporting whole-archive facet frequencies. Equally, a saved research selection cannot acquire a collective comparability verdict by applying per-Run eligibility rules; matched sets require same Test and matched contexts. + +## Decision + +1. Expose database-aggregated facets computed over the complete filtered eligible public research population, independent of pagination. +2. Count historical sealed non-calibration **Runs**, not Models or unique Tests; make this unit visible. +3. Type facet keys and response records; cap displayed values and report truncation rather than silently omit the remainder. +4. Represent missing context as null/unknown and maintain existing E4+ and cost availability semantics. +5. Evaluate saved collections' immutable Run memberships with a specifically selected append-only Comparability Policy version. +6. Reuse the existing per-Run evaluator and its historical replication/calibration rules; do not reinvent partial eligibility logic in the Web UI. +7. Label assessment `per_run_only_not_cross_provider_match`; no collective eligibility, winner or numeric ranking is inferred. +8. Keep derived policy assessments separate from collection selection digests and sourced JSON/CSV snapshots. +9. Do not allow paginated cursors to distort facet count scope. +10. Do not mutate original Run, Evaluation, qualification, calibration or research-collection records. + +## Consequences + +Research filters become explainable and auditable without conflating collection size with quality, while published snapshots can be reinterpreted under explicit policy versions without rewriting historical evidence. diff --git a/docs/archive-roadmap.md b/docs/archive-roadmap.md index 7abbaa7..34a6cea 100644 --- a/docs/archive-roadmap.md +++ b/docs/archive-roadmap.md @@ -317,9 +317,17 @@ Operators can publish bounded, immutable collections of up to 50 sealed public n Public collection pages support sharing, live filter replay and source-backed JSON/CSV exports with qualification, access, pricing, region/tier and evidence caveats. Empty, oversized and cursor-partial captures fail closed. The export is not a comparability or quality ranking. -### Archive v0.28+ — Typed Research Facets & Policy-Aware Workflows +### Archive v0.28 — Typed Research Facets & Policy-Aware Workflows -Next, consider facet discovery with observed counts, policy-version anchored research annotations and reproducible multi-page collection protocols. Explicitly distinguish current Provider policy from historical Run state; never aggregate heterogeneous Test results into a leaderboard. +**Implemented:** see `docs/research-facets-policy-annotations.md` and ADR 0024. + +Public `/research` now discovers counts from the full filtered sealed non-calibration Run population (not the current page), grouped by Provider, canonical Model, exact Test, evidence, historical region/account/service tier, service assurance and estimated/unknown native cost. Typed facets are bounded, with explicit missing values and overflow markers. + +Published immutable Research Collections can be reassessed Run-by-Run under a selected Comparability Policy version, reusing historical replication and prior same-context calibration. These interpretations never rewrite the saved Run membership or claim that the collection is a cross-Provider matched set. + +### Archive v0.29+ — Reproducible Multi-Page Research Captures + +Consider transaction-consistent snapshot capture beyond the 50-Run selection limit, plus policy-anchored structured annotation export. Keep immutable Run IDs, manifest digest verification and no-composite-ranking invariants. ## Scheduling principle diff --git a/docs/research-facets-policy-annotations.md b/docs/research-facets-policy-annotations.md new file mode 100644 index 0000000..2cb47e3 --- /dev/null +++ b/docs/research-facets-policy-annotations.md @@ -0,0 +1,93 @@ +# Typed Research Facets & Policy-Anchored Annotations — Archive v0.28 + +Archive v0.28 extends public evidence exploration without introducing cross-Test scores, unofficial leaderboards, or retroactive Run mutation. + +## Observed research facets + +Public endpoint: + +```text +GET /v1/archive/research/facets + ?provider= + &modelId= + &testCaseId= + &evidence=any|E4+|missing + ®ion= + &accountTier= + &serviceTier= + &cost=any|estimated|unknown +``` + +All filters use the same validation and historical field semantics as `/v1/archive/research`. No cursor is allowed: counts are computed over **all sealed completed public non-calibration Runs matching the entire active filter set**, not the first 20 results and not the entire unfiltered Archive unless filters are empty. + +Response: + +```json +{ + "facets": { + "scope": "sealed_public_non_calibration", + "counting": "matching_runs_all_pages", + "totalMatches": 0, + "perFacetLimit": 25, + "facets": { + "provider": { "values": [], "truncated": false }, + "model": { "values": [], "truncated": false }, + "test": { "values": [], "truncated": false }, + "evidence": { "values": [], "truncated": false }, + "region": { "values": [], "truncated": false }, + "accountTier": { "values": [], "truncated": false }, + "serviceTier": { "values": [], "truncated": false }, + "serviceAssurance": { "values": [], "truncated": false }, + "cost": { "values": [], "truncated": false } + } + } +} +``` + +Each facet value has a `value` string or `null` (missing/unknown) and an integer `count`. Top values sort by descending observed Run count then value, with at most 25 values per facet, and `truncated=true` when more exist. Unknown region, account or assurance is not treated as a valid tier or hidden. + +Evidence buckets are `E4+`, `below_E4`, and `missing`. These represent evidence availability, **not model performance**. The current filter API permits `any`, `E4+`, and `missing`; below-E4 is displayed for audit but not as a selectable filter. Cost buckets are `estimated` / `unknown`, never `free`. + +The public `/research` page displays observed counts and links to applicable facets. Counts represent **Run records, not unique Models, Tests or users**. Selecting a value applies it on top of all other active filters; a selected facet's counts are likewise already constrained by that filter. + +## Per-Run policy assessment of immutable research collections + +Public endpoint: + +```text +GET /v1/archive/research/collections/:collectionId/assessment?policyVersion= +``` + +The endpoint first verifies the saved collection's immutable Run membership and SHA-256 manifest. It then loads the requested append-only Comparability Policy, and reuses the **v0.23 per-Run evaluator** against each captured Run in stored manifest order. Results include: + +- exact `policy.version` and policy definition; +- `collectionId` and `manifestSha256`; +- `scope="per_run_only_not_cross_provider_match"`; +- each captured `runId`, `eligible/ineligible/unknown` status, machine-readable reasons; +- required/observed replication count in the Run's original execution context; +- only prior same-context calibration evidence and anomaly detail, never a later canary retroactively applied. + +The policy is selected at *interpretation time*; applying a new version to an old selection does not change the original Run, collection membership, or selection digest. The page makes the selected policy version and result scope visible. No aggregate readiness score or collection-wide matched status is calculated; to evaluate a matched cross-provider set, visitors must still use the **same-Test `/compare`** interface. + +The `/research/collections/:id` page has a user-triggered assessment action and a policy-version selector, so it does not assume a newest policy is equivalent to the policy in effect when a Run was executed. Stored JSON/CSV exports from v0.27 retain their *sourced immutable Run representation* and do not silently merge a dynamic policy assessment. + +## Provenance and invariants + +1. Facet counts are historical observations, not current Provider catalog capability or a live access-status check. +2. Facets count Runs across the whole filtered query; there is no accidental current-page sampling. +3. Missing qualification attributes remain unknown, never default or unrestricted. +4. A Run's individual eligibility is not a cross-Provider matched-set verdict. +5. Calibration evidence must predate each evaluated Run and match its frozen context. +6. Historical Run, cost, qualification, calibration and saved-selection manifest rows remain immutable. +7. All queries are bounded by a typed filter vocabulary, or by immutable collection IDs capped at 50 Runs. Distinct facet lists are bounded and explicitly marked if truncated. +8. Display order or observation volume is never a ranking or model performance score. + +## Verification + +- Public API tests assert facet count payload shape, malformed request rejection, versioned assessment metadata and immutable manifest linkage. +- PostgreSQL integration verifies total counts across two pages of historical Runs, scoped region/evidence facets, empty populations, cursor rejection, and policy-version Run assessment without modifying saved Runs. +- Standard CI validates TypeScript, unit tests, migrations, persistence/queue/provider integration and production Docker images. + +### Deferred follow-up (v0.29+) + +A future explicitly transaction-scoped, multi-page frozen collection protocol may lift the 50-Run snapshot limit; it must not silently truncate or treat a saved cursor as a complete research slice. diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index 1d47a8d..4df1591 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -7,6 +7,8 @@ import { encodeArchiveResearchCursor, validateArchiveResearchFilters, type ArchiveResearchFilters, + type ArchiveResearchFacetsView, + type ArchiveResearchFacetKey, } from "./archive-research.js"; export interface ArchiveModelView { @@ -460,6 +462,16 @@ export interface ArchiveComparisonView { }[]; } +export interface ArchiveResearchPolicyAssessmentView { + readonly policy: ComparabilityPolicyView; + readonly scope: "per_run_only_not_cross_provider_match"; + readonly collectionRunCount: number; + readonly rows: readonly { + readonly runId: string; + readonly comparability: ArchiveRunComparabilityView; + }[]; +} + export interface ArchiveResearchPageView { readonly runs: readonly ArchiveRunView[]; readonly nextCursor: string | null; @@ -1239,6 +1251,148 @@ export class PgArchiveRepository { return result.rows.map(runView); } + async researchFacets( + input: ArchiveResearchFilters = {}, + ): Promise { + const filters = validateArchiveResearchFilters(input); + // Facets cover the entire filtered archive, never merely the current page. + // Cursor/limit are not applicable to a facet count. + if (filters.cursor !== undefined) { + throw new Error("invalid_research_facet_cursor"); + } + const result = await this.pool.query<{ + total_matches: string; + facet: ArchiveResearchFacetKey | null; + value: string | null; + match_count: string | null; + position: string | null; + }>( + `WITH scope AS MATERIALIZED ( + SELECT + p.slug AS provider, + r.model_id::text AS model, + r.test_case_id::text AS test, + CASE + WHEN res.level IN ('E4', 'E5') THEN 'E4+' + WHEN res.level IS NULL THEN 'missing' + ELSE 'below_E4' + END AS evidence, + qualification.execution_region AS region, + qualification.account_tier AS account_tier, + COALESCE( + qualification.returned_service_tier, + qualification.service_tier + ) AS service_tier, + qualification.service_assurance AS service_assurance, + CASE + WHEN cost.estimated_native_cost IS NULL THEN 'unknown' + ELSE 'estimated' + END AS cost + FROM modelapse.runs r + JOIN modelapse.test_cases tc ON tc.id = r.test_case_id + LEFT JOIN modelapse.models m ON m.id = r.model_id + JOIN modelapse.providers p ON p.id = r.provider_id + LEFT JOIN modelapse.run_evidence_summary res ON res.run_id = r.id + LEFT JOIN modelapse.run_execution_qualification qualification + ON qualification.run_id = r.id + LEFT JOIN modelapse.run_cost_ledger cost ON cost.run_id = r.id + WHERE tc.visibility = 'public' + AND tc.case_type <> 'calibration' + AND r.status = 'completed' + AND r.sealed_at IS NOT NULL + AND r.completed_at IS NOT NULL + AND ($1::text IS NULL OR p.slug = $1) + AND ($2::uuid IS NULL OR r.model_id = $2) + AND ($3::uuid IS NULL OR r.test_case_id = $3) + AND ( + $4::text = 'any' OR + ($4 = 'E4+' AND res.level IN ('E4', 'E5')) OR + ($4 = 'missing' AND res.level IS NULL) + ) + AND ($5::text IS NULL OR qualification.execution_region = $5) + AND ($6::text IS NULL OR qualification.account_tier = $6) + AND ( + $7::text IS NULL OR + COALESCE( + qualification.returned_service_tier, + qualification.service_tier + ) = $7 + ) + AND ( + $8::text = 'any' OR + ($8 = 'estimated' AND cost.estimated_native_cost IS NOT NULL) OR + ($8 = 'unknown' AND cost.estimated_native_cost IS NULL) + ) + ), + counts AS ( + SELECT facet.key, facet.value, COUNT(*)::text AS match_count + FROM scope + CROSS JOIN LATERAL ( + VALUES + ('provider', scope.provider), + ('model', scope.model), + ('test', scope.test), + ('evidence', scope.evidence), + ('region', scope.region), + ('accountTier', scope.account_tier), + ('serviceTier', scope.service_tier), + ('serviceAssurance', scope.service_assurance), + ('cost', scope.cost) + ) AS facet(key, value) + GROUP BY facet.key, facet.value + ), + ranked AS ( + SELECT + key AS facet, + value, + match_count, + ROW_NUMBER() OVER ( + PARTITION BY key + ORDER BY match_count::bigint DESC, value ASC NULLS LAST + )::text AS position + FROM counts + ) + SELECT + (SELECT COUNT(*)::text FROM scope) AS total_matches, + ranked.facet, ranked.value, ranked.match_count, ranked.position + FROM (SELECT 1) AS singleton + LEFT JOIN ranked ON ranked.position::integer <= 26 + ORDER BY ranked.facet ASC, ranked.position::integer ASC`, + [ + filters.providerSlug ?? null, + filters.modelId ?? null, + filters.testCaseId ?? null, + filters.evidence, + filters.region ?? null, + filters.accountTier ?? null, + filters.serviceTier ?? null, + filters.cost, + ], + ); + const keys: readonly ArchiveResearchFacetKey[] = [ + "provider", "model", "test", "evidence", "region", + "accountTier", "serviceTier", "serviceAssurance", "cost", + ]; + const facets = {} as ArchiveResearchFacetsView["facets"]; + for (const key of keys) { + const matches = result.rows.filter((row) => row.facet === key); + facets[key] = { + values: matches.slice(0, 25).map((row) => ({ + value: row.value, + count: Number(row.match_count ?? "0"), + })), + truncated: matches.length > 25, + }; + } + return { + scope: "sealed_public_non_calibration", + counting: "matching_runs_all_pages", + totalMatches: Number(result.rows[0]?.total_matches ?? "0"), + perFacetLimit: 25, + facets, + }; + } + async researchRuns( input: ArchiveResearchFilters = {}, ): Promise { @@ -2761,6 +2915,65 @@ export class PgArchiveRepository { }; } + async assessResearchRuns( + runIds: readonly string[], + policyVersion?: string, + ): Promise { + if (runIds.length < 1 || runIds.length > 50 || + new Set(runIds).size !== runIds.length) { + throw new Error("invalid_research_run_count"); + } + if (policyVersion !== undefined && + !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion)) { + throw new Error("invalid_policy_version"); + } + + const [policy, runs] = await Promise.all([ + this.comparabilityPolicy(policyVersion), + this.getRunsByIds(runIds), + ]); + if (!policy) return null; + if (runs.length !== runIds.length) { + throw new Error("research_collection_snapshot_incomplete"); + } + + const tests = [...new Set(runs.map((run) => run.test.testCaseId))]; + const metadata = await this.pool.query<{ + id: string; + unstable: boolean; + explicit_min_repeats: string | null; + }>( + `SELECT id, + COALESCE((metadata->>'unstable')::boolean, false) AS unstable, + metadata->>'comparabilityMinRepeats' AS explicit_min_repeats + FROM modelapse.test_cases + WHERE id = ANY($1::uuid[])`, + [tests], + ); + const byTest = new Map(metadata.rows.map((row) => [row.id, row])); + + const rows = await Promise.all(runs.map(async (run) => { + const meta = byTest.get(run.test.testCaseId); + const raw = meta?.explicit_min_repeats; + const num = raw && /^\d+$/.test(raw) ? Number(raw) : null; + const explicitMinRepeats = num !== null && num >= 1 && num <= 20 + ? num : null; + return { + runId: run.id, + comparability: await this.assessRunComparability(run, policy, { + unstable: meta?.unstable ?? false, + explicitMinRepeats, + }), + }; + })); + return { + policy, + scope: "per_run_only_not_cross_provider_match", + collectionRunCount: runs.length, + rows, + }; + } + async compareLatest(input: { readonly modelIds: readonly string[]; readonly testCaseId: string; diff --git a/packages/persistence/src/archive-research.ts b/packages/persistence/src/archive-research.ts index 7de9108..d8072e1 100644 --- a/packages/persistence/src/archive-research.ts +++ b/packages/persistence/src/archive-research.ts @@ -60,6 +60,26 @@ export function encodeArchiveResearchCursor( ).toString("base64url"); } +export type ArchiveResearchFacetKey = + | "provider" | "model" | "test" | "evidence" | "region" + | "accountTier" | "serviceTier" | "cost" | "serviceAssurance"; + +export interface ArchiveResearchFacetValue { + readonly value: string | null; + readonly count: number; +} + +export interface ArchiveResearchFacetsView { + readonly scope: "sealed_public_non_calibration"; + readonly counting: "matching_runs_all_pages"; + readonly totalMatches: number; + readonly perFacetLimit: 25; + readonly facets: Record; +} + export type ArchiveResearchEvidence = "any" | "E4+" | "missing"; export type ArchiveResearchCost = "any" | "estimated" | "unknown"; diff --git a/packages/persistence/test/comparability.integration.test.ts b/packages/persistence/test/comparability.integration.test.ts index 000cfe0..937e8e5 100644 --- a/packages/persistence/test/comparability.integration.test.ts +++ b/packages/persistence/test/comparability.integration.test.ts @@ -594,6 +594,47 @@ describe("Archive comparability policy", () => { ), ).rejects.toThrow(/sealed public non-calibration Runs/i); + const facets = await archive.researchFacets({ + providerSlug: "comparability-provider-" + suffix + "-2", + testCaseId: benchmarkCase.rows[0]!.id, + }); + expect(facets).toMatchObject({ + scope: "sealed_public_non_calibration", + counting: "matching_runs_all_pages", + totalMatches: 2, + perFacetLimit: 25, + }); + expect(facets.facets.provider.values).toEqual([{ + value: "comparability-provider-" + suffix + "-2", + count: 2, + }]); + expect(facets.facets.region.values).toEqual( + expect.arrayContaining([ + { value: "US", count: 1 }, + { value: "JP", count: 1 }, + ]), + ); + expect(facets.facets.evidence.values).toEqual([ + { value: "E4+", count: 2 }, + ]); + const regionalFacets = await archive.researchFacets({ + testCaseId: benchmarkCase.rows[0]!.id, + region: "JP", + evidence: "E4+", + }); + expect(regionalFacets.totalMatches).toBe(1); + expect(regionalFacets.facets.region.values).toEqual([ + { value: "JP", count: 1 }, + ]); + const emptyFacets = await archive.researchFacets({ + providerSlug: "provider-not-present-" + suffix, + }); + expect(emptyFacets.totalMatches).toBe(0); + expect(emptyFacets.facets.provider.values).toEqual([]); + await expect( + archive.researchFacets({ cursor: research.nextCursor! }), + ).rejects.toThrow("invalid_research_facet_cursor"); + const custom = await policies.recordPolicy({ version: "comparability-integration-" + suffix, minimumEvidenceLevel: "E4", @@ -612,6 +653,29 @@ describe("Archive comparability policy", () => { }); expect(custom.version).toBe("comparability-integration-" + suffix); + const annotated = await archive.assessResearchRuns( + [research.runs[0]!.id, older.runs[0]!.id], + custom.version, + ); + expect(annotated).toMatchObject({ + scope: "per_run_only_not_cross_provider_match", + collectionRunCount: 2, + policy: { version: custom.version }, + rows: [ + { runId: research.runs[0]!.id }, + { runId: older.runs[0]!.id }, + ], + }); + expect(annotated!.rows.every((item) => + ["eligible", "unknown", "ineligible"].includes(item.comparability.status), + )).toBe(true); + expect(annotated!.rows.every((item) => + item.comparability.repeatCount >= 1, + )).toBe(true); + expect(await archive.assessResearchRuns( + [research.runs[0]!.id], "unknown-policy-" + suffix, + )).toBeNull(); + await expect( pool.query( `UPDATE modelapse.comparability_policies