Archive v0.28 — Typed Research Facets & Policy-Aware Workflows

apps/api/src/app.ts

@@ -120,6 +120,8 @@ type ArchiveRepository = Pick<
   | "listTests"
   | "listRuns"
   | "researchRuns"
+  | "researchFacets"
+  | "assessResearchRuns"
   | "listCatalogChanges"
   | "getRun"
   | "getModel"
@@ -417,6 +419,64 @@ export function createApp(deps: AppDependencies) {
     }
   });
 
+  app.get("/v1/archive/research/facets", async (c) => {
+    if (!deps.archive) return c.json({ error: "archive_unavailable" }, 503);
+    const query = c.req.query();
+    const filters: ArchiveResearchFilters = {
+      ...(query.provider !== undefined ? { providerSlug: query.provider } : {}),
+      ...(query.modelId !== undefined ? { modelId: query.modelId } : {}),
+      ...(query.testCaseId !== undefined ? { testCaseId: query.testCaseId } : {}),
+      ...(query.evidence !== undefined
+        ? { evidence: query.evidence as NonNullable<ArchiveResearchFilters["evidence"]> }
+        : {}),
+      ...(query.region !== undefined ? { region: query.region } : {}),
+      ...(query.accountTier !== undefined ? { accountTier: query.accountTier } : {}),
+      ...(query.serviceTier !== undefined ? { serviceTier: query.serviceTier } : {}),
+      ...(query.cost !== undefined
+        ? { cost: query.cost as NonNullable<ArchiveResearchFilters["cost"]> }
+        : {}),
+      ...(query.cursor !== undefined ? { cursor: query.cursor } : {}),
+    };
+    try {
+      const validated = validateArchiveResearchFilters(filters);
+      if (validated.cursor !== undefined) {
+        return c.json({ error: "invalid_research_facet_cursor" }, 400);
+      }
+      return c.json({ facets: await deps.archive.researchFacets(validated) });
+    } catch (error) {
+      if (error instanceof Error && error.message.startsWith("invalid_")) {
+        return c.json({ error: error.message }, 400);
+      }
+      throw error;
+    }
+  });
+
+  app.get("/v1/archive/research/collections/:collectionId/assessment", async (c) => {
+    if (!deps.researchCollections || !deps.archive) {
+      return c.json({ error: "research_collections_unavailable" }, 503);
+    }
+    const id = c.req.param("collectionId");
+    if (!UUID_RE.test(id)) {
+      return c.json({ error: "invalid_research_collection_id" }, 400);
+    }
+    const policyVersion = c.req.query("policyVersion");
+    if (policyVersion !== undefined && !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion)) {
+      return c.json({ error: "invalid_policy_version" }, 400);
+    }
+    const collection = await deps.researchCollections.get(id);
+    if (!collection) return c.json({ error: "research_collection_not_found" }, 404);
+    const assessment = await deps.archive.assessResearchRuns(
+      collection.runIds,
+      policyVersion,
+    );
+    if (!assessment) return c.json({ error: "comparability_policy_not_found" }, 404);
+    return c.json({
+      collectionId: collection.id,
+      manifestSha256: collection.contentSha256,
+      assessment,
+    });
+  });
+
   app.get("/v1/archive/changes", async (c) => {
     if (!deps.archive) {
       return c.json({ error: "archive_unavailable" }, 503);

apps/api/test/archive-api.test.ts

@@ -546,6 +546,38 @@ describe("Archive read API", () => {
           hasMore: false,
           scope: "sealed_public_non_calibration",
         }),
+        researchFacets: async () => ({
+          scope: "sealed_public_non_calibration" as const,
+          counting: "matching_runs_all_pages" as const,
+          totalMatches: 1,
+          perFacetLimit: 25 as const,
+          facets: {
+            provider: { values: [{ value: "deepseek", count: 1 }], truncated: false },
+            model: { values: [{ value: MODEL_ID, count: 1 }], truncated: false },
+            test: { values: [{ value: TEST_CASE_ID, count: 1 }], truncated: false },
+            evidence: { values: [{ value: "E4+", count: 1 }], truncated: false },
+            region: { values: [{ value: "US", count: 1 }], truncated: false },
+            accountTier: { values: [], truncated: false },
+            serviceTier: { values: [], truncated: false },
+            serviceAssurance: { values: [], truncated: false },
+            cost: { values: [{ value: "unknown", count: 1 }], truncated: false },
+          },
+        }),
+        assessResearchRuns: async () => ({
+          policy: archiveComparison.policy,
+          scope: "per_run_only_not_cross_provider_match" as const,
+          collectionRunCount: 1,
+          rows: [{
+            runId: RUN_ID,
+            comparability: {
+              status: "unknown" as const,
+              reasons: ["recent_calibration_missing"],
+              repeatCount: 1,
+              requiredRepeatCount: 1,
+              calibration: null,
+            },
+          }],
+        }),
         listCatalogChanges: async () => [archiveCatalogChange],
         getRun: async (runId) => (runId === RUN_ID ? archiveRun : null),
         getModel: async (modelId) =>
@@ -574,6 +606,18 @@ describe("Archive read API", () => {
       ],
     });
 
+    const facets = await app.request(
+      `/v1/archive/research/facets?provider=deepseek&region=US`,
+    );
+    expect(facets.status).toBe(200);
+    await expect(facets.json()).resolves.toMatchObject({
+      facets: {
+        scope: "sealed_public_non_calibration",
+        totalMatches: 1,
+        facets: { provider: { values: [{ value: "deepseek", count: 1 }] } },
+      },
+    });
+
     const changes = await app.request(
       `/v1/archive/changes?modelId=${MODEL_ID}&provider=deepseek&limit=10`,
     );
@@ -667,6 +711,18 @@ describe("Archive read API", () => {
           hasMore: false,
           scope: "sealed_public_non_calibration",
         }),
+        researchFacets: async () => ({
+          scope: "sealed_public_non_calibration" as const,
+          counting: "matching_runs_all_pages" as const,
+          totalMatches: 0,
+          perFacetLimit: 25 as const,
+          facets: Object.fromEntries(
+            ["provider","model","test","evidence","region","accountTier","serviceTier","serviceAssurance","cost"].map(key =>
+              [key, { values: [], truncated: false }],
+            ),
+          ) as never,
+        }),
+        assessResearchRuns: async () => null,
         listCatalogChanges: async () => [],
         getRun: async () => null,
         getModel: async () => null,
@@ -679,6 +735,12 @@ describe("Archive read API", () => {
     expect(
       (await app.request("/v1/archive/runs?modelId=nope")).status,
     ).toBe(400);
+    expect(
+      (await app.request("/v1/archive/research/facets?cursor=not-a-cursor!")).status,
+    ).toBe(400);
+    expect(
+      (await app.request("/v1/archive/research/facets?evidence=E7")).status,
+    ).toBe(400);
     for (const query of [
       "limit=51",
       "evidence=E6",
@@ -754,6 +816,44 @@ describe("Archive v0.27 immutable research collections API", () => {
     const app = createApp({
       runs: baseRuns(),
       controlToken: "control-secret",
+      archive: {
+        ping: async () => undefined,
+        listModels: async () => [],
+        listTests: async () => [],
+        listRuns: async () => [],
+        researchRuns: async () => ({
+          runs: [], nextCursor: null, hasMore: false,
+          scope: "sealed_public_non_calibration" as const,
+        }),
+        researchFacets: async () => ({
+          scope: "sealed_public_non_calibration" as const,
+          counting: "matching_runs_all_pages" as const,
+          totalMatches: 0,
+          perFacetLimit: 25 as const,
+          facets: {} as never,
+        }),
+        assessResearchRuns: async () => ({
+          policy: archiveComparison.policy,
+          scope: "per_run_only_not_cross_provider_match" as const,
+          collectionRunCount: 1,
+          rows: [{
+            runId: RUN_ID,
+            comparability: {
+              status: "unknown" as const,
+              reasons: ["recent_calibration_missing"],
+              repeatCount: 1,
+              requiredRepeatCount: 1,
+              calibration: null,
+            },
+          }],
+        }),
+        listCatalogChanges: async () => [],
+        getRun: async () => null,
+        getModel: async () => null,
+        getTest: async () => null,
+        compareLatest: async () => null,
+        getRunHistory: async () => null,
+      },
       researchCollections: {
         ping: async () => undefined,
         list: async () => [{ ...collection }],
@@ -778,6 +878,22 @@ describe("Archive v0.27 immutable research collections API", () => {
       collection: { id, runs: [{ id: RUN_ID }] },
     });
 
+    const annotated = await app.request(
+      "/v1/archive/research/collections/" + id +
+      "/assessment?policyVersion=" + archiveComparison.policy.version,
+    );
+    expect(annotated.status).toBe(200);
+    await expect(annotated.json()).resolves.toMatchObject({
+      collectionId: id,
+      manifestSha256: collection.contentSha256,
+      assessment: {
+        scope: "per_run_only_not_cross_provider_match",
+        rows: [{ runId: RUN_ID, comparability: { status: "unknown" } }],
+      },
+    });
+    expect((await app.request(
+      "/v1/archive/research/collections/" + id + "/assessment?policyVersion=BAD",
+    )).status).toBe(400);
     const json = await app.request(
       "/v1/archive/research/collections/" + id + "/export?format=json",
     );

apps/web/src/modelapse.ts

@@ -579,6 +579,31 @@ export interface ResearchCollectionDetail extends ResearchCollectionSummary {
   readonly runs: readonly ArchiveRun[];
 }
 
+export interface ArchiveResearchFacets {
+  readonly scope: "sealed_public_non_calibration";
+  readonly counting: "matching_runs_all_pages";
+  readonly totalMatches: number;
+  readonly perFacetLimit: 25;
+  readonly facets: Record<
+    "provider" | "model" | "test" | "evidence" | "region" |
+    "accountTier" | "serviceTier" | "serviceAssurance" | "cost",
+    {
+      readonly values: readonly { readonly value: string | null; readonly count: number }[];
+      readonly truncated: boolean;
+    }
+  >;
+}
+
+export interface ArchiveResearchPolicyAssessment {
+  readonly policy: ArchiveComparabilityPolicy;
+  readonly scope: "per_run_only_not_cross_provider_match";
+  readonly collectionRunCount: number;
+  readonly rows: readonly {
+    readonly runId: string;
+    readonly comparability: ArchiveRunComparability;
+  }[];
+}
+
 export interface ArchiveResearchPage {
   readonly runs: readonly ArchiveRun[];
   readonly nextCursor: string | null;
@@ -1564,6 +1589,10 @@ interface ResearchCollectionIdInput {
   readonly collectionId: string;
 }
 
+interface ResearchCollectionAssessmentInput extends ResearchCollectionIdInput {
+  readonly policyVersion: string;
+}
+
 interface ResearchCollectionExportInput extends ResearchCollectionIdInput {
   readonly format: "json" | "csv";
 }
@@ -2264,6 +2293,18 @@ function parseResearchCollectionId(value: unknown): ResearchCollectionIdInput {
   return { collectionId: value.collectionId };
 }
 
+function parseResearchCollectionAssessment(
+  value: unknown,
+): ResearchCollectionAssessmentInput {
+  const { collectionId } = parseResearchCollectionId(value);
+  if (!isRecord(value) ||
+      typeof value.policyVersion !== "string" ||
+      !/^[a-z0-9][a-z0-9._-]*$/.test(value.policyVersion)) {
+    throw new Error("Research assessment requires a valid policy version");
+  }
+  return { collectionId, policyVersion: value.policyVersion };
+}
+
 function parseResearchCollectionExport(value: unknown): ResearchCollectionExportInput {
   const { collectionId } = parseResearchCollectionId(value);
   if (!isRecord(value) ||
@@ -2424,6 +2465,39 @@ export const searchArchiveResearch = createServerFn({ method: "POST" })
     return result.research;
   });
 
+export const getArchiveResearchFacets = createServerFn({ method: "POST" })
+  .validator(parseArchiveResearchInput)
+  .handler(async ({ data }): Promise<ArchiveResearchFacets> => {
+    if (data.cursor) {
+      throw new Error("Research facets cannot use a pagination cursor");
+    }
+    const params = new URLSearchParams();
+    if (data.providerSlug) params.set("provider", data.providerSlug);
+    if (data.modelId) params.set("modelId", data.modelId);
+    if (data.testCaseId) params.set("testCaseId", data.testCaseId);
+    if (data.evidence && data.evidence !== "any") params.set("evidence", data.evidence);
+    if (data.region) params.set("region", data.region);
+    if (data.accountTier) params.set("accountTier", data.accountTier);
+    if (data.serviceTier) params.set("serviceTier", data.serviceTier);
+    if (data.cost && data.cost !== "any") params.set("cost", data.cost);
+    const result = await requestJson<{ facets: ArchiveResearchFacets }>(
+      `/v1/archive/research/facets?${params.toString()}`,
+    );
+    return result.facets;
+  });
+
+export const getResearchCollectionAssessment = createServerFn({ method: "POST" })
+  .validator(parseResearchCollectionAssessment)
+  .handler(async ({ data }): Promise<{
+    readonly collectionId: string;
+    readonly manifestSha256: string;
+    readonly assessment: ArchiveResearchPolicyAssessment;
+  }> => {
+    return requestJson(
+      `/v1/archive/research/collections/${data.collectionId}/assessment?policyVersion=${encodeURIComponent(data.policyVersion)}`,
+    );
+  });
+
 export const getResearchCollections = createServerFn({ method: "GET" }).handler(
   async (): Promise<readonly ResearchCollectionSummary[]> => {
     const result = await requestJson<{