From 302ade12ea22ad5ec718c483148246f02f04d23e Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Wed, 28 Oct 2026 15:26:22 -0700 Subject: [PATCH 01/32] feat(taxonome): integrate taxonomy revisions and evaluation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- apps/api/src/experiment-membership.ts | 3 + apps/api/src/index.ts | 44 +- apps/api/src/route-context.ts | 22 + apps/api/src/routes/experiments.ts | 45 ++ .../routes/requests/taxonomy-evaluation.ts | 104 +++ apps/api/src/routes/taxonomies.ts | 623 ++++++++++++++++++ apps/api/src/routes/taxonomy-readout.ts | 345 ++++++++++ apps/api/src/taxonomy-builtin-seeder.ts | 80 +++ apps/workers/post-processor/package.json | 2 + .../src/handlers/taxonomy-handler.test.ts | 250 +------ .../src/handlers/taxonomy-handler.ts | 409 +++++++----- .../post-processor/src/post-processor.test.ts | 2 + .../post-processor/src/post-processor.ts | 7 + apps/workers/post-processor/src/types.ts | 4 +- .../migrations/037-create-taxonomy-indexes.ts | 79 +++ .../db-migrations/src/required-migrations.ts | 1 + .../src/experiments/experiment-store.ts | 13 + packages/shared/src/experiments/schemas.ts | 22 + packages/shared/src/experiments/types.ts | 18 + packages/shared/src/index.ts | 1 + packages/shared/src/schemas/index.ts | 1 + packages/shared/src/schemas/request.ts | 10 + packages/shared/src/schemas/taxonomy.ts | 187 ++++++ packages/shared/src/storage/blob-storage.ts | 41 ++ packages/shared/src/taxonomies/index.ts | 5 + .../src/taxonomies/taxonomy-contract.test.ts | 54 ++ .../src/taxonomies/taxonomy-revision-store.ts | 114 ++++ .../shared/src/taxonomies/taxonomy-store.ts | 134 ++++ packages/shared/src/types/index.ts | 1 + packages/shared/src/types/taxonomy.ts | 101 +++ packages/shared/src/types/types.ts | 7 + packages/taxonomy-engine/src/taxonomy.ts | 123 ++-- pnpm-lock.yaml | 6 + 33 files changed, 2403 insertions(+), 455 deletions(-) create mode 100644 apps/api/src/routes/requests/taxonomy-evaluation.ts create mode 100644 apps/api/src/routes/taxonomies.ts create mode 100644 apps/api/src/routes/taxonomy-readout.ts create mode 100644 apps/api/src/taxonomy-builtin-seeder.ts create mode 100644 packages/db-migrations/src/migrations/037-create-taxonomy-indexes.ts create mode 100644 packages/shared/src/schemas/taxonomy.ts create mode 100644 packages/shared/src/taxonomies/index.ts create mode 100644 packages/shared/src/taxonomies/taxonomy-contract.test.ts create mode 100644 packages/shared/src/taxonomies/taxonomy-revision-store.ts create mode 100644 packages/shared/src/taxonomies/taxonomy-store.ts create mode 100644 packages/shared/src/types/taxonomy.ts diff --git a/apps/api/src/experiment-membership.ts b/apps/api/src/experiment-membership.ts index 99e486dc8..9c4f2f046 100644 --- a/apps/api/src/experiment-membership.ts +++ b/apps/api/src/experiment-membership.ts @@ -77,6 +77,7 @@ export class ExperimentMembershipCoordinator { metadata: ExperimentMetadata; requestIds: readonly string[]; reportTemplateIds?: readonly string[]; + taxonomy?: import("shared").ExperimentTaxonomySelection; now?: Date; }): Promise { const requestIds = uniqueCanonicalRequestIds(input.requestIds); @@ -163,7 +164,9 @@ export class ExperimentMembershipCoordinator { metadata: input.metadata, requestIds, reportTemplateIds, + taxonomy: input.taxonomy, }), + ...(input.taxonomy ? { taxonomy: input.taxonomy } : {}), createdAt: now, }, experimentExecutionId: input.experimentExecutionId, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 02bdff2a0..bd621aa2b 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,10 +9,10 @@ import { DefaultAzureCredential } from "@azure/identity"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import dotenv from "dotenv"; -import { TaskPromptStore, SkillRevisionStore, SkillResolver, CodebaseStore, CodebaseRevisionStore, CodebaseResolver, ResourceStore, ResourceRevisionStore, ResourceResolver, McpSecretClient, McpSecretUnavailableError, BlobStorage, RedisHeartbeatStore, ProjectStore, loadAuthConfigFromEnv } from "shared"; +import { TaskPromptStore, SkillRevisionStore, SkillResolver, CodebaseStore, CodebaseRevisionStore, CodebaseResolver, ResourceStore, ResourceRevisionStore, ResourceResolver, TaxonomyRootStore, TaxonomyRevisionStore, McpSecretClient, McpSecretUnavailableError, BlobStorage, RedisHeartbeatStore, ProjectStore, loadAuthConfigFromEnv } from "shared"; import type { ExperimentDocument, ExperimentExecutionHistoryDocument } from "shared"; import { initTelemetry } from "telemetry"; -import type { TaskPromptDocument, SkillDocument, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, ResourceDocument, ResourceRevisionDocument, ProfileDocument, ProfileVersionDocument, ProjectDocument, HeartbeatStore, AuthProvider, ProfileEnricher, UserDocument } from "shared"; +import type { TaskPromptDocument, SkillDocument, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, ResourceDocument, ResourceRevisionDocument, TaxonomyDocument, TaxonomyRevisionDocument, TaxonomyElementDocument, TaxonomyElementRevisionDocument, PortableTaxonomyDocument, PortableTaxonomyElementDocument, ProfileDocument, ProfileVersionDocument, ProjectDocument, HeartbeatStore, AuthProvider, ProfileEnricher, UserDocument } from "shared"; import { UserStore } from "./auth/user-store.js"; import { createAuthMiddleware, createUserAccessMiddleware } from "./auth/middleware.js"; import { authErrorHandler } from "./auth/error-handler.js"; @@ -32,6 +32,7 @@ import { registerRequestsVideoRoutes } from "./routes/requests/video.js"; import { registerRequestsToolCallsRoutes } from "./routes/requests/tool-calls.js"; import { registerRequestsSnapshotsRoutes } from "./routes/requests/snapshots.js"; import { registerRequestsArchiveRoutes } from "./routes/requests/archive.js"; +import { registerRequestsTaxonomyEvaluationRoutes } from "./routes/requests/taxonomy-evaluation.js"; import { registerCriteriaRoutes } from "./routes/criteria.js"; import { registerPromptFeaturesRoutes } from "./routes/prompt-features.js"; import { registerTaskPromptsRoutes } from "./routes/task-prompts.js"; @@ -49,6 +50,8 @@ import { registerSecretsRoutes } from "./routes/secrets.js"; import { registerProfilesRoutes } from "./routes/profiles.js"; import { registerProjectsRoutes } from "./routes/projects.js"; import { registerExperimentsRoutes } from "./routes/experiments.js"; +import { registerTaxonomyRoutes } from "./routes/taxonomies.js"; +import { registerTaxonomyReadoutRoutes } from "./routes/taxonomy-readout.js"; import { ProjectScopeError } from "./utils/project-scope.js"; import { registerUsersRoutes } from "./routes/users.js"; import { shutdownPortalCopilotClientPool } from "./copilot-client-pool.js"; @@ -144,6 +147,14 @@ let resourceRevisionCollection: Collection; let resourceStore: ResourceStore; let resourceRevisionStore: ResourceRevisionStore; let resourceResolver: ResourceResolver; +let taxonomyCollection: Collection; +let taxonomyRevisionCollection: Collection; +let taxonomyElementCollection: Collection; +let taxonomyElementRevisionCollection: Collection; +let taxonomyStore: TaxonomyRootStore; +let taxonomyRevisionStore: TaxonomyRevisionStore; +let taxonomyElementStore: TaxonomyRootStore; +let taxonomyElementRevisionStore: TaxonomyRevisionStore; let blobStorage: BlobStorage; let heartbeatStore: HeartbeatStore; let reportQueueClient: QueueClient; @@ -237,6 +248,23 @@ async function initializeClients(): Promise { resourceRevisionStore = new ResourceRevisionStore(resourceRevisionCollection, resourceStore); resourceResolver = new ResourceResolver(); + taxonomyCollection = db.collection("taxonomies"); + taxonomyRevisionCollection = db.collection("taxonomy-revisions"); + taxonomyElementCollection = db.collection("taxonomy-elements"); + taxonomyElementRevisionCollection = db.collection("taxonomy-element-revisions"); + taxonomyStore = new TaxonomyRootStore(taxonomyCollection); + taxonomyRevisionStore = new TaxonomyRevisionStore( + taxonomyRevisionCollection, + taxonomyStore, + "taxonomyId", + ); + taxonomyElementStore = new TaxonomyRootStore(taxonomyElementCollection); + taxonomyElementRevisionStore = new TaxonomyRevisionStore( + taxonomyElementRevisionCollection, + taxonomyElementStore, + "elementId", + ); + // Note: Collection indexes are managed by db-migrations (see 002-create-indexes.ts). // Run `pnpm migrate:up` to apply pending migrations. @@ -253,6 +281,7 @@ async function initializeClients(): Promise { // when ready. See apps/portal/src/lib/gates.ts (GATE_FEATURE_FLAGS). { key: "gates-run", label: "Run Gate", enabled: false }, { key: "gates-deploy", label: "Deploy Gate", enabled: false }, + { key: "taxonome", label: "Taxonome and Taxonomies", enabled: false }, ]; for (const flag of defaultFlags) { await featureFlagCollection.updateOne( @@ -344,6 +373,14 @@ const routeCtx: RouteContext = { get resourceStore() { return resourceStore; }, get resourceRevisionStore() { return resourceRevisionStore; }, get resourceResolver() { return resourceResolver; }, + get taxonomyCollection() { return taxonomyCollection; }, + get taxonomyRevisionCollection() { return taxonomyRevisionCollection; }, + get taxonomyElementCollection() { return taxonomyElementCollection; }, + get taxonomyElementRevisionCollection() { return taxonomyElementRevisionCollection; }, + get taxonomyStore() { return taxonomyStore; }, + get taxonomyRevisionStore() { return taxonomyRevisionStore; }, + get taxonomyElementStore() { return taxonomyElementStore; }, + get taxonomyElementRevisionStore() { return taxonomyElementRevisionStore; }, get projectStore() { return projectStore; }, get reportQueueClient() { return reportQueueClient; }, get blobStorage() { return blobStorage; }, @@ -371,6 +408,8 @@ registerSecretsRoutes(routeCtx); registerSystemRoutes(routeCtx); registerProjectsRoutes(routeCtx); registerExperimentsRoutes(routeCtx); +registerTaxonomyRoutes(routeCtx); +registerTaxonomyReadoutRoutes(routeCtx); registerRequestsRoutes(routeCtx); registerRequestsCancelRoutes(routeCtx); registerRequestsLogsRoutes(routeCtx); @@ -380,6 +419,7 @@ registerRequestsVideoRoutes(routeCtx); registerRequestsToolCallsRoutes(routeCtx); registerRequestsSnapshotsRoutes(routeCtx); registerRequestsArchiveRoutes(routeCtx); +registerRequestsTaxonomyEvaluationRoutes(routeCtx); registerCriteriaRoutes(routeCtx); registerPromptFeaturesRoutes(routeCtx); registerTaskPromptsRoutes(routeCtx); diff --git a/apps/api/src/route-context.ts b/apps/api/src/route-context.ts index 43e3db777..62722a11f 100644 --- a/apps/api/src/route-context.ts +++ b/apps/api/src/route-context.ts @@ -55,6 +55,12 @@ import type { RunHistoryDocumentSchema, ExperimentDocument, ExperimentExecutionHistoryDocument, + TaxonomyDocument, + TaxonomyElementDocument, + TaxonomyRevisionDocument, + TaxonomyElementRevisionDocument, + TaxonomyRootStore, + TaxonomyRevisionStore, } from "shared"; // ─── Document types (inferred from Zod schemas) ───────────────────────────── @@ -118,6 +124,10 @@ export interface RouteContext { codebaseRevisionCollection: Collection; resourceCollection: Collection; resourceRevisionCollection: Collection; + taxonomyCollection: Collection; + taxonomyRevisionCollection: Collection; + taxonomyElementCollection: Collection; + taxonomyElementRevisionCollection: Collection; // Services taskPromptStore: TaskPromptStore; @@ -129,6 +139,18 @@ export interface RouteContext { resourceStore: ResourceStore; resourceRevisionStore: ResourceRevisionStore; resourceResolver: ResourceResolver; + taxonomyStore: TaxonomyRootStore; + taxonomyRevisionStore: TaxonomyRevisionStore< + TaxonomyRevisionDocument, + TaxonomyDocument, + import("shared").PortableTaxonomyDocument + >; + taxonomyElementStore: TaxonomyRootStore; + taxonomyElementRevisionStore: TaxonomyRevisionStore< + TaxonomyElementRevisionDocument, + TaxonomyElementDocument, + import("shared").PortableTaxonomyElementDocument + >; projectStore: ProjectStore; // Token Manager client (null when TOKEN_MANAGER_URL not set) diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts index be3d5a431..8ba90da43 100644 --- a/apps/api/src/routes/experiments.ts +++ b/apps/api/src/routes/experiments.ts @@ -61,6 +61,7 @@ const CreateExperimentInputSchema = z requests: z.array(ExperimentRequestInputSchema).max(250_000).default([]), requestIds: z.array(z.string().min(1)).max(250_000).default([]), reportTemplateIds: z.array(z.string().min(1)).max(1_000).default([]), + taxonomyRevisionId: z.string().min(1).optional(), }) .superRefine((value, ctx) => { if ( @@ -166,6 +167,13 @@ const ExperimentResponseSchema = z revisionNumber: z.number().int(), memberCount: z.number().int(), reportTemplateIds: z.array(z.string()), + taxonomy: z + .object({ + taxonomyRevisionId: z.string(), + ref: z.string(), + contentSha256: z.string(), + }) + .optional(), createdAt: z.coerce.date(), }), currentExecution: z.object({ @@ -683,6 +691,7 @@ function experimentResponse(experiment: ExperimentDocument) { revisionNumber: experiment.revision.revisionNumber, memberCount: experiment.revision.memberCount, reportTemplateIds: experiment.revision.reportTemplateIds, + taxonomy: experiment.revision.taxonomy, createdAt: experiment.revision.createdAt, }, currentExecution: executionResponse(experiment.execution), @@ -827,6 +836,39 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { const reportTemplateIds = canonicalTemplateIds( req.body.reportTemplateIds, ); + let taxonomy: + | { + taxonomyRevisionId: string; + ref: string; + contentSha256: string; + } + | undefined; + if (req.body.taxonomyRevisionId) { + const enabled = await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }); + if (!enabled) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const revision = await ctx.taxonomyRevisionStore.get( + req.body.taxonomyRevisionId, + ); + if (!revision) { + res.status(400).json({ error: "Taxonomy revision not found" }); + return; + } + if (revision.projectId !== projectId) { + res.status(400).json({ error: "Taxonomy revision project mismatch" }); + return; + } + taxonomy = { + taxonomyRevisionId: revision._id, + ref: revision.ref, + contentSha256: revision.contentSha256, + }; + } const invalidReportTemplateIds = await invalidManualExperimentTemplateIds( ctx, @@ -854,6 +896,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { metadata: req.body.metadata, requestIds: req.body.requestIds, reportTemplateIds, + taxonomy, now, }); res.status(201).json({ @@ -918,7 +961,9 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { metadata, requests: req.body.requests, reportTemplateIds, + taxonomy, }), + ...(taxonomy ? { taxonomy } : {}), createdAt: now, }, experimentExecutionId, diff --git a/apps/api/src/routes/requests/taxonomy-evaluation.ts b/apps/api/src/routes/requests/taxonomy-evaluation.ts new file mode 100644 index 000000000..ba55ee2bb --- /dev/null +++ b/apps/api/src/routes/requests/taxonomy-evaluation.ts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { apiRoute } from "../../openapi/api-route.js"; +import type { RouteContext } from "../../route-context.js"; +import { resolveRunForRequest } from "./resolve-run.js"; + +const TaxonomyEvaluationReferenceSchema = z + .object({ + url: z.string(), + schemaVersion: z.string(), + contentSha256: z.string(), + sizeBytes: z.number().int().nonnegative(), + }) + .openapi("TaxonomyEvaluationReference"); + +async function enabled(ctx: RouteContext): Promise { + return Boolean( + await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }), + ); +} + +/** Registers exact-run Taxonome result metadata and stable blob proxy routes. */ +export function registerRequestsTaxonomyEvaluationRoutes( + ctx: RouteContext, +): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/requests/:id/runs/:runId/taxonomy-evaluation", + tags: ["Requests"], + summary: "Get the exact-run Taxonome result reference", + response: TaxonomyEvaluationReferenceSchema, + handler: async (req, res, next) => { + try { + if (!(await enabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const resolved = await resolveRunForRequest( + ctx, + req.params.id, + req.params.runId, + ); + if ("error" in resolved) { + res.status(resolved.status).json({ error: resolved.error }); + return; + } + if (!resolved.run.taxonomyEvaluation) { + res.status(404).json({ error: "Taxonomy evaluation not available" }); + return; + } + res.json(resolved.run.taxonomyEvaluation); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/requests/:id/runs/:runId/taxonomy-evaluation/content", + tags: ["Requests"], + summary: "Download the exact-run compressed Taxonome result", + response: z.any(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await enabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const resolved = await resolveRunForRequest( + ctx, + req.params.id, + req.params.runId, + ); + if ("error" in resolved) { + res.status(resolved.status).json({ error: resolved.error }); + return; + } + const reference = resolved.run.taxonomyEvaluation; + if (!reference) { + res.status(404).json({ error: "Taxonomy evaluation not available" }); + return; + } + const body = await ctx.blobStorage.downloadBuffer(reference.url); + res + .type("application/json") + .set("Content-Encoding", "gzip") + .set( + "Content-Disposition", + `attachment; filename="taxonomy-${req.params.runId}.json.gz"`, + ) + .send(body); + } catch (error) { + next(error); + } + }, + }); +} diff --git a/apps/api/src/routes/taxonomies.ts b/apps/api/src/routes/taxonomies.ts new file mode 100644 index 000000000..b4504ebef --- /dev/null +++ b/apps/api/src/routes/taxonomies.ts @@ -0,0 +1,623 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; +import { + CopyBuiltinTaxonomyRevisionInputSchema, + CreateTaxonomyElementInputSchema, + CreateTaxonomyElementRevisionInputSchema, + CreateTaxonomyInputSchema, + CreateTaxonomyRevisionInputSchema, + PortableTaxonomyElementSchema, + PortableTaxonomySchema, + TaxonomyElementResponseSchema, + TaxonomyElementRevisionResponseSchema, + TaxonomyResponseSchema, + TaxonomyRevisionResponseSchema, + UpdateTaxonomyMetadataInputSchema, + type PortableTaxonomyDocument, + type TaxonomyDocument, + type TaxonomyElementDocument, +} from "shared"; +import { apiRoute } from "../openapi/api-route.js"; +import type { RouteContext } from "../route-context.js"; +import { + ProjectIdQuerySchema, + getQueryProjectId, +} from "../utils/project-scope.js"; +import { ensureBuiltinTaxonomy } from "../taxonomy-builtin-seeder.js"; + +const ImportTaxonomyYamlInputSchema = z + .object({ + name: z.string().trim().min(1).max(256), + description: z.string().max(4096).optional(), + creator: z.string().optional(), + taxonomyYaml: z.string().min(1), + elements: z.record(z.string(), z.string()).default({}), + }) + .strict() + .openapi("ImportTaxonomyYamlInput"); + +/** Taxonome entry points are hidden unless the operator explicitly enables them. */ +async function requireTaxonome(ctx: RouteContext, response: { + status(code: number): { json(body: unknown): void }; +}): Promise { + const flag = await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }); + if (!flag) { + response.status(404).json({ error: "Taxonome is not enabled" }); + return false; + } + return true; +} + +function assertUserRoot( + root: TaxonomyDocument | TaxonomyElementDocument, +): void { + if (root.sourceType !== "user") { + throw new Error("Built-in taxonomy content is read-only"); + } +} + +function portableTaxonomyRevision(revision: { + schema_version: "scope/taxonomy/v1"; + id: string; + version: string; + surfaces: PortableTaxonomyDocument["surfaces"]; + elements: PortableTaxonomyDocument["elements"]; +}): PortableTaxonomyDocument { + return { + schema_version: revision.schema_version, + id: revision.id, + version: revision.version, + surfaces: revision.surfaces, + elements: revision.elements, + }; +} + +/** Registers the complete project-scoped taxonomy and element product API. */ +export function registerTaxonomyRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-elements", + tags: ["Taxonomies"], + summary: "List project taxonomy elements", + query: ProjectIdQuerySchema, + response: z.array(TaxonomyElementResponseSchema), + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + await ensureBuiltinTaxonomy(ctx, getQueryProjectId(req)); + res.json( + await ctx.taxonomyElementStore.list(getQueryProjectId(req)), + ); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomy-elements", + tags: ["Taxonomies"], + summary: "Create a user taxonomy element and revision 1", + query: ProjectIdQuerySchema, + body: CreateTaxonomyElementInputSchema, + response: TaxonomyElementResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + if (req.body.sourceType !== "user") { + res.status(400).json({ error: "Only user taxonomy elements can be created" }); + return; + } + if (req.body.id && req.body.id !== req.body.content.id) { + res.status(400).json({ error: "Root id must match content id" }); + return; + } + const root = await ctx.taxonomyElementStore.create({ + projectId: getQueryProjectId(req), + id: req.body.content.id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyElementRevisionStore.create( + root._id, + req.body.content, + req.body.creator, + ); + res + .status(201) + .json((await ctx.taxonomyElementStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomies", + tags: ["Taxonomies"], + summary: "List project taxonomies", + query: ProjectIdQuerySchema, + response: z.array(TaxonomyResponseSchema), + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + await ensureBuiltinTaxonomy(ctx, getQueryProjectId(req)); + res.json( + await ctx.taxonomyStore.list(getQueryProjectId(req)), + ); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-element-revisions/:revisionId/export", + tags: ["Taxonomies"], + summary: "Export an exact taxonomy element revision as portable YAML", + response: z.string(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyElementRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy element revision not found" }); + return; + } + const { + _id: _id, + elementId: _elementId, + projectId: _projectId, + sourceType: _sourceType, + revision: _revision, + ref: _ref, + contentSha256: _contentSha256, + creator: _creator, + createdAt: _createdAt, + ...portable + } = revision; + res.type("application/yaml").send(stringifyYaml(portable)); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomies", + tags: ["Taxonomies"], + summary: "Create a user taxonomy and revision 1", + query: ProjectIdQuerySchema, + body: CreateTaxonomyInputSchema, + response: TaxonomyResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + if (req.body.sourceType !== "user") { + res.status(400).json({ error: "Only user taxonomies can be created" }); + return; + } + if (req.body.id && req.body.id !== req.body.content.id) { + res.status(400).json({ error: "Root id must match content id" }); + return; + } + const root = await ctx.taxonomyStore.create({ + projectId: getQueryProjectId(req), + id: req.body.content.id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyRevisionStore.create( + root._id, + req.body.content, + req.body.creator, + ); + res.status(201).json((await ctx.taxonomyStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomies/import", + tags: ["Taxonomies"], + summary: "Import taxonomy YAML and referenced element YAML", + query: ProjectIdQuerySchema, + body: ImportTaxonomyYamlInputSchema, + response: TaxonomyResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const taxonomy = PortableTaxonomySchema.parse( + parseYaml(req.body.taxonomyYaml), + ); + const projectId = getQueryProjectId(req); + const refs: Array<{ ref: string }> = []; + const ids = new Set(); + + for (const reference of taxonomy.elements) { + if ("ref" in reference) { + const resolved = await ctx.taxonomyElementRevisionStore.getByRef( + projectId, + reference.ref, + ); + if (!resolved) { + res.status(400).json({ + error: `Element revision '${reference.ref}' was not found in this project`, + }); + return; + } + if (ids.has(resolved.id)) { + res.status(400).json({ error: `Duplicate element id '${resolved.id}'` }); + return; + } + ids.add(resolved.id); + refs.push({ ref: reference.ref }); + continue; + } + + const yaml = req.body.elements[reference.path]; + if (!yaml) { + res.status(400).json({ + error: `Missing element file '${reference.path}'`, + }); + return; + } + const content = PortableTaxonomyElementSchema.parse(parseYaml(yaml)); + if (ids.has(content.id)) { + res.status(400).json({ error: `Duplicate element id '${content.id}'` }); + return; + } + ids.add(content.id); + const element = await ctx.taxonomyElementStore.create({ + projectId, + id: content.id, + name: content.label, + ...(content.description ? { description: content.description } : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + const created = await ctx.taxonomyElementRevisionStore.create( + element._id, + content, + req.body.creator, + ); + refs.push({ ref: created.revision.ref }); + } + + const root = await ctx.taxonomyStore.create({ + projectId, + id: taxonomy.id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyRevisionStore.create( + root._id, + { ...taxonomy, elements: refs }, + req.body.creator, + ); + res.status(201).json((await ctx.taxonomyStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); + + for (const operation of [ + { path: "/api/v1/taxonomies/:id", element: false }, + { path: "/api/v1/taxonomy-elements/:id", element: true }, + ] as const) { + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const responseSchema = operation.element + ? TaxonomyElementResponseSchema + : TaxonomyResponseSchema; + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: operation.path, + tags: ["Taxonomies"], + summary: `Get a taxonomy${operation.element ? " element" : ""}`, + response: responseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const root = await rootStore.get(req.params.id); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + res.json(root); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "patch", + path: operation.path, + tags: ["Taxonomies"], + summary: `Update taxonomy${operation.element ? " element" : ""} metadata`, + body: UpdateTaxonomyMetadataInputSchema, + response: responseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const root = await rootStore.get(req.params.id); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + assertUserRoot(root); + res.json(await rootStore.updateMetadata(req.params.id, req.body)); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "delete", + path: operation.path, + tags: ["Taxonomies"], + summary: `Soft-delete a taxonomy${operation.element ? " element" : ""}`, + response: z.any(), + rawResponse: true, + successStatus: 204, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + if (!(await rootStore.softDelete(req.params.id))) { + res.status(404).json({ error: "User taxonomy root not found" }); + return; + } + res.status(204).send(); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: `${operation.path}/restore`, + tags: ["Taxonomies"], + summary: `Restore a taxonomy${operation.element ? " element" : ""}`, + response: responseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const restored = await rootStore.restore(req.params.id); + if (!restored) { + res.status(404).json({ error: "Deleted user taxonomy root not found" }); + return; + } + res.json(restored); + } catch (error) { + next(error); + } + }, + }); + } + + for (const operation of [ + { path: "/api/v1/taxonomies/:id/revisions", element: false }, + { path: "/api/v1/taxonomy-elements/:id/revisions", element: true }, + ] as const) { + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const revisionStore = operation.element + ? ctx.taxonomyElementRevisionStore + : ctx.taxonomyRevisionStore; + const bodySchema = operation.element + ? CreateTaxonomyElementRevisionInputSchema + : CreateTaxonomyRevisionInputSchema; + const responseSchema = operation.element + ? TaxonomyElementRevisionResponseSchema + : TaxonomyRevisionResponseSchema; + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: operation.path, + tags: ["Taxonomies"], + summary: `List taxonomy${operation.element ? " element" : ""} revisions`, + response: z.array(responseSchema), + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const root = await rootStore.get(req.params.id, true); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + res.json(await revisionStore.list(root._id)); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: operation.path, + tags: ["Taxonomies"], + summary: `Create taxonomy${operation.element ? " element" : ""} revision`, + body: bodySchema, + response: responseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const root = await rootStore.get(req.params.id); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + assertUserRoot(root); + if (req.body.content.id !== root.id) { + res.status(400).json({ error: "Revision id must match root id" }); + return; + } + const result = await revisionStore.create( + root._id, + req.body.content as never, + req.body.creator, + ); + res.status(result.deduplicated ? 200 : 201).json(result.revision); + } catch (error) { + next(error); + } + }, + }); + } + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-revisions/:revisionId", + tags: ["Taxonomies"], + summary: "Get an exact taxonomy revision", + response: TaxonomyRevisionResponseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy revision not found" }); + return; + } + res.json(revision); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-element-revisions/:revisionId", + tags: ["Taxonomies"], + summary: "Get an exact taxonomy element revision", + response: TaxonomyElementRevisionResponseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyElementRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy element revision not found" }); + return; + } + res.json(revision); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-revisions/:revisionId/export", + tags: ["Taxonomies"], + summary: "Export an exact taxonomy revision as portable YAML", + response: z.string(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy revision not found" }); + return; + } + res.type("application/yaml").send( + stringifyYaml(portableTaxonomyRevision(revision)), + ); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomy-revisions/:revisionId/copy", + tags: ["Taxonomies"], + summary: "Copy an exact built-in revision to user revision 1", + query: ProjectIdQuerySchema, + body: CopyBuiltinTaxonomyRevisionInputSchema, + response: TaxonomyResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const source = await ctx.taxonomyRevisionStore.get(req.params.revisionId); + if (!source || source.sourceType !== "builtin") { + res.status(404).json({ error: "Built-in taxonomy revision not found" }); + return; + } + if (source.projectId !== getQueryProjectId(req)) { + res.status(400).json({ error: "Taxonomy revision project mismatch" }); + return; + } + const content = portableTaxonomyRevision(source); + const id = req.body.id ?? `${source.id}-copy`; + const root = await ctx.taxonomyStore.create({ + projectId: source.projectId, + id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyRevisionStore.create( + root._id, + { ...content, id }, + req.body.creator, + ); + res.status(201).json((await ctx.taxonomyStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); +} diff --git a/apps/api/src/routes/taxonomy-readout.ts b/apps/api/src/routes/taxonomy-readout.ts new file mode 100644 index 000000000..608ca8cd6 --- /dev/null +++ b/apps/api/src/routes/taxonomy-readout.ts @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID } from "node:crypto"; +import { promisify } from "node:util"; +import { gunzip } from "node:zlib"; +import { z } from "zod"; +import { ExperimentStore, type ExperimentRequestState } from "shared"; +import { apiRoute } from "../openapi/api-route.js"; +import type { RouteContext } from "../route-context.js"; +import { + ProjectIdQuerySchema, + getQueryProjectId, +} from "../utils/project-scope.js"; +import { resolveRunForRequest } from "./requests/resolve-run.js"; + +const gunzipAsync = promisify(gunzip); + +const TaxonomyReadoutReferenceSchema = z + .object({ + status: z.enum(["processing", "done", "failed"]), + experimentExecutionId: z.string(), + taxonomyRevisionId: z.string(), + jsonUrl: z.string().optional(), + markdownUrl: z.string().optional(), + updatedAt: z.coerce.date(), + error: z.string().optional(), + }) + .openapi("TaxonomyReadoutReference"); + +function experimentStore(ctx: RouteContext): ExperimentStore { + return new ExperimentStore({ + experiments: ctx.experimentCollection, + executions: ctx.experimentExecutionsCollection, + chunks: ctx.experimentChunksCollection, + }); +} + +async function requireEnabled(ctx: RouteContext): Promise { + return Boolean( + await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }), + ); +} + +function renderMarkdown(readout: { + experimentId: string; + experimentExecutionId: string; + taxonomyRef: string; + runs: Array<{ requestId: string; runId: string; state: string }>; +}): string { + const counts = new Map(); + for (const run of readout.runs) { + counts.set(run.state, (counts.get(run.state) ?? 0) + 1); + } + const lines = [ + "# Taxonome readout", + "", + `- Experiment: \`${readout.experimentId}\``, + `- Execution: \`${readout.experimentExecutionId}\``, + `- Taxonomy: \`${readout.taxonomyRef}\``, + `- Runs: ${readout.runs.length}`, + "", + "## States", + "", + ...[...counts.entries()].map(([state, count]) => `- ${state}: ${count}`), + "", + "## Exact runs", + "", + "| Request | Run | State |", + "|---|---|---|", + ...readout.runs.map( + (run) => `| \`${run.requestId}\` | \`${run.runId}\` | ${run.state} |`, + ), + "", + ]; + return lines.join("\n"); +} + +async function buildReadout( + ctx: RouteContext, + input: { + projectId: string; + experimentId: string; + executionId: string; + requests: ExperimentRequestState[]; + taxonomy: { + taxonomyRevisionId: string; + ref: string; + contentSha256: string; + }; + }, +) { + const runs: Array> = []; + for (const pair of input.requests) { + const resolved = await resolveRunForRequest( + ctx, + pair.requestId, + pair.runId, + ); + if ("error" in resolved) { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "missing", + }); + continue; + } + const handler = resolved.run.handlerStatus?.["pp-taxonomy"]; + if (handler?.status === "failed") { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "runtime-failure", + error: handler.error, + }); + continue; + } + const reference = resolved.run.taxonomyEvaluation; + if (!reference) { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "missing", + }); + continue; + } + const compressed = await ctx.blobStorage.downloadBuffer(reference.url); + const body = await gunzipAsync(compressed); + const hash = createHash("sha256").update(body).digest("hex"); + if (hash !== reference.contentSha256) { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "runtime-failure", + error: "Taxonomy evaluation integrity check failed", + }); + continue; + } + const result: unknown = JSON.parse(body.toString("utf8")); + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "completed", + result, + }); + } + return { + schemaVersion: "scope/taxonomy-experiment-readout/v1", + projectId: input.projectId, + experimentId: input.experimentId, + experimentExecutionId: input.executionId, + taxonomy: input.taxonomy, + runs, + generatedAt: new Date().toISOString(), + }; +} + +/** Registers typed Taxonome readout status, generation, content, and download routes. */ +export function registerTaxonomyReadoutRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/taxonomy-readout", + tags: ["Experiments"], + summary: "Get typed Taxonome readout status", + query: ProjectIdQuerySchema, + response: TaxonomyReadoutReferenceSchema, + handler: async (req, res, next) => { + try { + if (!(await requireEnabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const experiment = await experimentStore(ctx).get( + getQueryProjectId(req), + req.params.id, + ); + if (!experiment?.execution.taxonomyReadout) { + res.status(404).json({ error: "Taxonome readout not available" }); + return; + } + res.json(experiment.execution.taxonomyReadout); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/experiments/:id/taxonomy-readout/retry", + tags: ["Experiments"], + summary: "Generate or retry the typed Taxonome readout", + query: ProjectIdQuerySchema, + response: TaxonomyReadoutReferenceSchema, + successStatus: 202, + handler: async (req, res, next) => { + const projectId = getQueryProjectId(req); + const store = experimentStore(ctx); + const claimToken = randomUUID(); + try { + if (!(await requireEnabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const experiment = await store.get(projectId, req.params.id); + const taxonomy = experiment?.execution.taxonomy; + if (!experiment || !taxonomy) { + res.status(404).json({ error: "Taxonomy-enabled Experiment not found" }); + return; + } + const claim = await ctx.experimentCollection.findOneAndUpdate( + { + _id: experiment._id, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + }, + { + $set: { + "execution.taxonomyReadout": { + status: "processing", + claimToken, + experimentExecutionId: + experiment.execution.experimentExecutionId, + taxonomyRevisionId: taxonomy.taxonomyRevisionId, + updatedAt: new Date(), + }, + }, + $inc: { "execution.version": 1 }, + }, + { returnDocument: "after" }, + ); + if (!claim) { + res.status(409).json({ error: "Experiment execution changed" }); + return; + } + const requests = await store.readExecutionRequests({ + projectId, + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + }); + if (!requests) throw new Error("Experiment request vector is unavailable"); + const readout = await buildReadout(ctx, { + projectId, + experimentId: experiment.experimentId, + executionId: experiment.execution.experimentExecutionId, + requests, + taxonomy, + }); + const prefix = + `${experiment.experimentId}/executions/` + + `${experiment.execution.experimentExecutionId}/taxonomy-readout/${claimToken}`; + const jsonUrl = await ctx.blobStorage.uploadJson( + `${prefix}/readout.json`, + readout, + ); + const markdownUrl = await ctx.blobStorage.uploadText( + `${prefix}/readout.md`, + renderMarkdown(readout as never), + "text/markdown; charset=utf-8", + ); + const completed = { + status: "done" as const, + experimentExecutionId: experiment.execution.experimentExecutionId, + taxonomyRevisionId: taxonomy.taxonomyRevisionId, + jsonUrl, + markdownUrl, + updatedAt: new Date(), + }; + const publication = await ctx.experimentCollection.updateOne( + { + _id: experiment._id, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.taxonomyReadout.claimToken": claimToken, + }, + { $set: { "execution.taxonomyReadout": completed } }, + ); + if (publication.matchedCount !== 1) { + res.status(409).json({ error: "Experiment execution changed" }); + return; + } + res.status(202).json(completed); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await ctx.experimentCollection.updateOne( + { + _id: req.params.id, + projectId, + "execution.taxonomyReadout.claimToken": claimToken, + }, + { + $set: { + "execution.taxonomyReadout.status": "failed", + "execution.taxonomyReadout.error": message, + "execution.taxonomyReadout.updatedAt": new Date(), + }, + }, + ); + next(error); + } + }, + }); + + for (const format of ["json", "markdown"] as const) { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: `/api/v1/experiments/:id/taxonomy-readout/${format}`, + tags: ["Experiments"], + summary: `Download Taxonome readout ${format}`, + query: ProjectIdQuerySchema, + response: z.any(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await requireEnabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const experiment = await experimentStore(ctx).get( + getQueryProjectId(req), + req.params.id, + ); + const reference = experiment?.execution.taxonomyReadout; + const url = format === "json" ? reference?.jsonUrl : reference?.markdownUrl; + if (!url) { + res.status(404).json({ error: "Taxonome readout not available" }); + return; + } + const body = await ctx.blobStorage.downloadBuffer(url); + res + .type(format === "json" ? "application/json" : "text/markdown") + .send(body); + } catch (error) { + next(error); + } + }, + }); + } +} diff --git a/apps/api/src/taxonomy-builtin-seeder.ts b/apps/api/src/taxonomy-builtin-seeder.ts new file mode 100644 index 000000000..024c61091 --- /dev/null +++ b/apps/api/src/taxonomy-builtin-seeder.ts @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { + PortableTaxonomyElementSchema, + PortableTaxonomySchema, +} from "shared"; +import type { RouteContext } from "./route-context.js"; + +const CATALOG_DIRECTORY = fileURLToPath( + new URL("../../../packages/ax-taxonomy/taxonomy/", import.meta.url), +); + +/** Idempotently seeds the canonical Default Taxonomy into one project. */ +export async function ensureBuiltinTaxonomy( + ctx: RouteContext, + projectId: string, +): Promise { + const taxonomyPath = resolve(CATALOG_DIRECTORY, "taxonomy.yaml"); + const taxonomy = PortableTaxonomySchema.parse( + parseYaml(await readFile(taxonomyPath, "utf8")), + ); + const existing = await ctx.taxonomyStore.getByBusinessId( + projectId, + taxonomy.id, + true, + ); + if (existing) return; + + const elementRefs: Array<{ ref: string }> = []; + for (const reference of taxonomy.elements) { + if (!("path" in reference)) { + throw new Error("Canonical built-in taxonomy must use local element paths"); + } + const element = PortableTaxonomyElementSchema.parse( + parseYaml( + await readFile(resolve(dirname(taxonomyPath), reference.path), "utf8"), + ), + ); + const existingElement = await ctx.taxonomyElementStore.getByBusinessId( + projectId, + element.id, + true, + ); + const root = + existingElement ?? + (await ctx.taxonomyElementStore.create({ + projectId, + id: element.id, + name: element.label, + ...(element.description ? { description: element.description } : {}), + sourceType: "builtin", + creator: "system", + })); + const revision = await ctx.taxonomyElementRevisionStore.create( + root._id, + element, + "system", + ); + elementRefs.push({ ref: revision.revision.ref }); + } + + const root = await ctx.taxonomyStore.create({ + projectId, + id: taxonomy.id, + name: "Default Taxonomy", + description: "Canonical built-in Taxonome taxonomy", + sourceType: "builtin", + creator: "system", + }); + await ctx.taxonomyRevisionStore.create( + root._id, + { ...taxonomy, elements: elementRefs }, + "system", + ); +} diff --git a/apps/workers/post-processor/package.json b/apps/workers/post-processor/package.json index 95eb93f39..b1b6d096d 100644 --- a/apps/workers/post-processor/package.json +++ b/apps/workers/post-processor/package.json @@ -12,6 +12,8 @@ "dependencies": { "shared": "workspace:*", "telemetry": "workspace:*", + "taxonomy-engine": "workspace:*", + "ax-taxonomy": "workspace:*", "@azure/storage-queue": "^12.25.0", "atifact": "^0.10.1", "dotenv": "^16.4.5", diff --git a/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts b/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts index 1f191caa9..e779da4cf 100644 --- a/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts +++ b/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts @@ -1,230 +1,36 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { JudgeInfrastructureError, type JudgeClient } from "shared"; +import { describe, expect, it, vi } from "vitest"; import { TaxonomyHandler } from "./taxonomy-handler.js"; -import type { HandlerContext, PostProcessorMessage } from "../types.js"; - -interface CriterionMeta { - id: string; - subject?: "run" | "iteration"; -} - -function makeCtx( - doc: unknown, - metas: CriterionMeta[] = [], -): HandlerContext & { collection: { updateOne: ReturnType } } { - const scopedDoc = - typeof doc === "object" && doc !== null - ? { projectId: "proj-test", ...doc } - : doc; - return { - blobStorage: {} as any, - collection: { - findOne: vi.fn().mockResolvedValue(scopedDoc), - updateOne: vi.fn().mockResolvedValue({ modifiedCount: 1 }), - } as any, - criteriaCollection: { - find: vi.fn().mockReturnValue({ toArray: vi.fn().mockResolvedValue(metas) }), - } as any, - log: vi.fn().mockResolvedValue(undefined), - } as any; -} - -const msg: PostProcessorMessage = { type: "taxonomy", requestId: "req-1", runId: "run-1" }; describe("TaxonomyHandler", () => { - let judge: { evaluate: ReturnType }; - - beforeEach(() => { - judge = { evaluate: vi.fn() }; - }); - - it("has type 'taxonomy'", () => { - expect(new TaxonomyHandler(judge as unknown as JudgeClient).type).toBe("taxonomy"); - }); - - it("no-ops when no observations are selected", async () => { - const ctx = makeCtx({ _id: "req-1", observations: [], run: { turns: [{ iteration: 1, snapshotUrl: "s://1" }] } }); - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - expect(judge.evaluate).not.toHaveBeenCalled(); - expect(ctx.collection.updateOne).not.toHaveBeenCalled(); - }); - - it("reads observations from the request top-level, not run.observations (#1156 regression)", async () => { - // The API persists selected observation ids at the request top-level - // (RequestDocument.observations). A doc that only carries the legacy - // run.observations must be treated as "none selected" → no-op, proving the - // handler reads the canonical location the submit path actually writes. - const ctx = makeCtx({ - _id: "req-1", - run: { observations: ["dep_removed"], turns: [{ iteration: 1, snapshotUrl: "s://1", atifUrl: "a://1" }] }, - }); - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - expect(judge.evaluate).not.toHaveBeenCalled(); - expect(ctx.collection.updateOne).not.toHaveBeenCalled(); - }); - - describe("subject:iteration", () => { - it("writes judge criteriaResults to each turn's observationResults and sends atifUrl", async () => { - judge.evaluate.mockResolvedValue({ - passed: true, - feedback: "ok", - criteriaResults: [{ criterionId: "uses_ts", passed: true, feedback: "tsconfig present", evaluated: true }], - }); - const ctx = makeCtx( - { - _id: "req-1", - observations: ["uses_ts"], - run: { turns: [{ iteration: 1, snapshotUrl: "s://1", atifUrl: "a://1" }] }, - }, - [{ id: "uses_ts", subject: "iteration" }], - ); - - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - - expect(judge.evaluate).toHaveBeenCalledWith( - expect.objectContaining({ snapshotUrl: "s://1", criteria: ["uses_ts"], atifUrl: "a://1" }), - ); - const call = judge.evaluate.mock.calls[0][0]; - expect(call.gate).toBeUndefined(); - expect(call.toolCallsUrl).toBeUndefined(); - expect(call.atifUrls).toBeUndefined(); - expect(ctx.collection.updateOne).toHaveBeenCalledWith( - { _id: "req-1", "run.turns.iteration": 1 }, - { $set: { "run.turns.$.observationResults": [{ criterionId: "uses_ts", passed: true, feedback: "tsconfig present", evaluated: true }] } }, - ); - }); - - it("skips iterations without a snapshotUrl", async () => { - judge.evaluate.mockResolvedValue({ passed: true, feedback: "", criteriaResults: [] }); - const ctx = makeCtx( - { _id: "req-1", observations: ["c1"], run: { turns: [{ iteration: 1 }] } }, - [{ id: "c1", subject: "iteration" }], - ); - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - expect(judge.evaluate).not.toHaveBeenCalled(); - }); - - it("throws on judge infrastructure error so the handler is marked failed", async () => { - judge.evaluate.mockRejectedValue(new JudgeInfrastructureError("down", { httpStatus: 503 })); - const ctx = makeCtx( - { _id: "req-1", observations: ["c1"], run: { turns: [{ iteration: 1, snapshotUrl: "s://1" }] } }, - [{ id: "c1", subject: "iteration" }], - ); - await expect(new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx)).rejects.toThrow("down"); - }); - }); - - describe("subject:run", () => { - it("evaluates ONCE over the merged trajectory and writes run-level observationResults", async () => { - judge.evaluate.mockResolvedValue({ - passed: true, - feedback: "ok", - criteriaResults: [ - { criterionId: "dep_added_then_removed", passed: true, feedback: "left-pad added then removed", evaluated: true }, - ], - }); - const ctx = makeCtx( - { - _id: "req-1", - observations: ["dep_added_then_removed"], - run: { - turns: [ - { iteration: 1, snapshotUrl: "s://1", atifUrl: "a://1" }, - { iteration: 2, snapshotUrl: "s://2", atifUrl: "a://2" }, - ], - }, - }, - [{ id: "dep_added_then_removed", subject: "run" }], - ); - - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - - // One judge call, using the FINAL snapshot + EVERY iteration's atif merged. - expect(judge.evaluate).toHaveBeenCalledTimes(1); - expect(judge.evaluate).toHaveBeenCalledWith( - expect.objectContaining({ - snapshotUrl: "s://2", - criteria: ["dep_added_then_removed"], - atifUrls: ["a://1", "a://2"], - }), - ); - const call = judge.evaluate.mock.calls[0][0]; - expect(call.atifUrl).toBeUndefined(); - // Written at the RUN level, not on any turn. - expect(ctx.collection.updateOne).toHaveBeenCalledWith( - { _id: "req-1" }, - { $set: { "run.observationResults": [{ criterionId: "dep_added_then_removed", passed: true, feedback: "left-pad added then removed", evaluated: true }] } }, - ); - }); - - it("defaults a subject-less observation to run (whole-run evaluation)", async () => { - judge.evaluate.mockResolvedValue({ passed: false, feedback: "no", criteriaResults: [] }); - const ctx = makeCtx( - { - _id: "req-1", - observations: ["legacy_obs"], - run: { turns: [{ iteration: 1, snapshotUrl: "s://1", atifUrl: "a://1" }] }, - }, - [{ id: "legacy_obs" }], // no subject → defaults to run - ); - - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - - expect(judge.evaluate).toHaveBeenCalledTimes(1); - const call = judge.evaluate.mock.calls[0][0]; - expect(call.atifUrls).toEqual(["a://1"]); - expect(ctx.collection.updateOne).toHaveBeenCalledWith( - { _id: "req-1" }, - expect.objectContaining({ $set: expect.objectContaining({ "run.observationResults": [] }) }), - ); - }); - - it("throws on judge infrastructure error during the run-subject evaluation", async () => { - judge.evaluate.mockRejectedValue(new JudgeInfrastructureError("down", { httpStatus: 503 })); - const ctx = makeCtx( - { _id: "req-1", observations: ["r1"], run: { turns: [{ iteration: 1, snapshotUrl: "s://1", atifUrl: "a://1" }] } }, - [{ id: "r1", subject: "run" }], - ); - await expect(new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx)).rejects.toThrow("down"); - }); - }); - - describe("mixed subjects", () => { - it("routes iteration and run observations to their respective evaluations", async () => { - judge.evaluate.mockResolvedValue({ passed: true, feedback: "ok", criteriaResults: [] }); - const ctx = makeCtx( - { - _id: "req-1", - observations: ["iter_obs", "run_obs"], - run: { - turns: [ - { iteration: 1, snapshotUrl: "s://1", atifUrl: "a://1" }, - { iteration: 2, snapshotUrl: "s://2", atifUrl: "a://2" }, - ], - }, - }, - [ - { id: "iter_obs", subject: "iteration" }, - { id: "run_obs", subject: "run" }, - ], - ); - - await new TaxonomyHandler(judge as unknown as JudgeClient).process(msg, ctx); - - // 2 iteration calls (one per turn) + 1 run call = 3 total. - expect(judge.evaluate).toHaveBeenCalledTimes(3); - const iterationCalls = judge.evaluate.mock.calls.filter((c) => c[0].criteria.includes("iter_obs")); - const runCalls = judge.evaluate.mock.calls.filter((c) => c[0].criteria.includes("run_obs")); - expect(iterationCalls).toHaveLength(2); - expect(runCalls).toHaveLength(1); - // The iteration eval only carries the iteration-subject criterion. - expect(iterationCalls[0][0].criteria).toEqual(["iter_obs"]); - // The run eval only carries the run-subject criterion, merged trajectory. - expect(runCalls[0][0].criteria).toEqual(["run_obs"]); - expect(runCalls[0][0].atifUrls).toEqual(["a://1", "a://2"]); - }); + it("does not initialize a prompt executor for runs without a taxonomy", async () => { + const executorFactory = vi.fn(); + const log = vi.fn(async () => undefined); + const handler = new TaxonomyHandler(executorFactory); + + await handler.process( + { type: "taxonomy", requestId: "request-1", runId: "run-1" }, + { + blobStorage: {} as never, + collection: { + findOne: vi.fn(async () => ({ + _id: "request-1", + projectId: "project-1", + scenario: { task: "Test task" }, + run: { _id: "run-1", attemptNumber: 1, status: "done" }, + })), + } as never, + criteriaCollection: {} as never, + log, + }, + ); + + expect(executorFactory).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith( + "info", + "Run is not attached to an Experiment; skipping Taxonome", + ); }); }); diff --git a/apps/workers/post-processor/src/handlers/taxonomy-handler.ts b/apps/workers/post-processor/src/handlers/taxonomy-handler.ts index 62cf41a85..29ef4d53c 100644 --- a/apps/workers/post-processor/src/handlers/taxonomy-handler.ts +++ b/apps/workers/post-processor/src/handlers/taxonomy-handler.ts @@ -1,214 +1,269 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { JudgeClient, isRetryableJudgeError } from "shared"; -import type { ConversationTurn, CriterionResult, CriterionSubject } from "shared"; -import type { PostProcessHandler, PostProcessorMessage, HandlerContext } from "../types.js"; +import { createHash } from "node:crypto"; +import { promisify } from "node:util"; +import { gzip } from "node:zlib"; +import { createAgentBehaviorHandlerRegistry } from "ax-taxonomy"; +import { + EVALUATION_SCHEMA_VERSION, + evaluate, + normalizeTaxonomyDocuments, + parseScopeAtif, + taxonomyRunResultSchema, + type PromptExecutor, +} from "taxonomy-engine"; +import { + copilotAI, + openCopilotTransport, + taxonomyPromptExecutor, +} from "taxonomy-engine/copilot"; +import type { + ExperimentDocument, + PortableTaxonomyDocument, + PortableTaxonomyElementDocument, + RequestDocument, + TaxonomyElementRevisionDocument, + TaxonomyRevisionDocument, +} from "shared"; +import type { + HandlerContext, + PostProcessHandler, + PostProcessorMessage, +} from "../types.js"; import { POST_PROCESSOR_VERSION } from "../version.js"; -/** Minimal shape of the request document fields this handler reads/writes. */ -interface RequestDoc { - projectId: string; - /** Observation-criteria ids selected at submit. Stored at the request - * top-level by the API (see RequestDocument.observations / issue #1156), - * NOT under `run` — the run subdocument only carries per-turn results. */ - observations?: string[]; - run?: { - _id?: string; - turns?: ConversationTurn[]; +const gzipAsync = promisify(gzip); + +interface PromptExecutorLease { + executor?: PromptExecutor; + close(): Promise; +} + +export type PromptExecutorFactory = ( + hasPromptElements: boolean, +) => Promise; + +async function defaultPromptExecutorFactory( + hasPromptElements: boolean, +): Promise { + if (!hasPromptElements) { + return { close: async () => undefined }; + } + const model = process.env.TAXONOME_MODEL ?? "claude-haiku-4.5"; + const transport = await openCopilotTransport(model); + return { + executor: taxonomyPromptExecutor(copilotAI(transport, model)), + close: () => transport.close(), }; } -/** Minimal projection of a criterion needed to route by evaluation granularity. */ -interface CriterionMeta { - id: string; - subject?: CriterionSubject; +function portableTaxonomy( + revision: TaxonomyRevisionDocument, +): PortableTaxonomyDocument { + return { + schema_version: revision.schema_version, + id: revision.id, + version: revision.version, + surfaces: revision.surfaces, + elements: revision.elements, + }; } -/** - * Taxonomy Handler (pp-taxonomy): records boolean observations. - * - * Each selected observation criterion is routed by its `subject` (#1156): - * - * - **`subject:"iteration"`** — evaluated on each iteration in isolation against - * that turn's snapshot + ATIF; written to `turn.observationResults` (mirrors - * gate `criteriaResults`). - * - **`subject:"run"`** (the observation default) — evaluated ONCE against the - * whole run: the final iteration's snapshot + the MERGED trajectory of every - * iteration's ATIF (`atifUrls`) + the full conversation history; written to the - * run-level `run.observationResults`. This is what makes cross-iteration - * behavior (e.g. a dependency added in one turn and removed in another) visible. - * - * Reuses the judge service; never gates, never steers the agent. Empty/absent - * `observations` → no-op. pp-taxonomy is dispatched exactly once per run (the - * scheduler never sets `message.iteration`), so the run-subject branch runs once - * with no idempotency guard. - */ +function portableElement( + revision: TaxonomyElementRevisionDocument, +): PortableTaxonomyElementDocument { + return { + ...(revision.schema_version + ? { schema_version: revision.schema_version } + : {}), + id: revision.id, + label: revision.label, + ...(revision.description ? { description: revision.description } : {}), + ...(revision.category_id ? { category_id: revision.category_id } : {}), + implementation: revision.implementation, + preconditions: revision.preconditions, + output: revision.output, + }; +} + +function mergeAtif(values: unknown[]): unknown { + if (values.length === 1) return values[0]; + const documents = values.map((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("ATIF artifact must be a JSON object"); + } + return value as Record; + }); + const first = structuredClone(documents[0]!); + first.steps = documents.flatMap((document) => + Array.isArray(document.steps) ? document.steps : [], + ); + return first; +} + +/** Evaluates one exact run with one exact pinned taxonomy revision. */ export class TaxonomyHandler implements PostProcessHandler { readonly type = "taxonomy"; readonly version = POST_PROCESSOR_VERSION; readonly autoBackfill = true; - private readonly judge: JudgeClient; - - constructor(judge?: JudgeClient, judgeUrl = process.env.JUDGE_URL || process.env.JUDGE_SERVICE_URL || "http://localhost:3002") { - this.judge = judge ?? new JudgeClient(judgeUrl); - } - - async process(message: PostProcessorMessage, ctx: HandlerContext): Promise { - const { requestId, iteration } = message; - const doc = (await ctx.collection.findOne({ _id: requestId } as any)) as RequestDoc | null; - const observations = doc?.observations ?? []; - const turns = doc?.run?.turns ?? []; - const projectId = doc?.projectId; + constructor( + private readonly executorFactory: PromptExecutorFactory = + defaultPromptExecutorFactory, + ) {} - if (observations.length === 0) { - await ctx.log("info", "No observations selected, skipping taxonomy evaluation"); - return; + async process( + message: PostProcessorMessage, + ctx: HandlerContext, + ): Promise { + const request = await ctx.collection.findOne({ + _id: message.requestId, + "run._id": message.runId, + } as never) as unknown as RequestDocument | null; + if (!request?.run || request.run._id !== message.runId) { + throw new Error("Exact run no longer exists"); } - if (turns.length === 0) { - await ctx.log("info", "No turns found, skipping taxonomy evaluation"); + if (!request.experimentId) { + await ctx.log("info", "Run is not attached to an Experiment; skipping Taxonome"); return; } - if (!projectId) { - throw new Error(`Request '${requestId}' is missing projectId`); + if (!ctx.db || !ctx.claimToken) { + throw new Error("Taxonome handler requires database and claim context"); } - // Resolve each selected observation's evaluation granularity. A missing/unknown - // subject defaults to "run" (the observation default), so legacy criteria authored - // before the `subject` field get whole-run evaluation. - const metas = (await ctx.criteriaCollection - .find( - { projectId, id: { $in: observations } }, - { projection: { id: 1, subject: 1 } } as any, - ) - .toArray()) as unknown as CriterionMeta[]; - const subjectById = new Map(metas.map((m) => [m.id, m.subject])); - const iterationObs = observations.filter((id) => subjectById.get(id) === "iteration"); - const runObs = observations.filter((id) => subjectById.get(id) !== "iteration"); - - await this.evaluateIterationObservations(iterationObs, turns, iteration, requestId, projectId, ctx); - await this.evaluateRunObservations(runObs, turns, iteration, requestId, projectId, ctx); - } - - /** Per-iteration observations → `turn.observationResults` (one judge call per turn). */ - private async evaluateIterationObservations( - iterationObs: string[], - turns: ConversationTurn[], - iteration: number | undefined, - requestId: string, - projectId: string, - ctx: HandlerContext, - ): Promise { - if (iterationObs.length === 0) return; - - const turnsToProcess = iteration ? turns.filter((t) => t.iteration === iteration) : turns; - let processedCount = 0; - - for (const turn of turnsToProcess) { - if (!turn.snapshotUrl) { - await ctx.log("info", `Iteration ${turn.iteration}: no snapshot, skipping`); - continue; - } - - try { - const result = await this.judge.evaluate({ - snapshotUrl: turn.snapshotUrl, - criteria: iterationObs, - conversationHistory: turns, - ...(turn.atifUrl ? { atifUrl: turn.atifUrl } : {}), - requestId, - projectId, - }); - - const observationResults: CriterionResult[] = result.criteriaResults ?? []; + const experiment = await ctx.db + .collection("experiments") + .findOne({ + _id: request.experimentId, + projectId: request.projectId, + }); + const selection = experiment?.execution.taxonomy; + if (!experiment || !selection) { + await ctx.log("info", "Experiment has no pinned taxonomy; skipping Taxonome"); + return; + } + const taxonomyRevision = await ctx.db + .collection("taxonomy-revisions") + .findOne({ + _id: selection.taxonomyRevisionId, + projectId: request.projectId, + ref: selection.ref, + contentSha256: selection.contentSha256, + }); + if (!taxonomyRevision) { + throw new Error("Pinned taxonomy revision integrity check failed"); + } - await ctx.collection.updateOne( - { _id: requestId, "run.turns.iteration": turn.iteration } as any, - { $set: { "run.turns.$.observationResults": observationResults } } as any, + const elementRevisions: TaxonomyElementRevisionDocument[] = []; + for (const reference of taxonomyRevision.elements) { + if (!("ref" in reference)) { + throw new Error( + `Stored taxonomy revision '${taxonomyRevision.ref}' contains unresolved path '${reference.path}'`, ); - - processedCount++; - await ctx.log("info", `Iteration ${turn.iteration}: ${observationResults.length} observations recorded`); - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - if (isRetryableJudgeError(err)) { - await ctx.log("error", `Iteration ${turn.iteration}: judge infrastructure error, failing handler: ${errMsg}`); - throw err; - } - await ctx.log("warn", `Iteration ${turn.iteration}: observation evaluation failed: ${errMsg}`); } + const element = await ctx.db + .collection( + "taxonomy-element-revisions", + ) + .findOne({ projectId: request.projectId, ref: reference.ref }); + if (!element) { + throw new Error(`Pinned taxonomy element '${reference.ref}' was not found`); + } + elementRevisions.push(element); } - await ctx.log( - "info", - `Iteration-subject evaluation complete: ${processedCount}/${turnsToProcess.length} iterations processed`, + const taxonomy = normalizeTaxonomyDocuments( + { + ...portableTaxonomy(taxonomyRevision), + elements: elementRevisions.map(portableElement), + }, + elementRevisions.map(portableElement), + { handlers: createAgentBehaviorHandlerRegistry() }, ); - } - - /** - * Whole-run observations → `run.observationResults` (one judge call over the - * merged trajectory). Skipped when a single-iteration dispatch is requested - * (forward-compat: a whole-run eval needs every turn). Today `message.iteration` - * is never set, so this runs once per run. - */ - private async evaluateRunObservations( - runObs: string[], - turns: ConversationTurn[], - iteration: number | undefined, - requestId: string, - projectId: string, - ctx: HandlerContext, - ): Promise { - if (runObs.length === 0) return; - if (iteration) { - await ctx.log( - "info", - `Skipping ${runObs.length} run-subject observation(s): single-iteration dispatch cannot evaluate the whole run`, - ); - return; - } - - const turnsWithSnapshot = turns.filter((t) => t.snapshotUrl); - const finalTurn = turnsWithSnapshot[turnsWithSnapshot.length - 1]; - if (!finalTurn?.snapshotUrl) { - await ctx.log("info", "No turn with a snapshot, skipping run-subject observations"); - return; + const atifUrls = (request.run.turns ?? []) + .map((turn) => turn.atifUrl) + .filter((url): url is string => Boolean(url)); + if (atifUrls.length === 0) { + throw new Error("Exact run has no ATIF artifacts"); } + const atifDocuments = await Promise.all( + atifUrls.map((url) => ctx.blobStorage.downloadJson(url)), + ); + const atif = parseScopeAtif(mergeAtif(atifDocuments), { + label: `${message.requestId}/${message.runId}`, + }); - // Merge every iteration's ATIF into one whole-run trajectory. - const atifUrls = turns - .map((t) => t.atifUrl) - .filter((u): u is string => typeof u === "string" && u.length > 0); - + const lease = await this.executorFactory( + taxonomy.elements.some((element) => element.implementation.type === "prompt"), + ); try { - const result = await this.judge.evaluate({ - snapshotUrl: finalTurn.snapshotUrl, - criteria: runObs, - conversationHistory: turns, - ...(atifUrls.length > 0 ? { atifUrls } : {}), - requestId, - projectId, + const results = await evaluate( + { atif, taxonomy }, + taxonomy.elements.map((element) => element.id), + lease.executor, + undefined, + { requestId: message.requestId, runId: message.runId }, + ); + const scenario = request.scenario; + const result = taxonomyRunResultSchema.parse({ + run: { + id: message.runId, + scenario: { + id: message.requestId, + label: "Scope scenario", + ...(scenario.task ? { task: scenario.task } : {}), + }, + profile: { + id: request.profileId ?? "unprofiled", + label: request.profileId ?? "Unprofiled", + }, + }, + results, + metadata: { + requestId: message.requestId, + runId: message.runId, + experimentId: request.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + taxonomyRevisionId: taxonomyRevision._id, + taxonomyRef: taxonomyRevision.ref, + }, }); - - const runObservationResults: CriterionResult[] = result.criteriaResults ?? []; - - await ctx.collection.updateOne( - { _id: requestId } as any, - { $set: { "run.observationResults": runObservationResults } } as any, + const json = Buffer.from(JSON.stringify(result), "utf8"); + const compressed = await gzipAsync(json); + const contentSha256 = createHash("sha256").update(json).digest("hex"); + const blobName = + `${message.requestId}/runs/${message.runId}/taxonomy-evaluation/` + + `${ctx.claimToken}/result.json.gz`; + const url = await ctx.blobStorage.uploadBuffer( + blobName, + compressed, + "application/json", + "gzip", ); - - await ctx.log( - "info", - `Run-subject evaluation complete: ${runObservationResults.length} whole-run observation(s) recorded over ${atifUrls.length} merged trajectory(ies)`, + const publication = await ctx.collection.updateOne( + { + _id: message.requestId, + "run._id": message.runId, + "run.handlerStatus.pp-taxonomy.status": "processing", + "run.handlerStatus.pp-taxonomy.claimToken": ctx.claimToken, + } as never, + { + $set: { + "run.taxonomyEvaluation": { + url, + schemaVersion: EVALUATION_SCHEMA_VERSION, + contentSha256, + sizeBytes: json.byteLength, + }, + }, + }, ); - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - if (isRetryableJudgeError(err)) { - await ctx.log("error", `Run-subject: judge infrastructure error, failing handler: ${errMsg}`); - throw err; + if (publication.matchedCount !== 1) { + throw new Error("Taxonomy evaluation claim became stale before publication"); } - await ctx.log("warn", `Run-subject observation evaluation failed: ${errMsg}`); + } finally { + await lease.close(); } } } diff --git a/apps/workers/post-processor/src/post-processor.test.ts b/apps/workers/post-processor/src/post-processor.test.ts index 4d06a771a..968745231 100644 --- a/apps/workers/post-processor/src/post-processor.test.ts +++ b/apps/workers/post-processor/src/post-processor.test.ts @@ -171,6 +171,7 @@ describe("PostProcessor", () => { _id: "req-1", "run._id": "run-1", "run.handlerStatus.pp-atif.status": "processing", + "run.handlerStatus.pp-atif.claimToken": expect.any(String), }, { $set: expect.objectContaining({ @@ -228,6 +229,7 @@ describe("PostProcessor", () => { _id: "req-1", "run._id": "run-1", "run.handlerStatus.pp-atif.status": "processing", + "run.handlerStatus.pp-atif.claimToken": expect.any(String), }, { $set: expect.objectContaining({ diff --git a/apps/workers/post-processor/src/post-processor.ts b/apps/workers/post-processor/src/post-processor.ts index 422019265..c5f670bcc 100644 --- a/apps/workers/post-processor/src/post-processor.ts +++ b/apps/workers/post-processor/src/post-processor.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import type { DequeuedMessageItem } from "@azure/storage-queue"; +import { randomUUID } from "node:crypto"; import { BaseQueueProcessor, BlobStorage, type BaseQueueProcessorConfig, type LogEvent, type VisibilityHeartbeat } from "shared"; import type { RequestDocument, HandlerRunStatus } from "shared"; import { trackMetric, trackEvent } from "telemetry"; @@ -126,6 +127,7 @@ export class PostProcessor extends BaseQueueProcessor { } const handlerId = `pp-${msg.type}`; + const claimToken = randomUUID(); const currentStatus = doc.run?.handlerStatus?.[handlerId] as HandlerRunStatus | undefined; // This is the authoritative claim. A queued message alone does not mean the @@ -139,6 +141,7 @@ export class PostProcessor extends BaseQueueProcessor { $set: { "run.postProcessorStatus": "processing", [`run.handlerStatus.${handlerId}.status`]: "processing", + [`run.handlerStatus.${handlerId}.claimToken`]: claimToken, [`run.handlerStatus.${handlerId}.updatedAt`]: new Date(), }, $unset: { @@ -173,6 +176,8 @@ export class PostProcessor extends BaseQueueProcessor { blobStorage: this.blobStorage, collection: this.collection as any, criteriaCollection: this.db.collection("criteria"), + db: this.db, + claimToken, log, }; @@ -184,6 +189,7 @@ export class PostProcessor extends BaseQueueProcessor { _id: doc._id, "run._id": msg.runId, [`run.handlerStatus.${handlerId}.status`]: "processing", + [`run.handlerStatus.${handlerId}.claimToken`]: claimToken, } as any, { $set: { @@ -228,6 +234,7 @@ export class PostProcessor extends BaseQueueProcessor { _id: doc._id, "run._id": msg.runId, [`run.handlerStatus.${handlerId}.status`]: "processing", + [`run.handlerStatus.${handlerId}.claimToken`]: claimToken, } as any, { $set: { diff --git a/apps/workers/post-processor/src/types.ts b/apps/workers/post-processor/src/types.ts index c93748cf0..26c010358 100644 --- a/apps/workers/post-processor/src/types.ts +++ b/apps/workers/post-processor/src/types.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Collection } from "mongodb"; +import type { Collection, Db } from "mongodb"; import type { BlobStorage, LogEvent } from "shared"; /** Queue message dispatched by the PostProcessorDispatcher in the scheduler. */ @@ -19,6 +19,8 @@ export interface HandlerContext { /** The `criteria` collection — lets handlers resolve criterion metadata * (e.g. observation `subject`/`taxonomyElementId`) by id. #1156. */ criteriaCollection: Collection; + db?: Db; + claimToken?: string; log: (level: LogEvent["level"], msg: string, data?: Record) => Promise; } diff --git a/packages/db-migrations/src/migrations/037-create-taxonomy-indexes.ts b/packages/db-migrations/src/migrations/037-create-taxonomy-indexes.ts new file mode 100644 index 000000000..3e0705407 --- /dev/null +++ b/packages/db-migrations/src/migrations/037-create-taxonomy-indexes.ts @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Db } from "mongodb"; +import type { MigrationInterface } from "mongo-migrate-ts"; + +const INDEXES = [ + { + collection: "taxonomies", + indexes: [ + { key: { projectId: 1 } }, + { key: { id: 1 } }, + { key: { deletedAt: 1 } }, + { key: { projectId: 1, id: 1 }, unique: true }, + ], + }, + { + collection: "taxonomy-revisions", + indexes: [ + { key: { taxonomyId: 1 } }, + { key: { projectId: 1 } }, + { key: { ref: 1 } }, + { key: { taxonomyId: 1, revision: -1 }, unique: true }, + ], + }, + { + collection: "taxonomy-elements", + indexes: [ + { key: { projectId: 1 } }, + { key: { id: 1 } }, + { key: { deletedAt: 1 } }, + { key: { projectId: 1, id: 1 }, unique: true }, + ], + }, + { + collection: "taxonomy-element-revisions", + indexes: [ + { key: { elementId: 1 } }, + { key: { projectId: 1 } }, + { key: { ref: 1 } }, + { key: { elementId: 1, revision: -1 }, unique: true }, + ], + }, +] as const; + +/** Creates the four focused Cosmos-compatible taxonomy collection indexes. */ +export class CreateTaxonomyIndexes implements MigrationInterface { + async up(db: Db): Promise { + for (const definition of INDEXES) { + const collection = db.collection(definition.collection); + for (const index of definition.indexes) { + try { + await collection.createIndex(index.key, { + ...("unique" in index ? { unique: index.unique } : {}), + }); + console.log( + ` Created index ${JSON.stringify(index.key)} on ${definition.collection}`, + ); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.log( + ` Index ${JSON.stringify(index.key)} on ${definition.collection} already exists or could not be created: ${message}`, + ); + } + } + + const indexes = await collection.indexes(); + console.log( + ` Verified ${definition.collection} indexes: ${indexes.map((index) => index.name).join(", ")}`, + ); + } + } + + async down(): Promise { + console.log( + " Skipping taxonomy index drops — indexes should be removed manually if needed", + ); + } +} diff --git a/packages/db-migrations/src/required-migrations.ts b/packages/db-migrations/src/required-migrations.ts index 1c19619f7..3f26b7f4c 100644 --- a/packages/db-migrations/src/required-migrations.ts +++ b/packages/db-migrations/src/required-migrations.ts @@ -49,6 +49,7 @@ export const REQUIRED_MIGRATIONS: readonly string[] = [ "034-create-experiment-storage-indexes.ts", "035-version-report-templates.ts", "036-add-experiment-lifecycle-indexes.ts", + "037-create-taxonomy-indexes.ts", "021-backfill-criteria-kind.ts", "022-backfill-criteria-subject.ts", ]; diff --git a/packages/shared/src/experiments/experiment-store.ts b/packages/shared/src/experiments/experiment-store.ts index 884de07da..7e5d17944 100644 --- a/packages/shared/src/experiments/experiment-store.ts +++ b/packages/shared/src/experiments/experiment-store.ts @@ -163,6 +163,7 @@ export class ExperimentStore { version: 1, requests: payload, vectorFingerprint: fingerprintExperimentRequestVector(requests), + ...(revision.taxonomy ? { taxonomy: revision.taxonomy } : {}), handlerSummary: input.handlerSummary ?? {}, blocking: input.blocking ?? [], createdAt: now, @@ -1012,7 +1013,11 @@ export class ExperimentStore { metadata: experiment.metadata, requestIds: canonicalRequests.map(({ requestId }) => requestId), reportTemplateIds: experiment.revision.reportTemplateIds, + taxonomy: experiment.revision.taxonomy, }), + ...(experiment.revision.taxonomy + ? { taxonomy: experiment.revision.taxonomy } + : {}), createdAt: input.now ?? new Date(), }) as ExperimentRevision; @@ -1061,6 +1066,7 @@ export class ExperimentStore { candidate, canonicalRequests, now, + nextRevision.taxonomy, ), }), now, @@ -1070,6 +1076,7 @@ export class ExperimentStore { payload, canonicalRequests, now, + nextRevision.taxonomy, ); const claimed = await this.collections.experiments.findOneAndUpdate( { @@ -1172,6 +1179,9 @@ export class ExperimentStore { version: 1, requests: payload, vectorFingerprint: experiment.execution.vectorFingerprint, + ...(experiment.revision.taxonomy + ? { taxonomy: experiment.revision.taxonomy } + : {}), handlerSummary: reusableHandlerSummary, reportTemplates: reusableReportTemplates, blocking: [], @@ -1401,6 +1411,7 @@ export class ExperimentStore { version: 1, requests, vectorFingerprint: fingerprintExperimentRequestVector(entries), + ...(current.taxonomy ? { taxonomy: current.taxonomy } : {}), handlerSummary: {}, reportTemplates: {}, blocking: [], @@ -1414,6 +1425,7 @@ export class ExperimentStore { requests: FixedPayload, entries: ExperimentRequestState[], now: Date, + taxonomy?: ExperimentRevision["taxonomy"], ): ExperimentExecution { return { experimentExecutionId, @@ -1421,6 +1433,7 @@ export class ExperimentStore { version: 1, requests, vectorFingerprint: fingerprintExperimentRequestVector(entries), + ...(taxonomy ? { taxonomy } : {}), handlerSummary: {}, reportTemplates: {}, blocking: [], diff --git a/packages/shared/src/experiments/schemas.ts b/packages/shared/src/experiments/schemas.ts index 5d8616089..ee2310e08 100644 --- a/packages/shared/src/experiments/schemas.ts +++ b/packages/shared/src/experiments/schemas.ts @@ -60,6 +60,14 @@ export const ExperimentRevisionSchema = z memberCount: z.number().int().nonnegative(), membershipFingerprint: IdSchema, reportTemplateIds: z.array(IdSchema).max(64).default([]), + taxonomy: z + .object({ + taxonomyRevisionId: IdSchema, + ref: IdSchema, + contentSha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + .optional(), definitionFingerprint: IdSchema, createdAt: z.coerce.date(), }) @@ -116,6 +124,7 @@ const ExperimentExecutionBaseSchema = z version: z.number().int().min(1), requests: ExperimentFixedRequestPayloadSchema, vectorFingerprint: IdSchema, + taxonomy: ExperimentRevisionSchema.shape.taxonomy, handlerSummary: z .record(IdSchema, ExperimentHandlerSummarySchema) .refine((value) => Object.keys(value).length <= 64, { @@ -168,6 +177,19 @@ const ExperimentExecutionBaseSchema = z }) .strict() .optional(), + taxonomyReadout: z + .object({ + status: z.enum(["processing", "done", "failed"]), + claimToken: IdSchema.optional(), + experimentExecutionId: IdSchema, + taxonomyRevisionId: IdSchema, + jsonUrl: z.string().min(1).max(2_048).optional(), + markdownUrl: z.string().min(1).max(2_048).optional(), + updatedAt: z.coerce.date(), + error: ErrorSchema.optional(), + }) + .strict() + .optional(), updatedAt: z.coerce.date(), error: ErrorSchema.optional(), }) diff --git a/packages/shared/src/experiments/types.ts b/packages/shared/src/experiments/types.ts index 88719b028..014079840 100644 --- a/packages/shared/src/experiments/types.ts +++ b/packages/shared/src/experiments/types.ts @@ -114,12 +114,19 @@ export interface ExperimentMetadata { createdBy?: string; } +export interface ExperimentTaxonomySelection { + taxonomyRevisionId: string; + ref: string; + contentSha256: string; +} + export interface ExperimentRevision { experimentRevisionId: string; revisionNumber: number; memberCount: number; membershipFingerprint: string; reportTemplateIds: string[]; + taxonomy?: ExperimentTaxonomySelection; definitionFingerprint: string; createdAt: Date; } @@ -200,10 +207,21 @@ export interface ExperimentExecution { version: number; requests: FixedPayload; vectorFingerprint: string; + taxonomy?: ExperimentTaxonomySelection; handlerSummary: Record; reportTemplates: Record; blocking: ExperimentBlocker[]; report?: ExperimentReportReference; + taxonomyReadout?: { + status: "processing" | "done" | "failed"; + claimToken?: string; + experimentExecutionId: string; + taxonomyRevisionId: string; + jsonUrl?: string; + markdownUrl?: string; + updatedAt: Date; + error?: string; + }; createdAt: Date; updatedAt: Date; startedAt?: Date; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 1b6abd4b1..4bce83184 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -21,6 +21,7 @@ export * from "./resources/index.js"; export * from "./extensions/index.js"; export * from "./projects/index.js"; export * from "./experiments/index.js"; +export * from "./taxonomies/index.js"; export * from "./agent-version.js"; export * from "./resolve-agent-version.js"; export * from "./resolve-agent-target.js"; diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index 622595c7e..9fce8ccec 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -23,3 +23,4 @@ export * from "./feature-flag.js"; export * from "./profile.js"; export * from "./project.js"; export * from "./readout.js"; +export * from "./taxonomy.js"; diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 64e426a66..56dd6e89b 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -279,6 +279,16 @@ export const RunStateSchema = z observationResults: z.array(CriterionResultSchema).optional(), /** Canonical run-scoped report-template IDs frozen for this exact attempt. */ reportTemplateIds: z.array(z.string().min(1)).optional(), + /** Immutable compressed Taxonome SDK result for this exact run attempt. */ + taxonomyEvaluation: z + .object({ + url: z.string().min(1), + schemaVersion: z.string().min(1), + contentSha256: z.string().regex(/^[a-f0-9]{64}$/), + sizeBytes: z.number().int().nonnegative(), + }) + .strict() + .optional(), }) .openapi("RunState"); diff --git a/packages/shared/src/schemas/taxonomy.ts b/packages/shared/src/schemas/taxonomy.ts new file mode 100644 index 000000000..c6aa7afa5 --- /dev/null +++ b/packages/shared/src/schemas/taxonomy.ts @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi"; + +extendZodWithOpenApi(z); + +export const TaxonomySourceTypeSchema = z.enum(["builtin", "user"]); +export const TaxonomyBusinessIdSchema = z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); +export const TaxonomyRevisionRefSchema = z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*@r[1-9]\d*$/); + +const RelativeElementPathSchema = z.string().superRefine((value, ctx) => { + if ( + value.startsWith("/") || + /^[A-Za-z]:[\\/]/.test(value) || + value.split(/[\\/]/).some((part) => part === "..") + ) { + ctx.addIssue({ + code: "custom", + message: "Element path must be relative and must not traverse parent directories", + }); + } +}); + +export const TaxonomyElementReferenceSchema = z.union([ + z.object({ path: RelativeElementPathSchema }).strict(), + z.object({ ref: TaxonomyRevisionRefSchema }).strict(), +]); + +export const PortableTaxonomySchema = z + .object({ + schema_version: z.literal("scope/taxonomy/v1"), + id: TaxonomyBusinessIdSchema, + version: z.string().regex(/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/), + surfaces: z + .array( + z + .object({ + id: TaxonomyBusinessIdSchema, + label: z.string().min(1), + description: z.string().min(1).optional(), + }) + .strict(), + ) + .max(100), + elements: z.array(TaxonomyElementReferenceSchema).min(1).max(200), + }) + .strict() + .openapi("PortableTaxonomy"); + +export const PortableTaxonomyElementSchema = z + .object({ + schema_version: z.literal("scope/taxonomy/v1").optional(), + id: TaxonomyBusinessIdSchema, + label: z.string().min(1), + description: z.string().min(1).optional(), + category_id: TaxonomyBusinessIdSchema.optional(), + implementation: z.discriminatedUnion("type", [ + z.object({ type: z.literal("prompt"), prompt: z.string().min(1) }).strict(), + z + .object({ + type: z.literal("deterministic"), + handler: z.string().min(1), + config: z.unknown().optional(), + }) + .strict(), + ]), + preconditions: z.record(z.string(), z.boolean()), + output: z + .object({ + evidence_roles: z.record(z.string(), z.unknown()), + constraints: z.array(z.unknown()).optional(), + }) + .strict(), + }) + .strict() + .openapi("PortableTaxonomyElement"); + +export const CreateTaxonomyInputSchema = z + .object({ + name: z.string().trim().min(1).max(256), + id: TaxonomyBusinessIdSchema.optional(), + description: z.string().max(4096).optional(), + sourceType: TaxonomySourceTypeSchema.default("user"), + creator: z.string().optional(), + content: PortableTaxonomySchema, + }) + .strict() + .openapi("CreateTaxonomyInput"); + +export const CreateTaxonomyElementInputSchema = z + .object({ + name: z.string().trim().min(1).max(256), + id: TaxonomyBusinessIdSchema.optional(), + description: z.string().max(4096).optional(), + sourceType: TaxonomySourceTypeSchema.default("user"), + creator: z.string().optional(), + content: PortableTaxonomyElementSchema, + }) + .strict() + .openapi("CreateTaxonomyElementInput"); + +export const UpdateTaxonomyMetadataInputSchema = z + .object({ + name: z.string().trim().min(1).max(256).optional(), + description: z.string().max(4096).optional(), + }) + .strict() + .openapi("UpdateTaxonomyMetadataInput"); + +export const CreateTaxonomyRevisionInputSchema = z + .object({ + content: PortableTaxonomySchema, + creator: z.string().optional(), + }) + .strict() + .openapi("CreateTaxonomyRevisionInput"); + +export const CreateTaxonomyElementRevisionInputSchema = z + .object({ + content: PortableTaxonomyElementSchema, + creator: z.string().optional(), + }) + .strict() + .openapi("CreateTaxonomyElementRevisionInput"); + +export const CopyBuiltinTaxonomyRevisionInputSchema = z + .object({ + name: z.string().trim().min(1).max(256), + id: TaxonomyBusinessIdSchema.optional(), + description: z.string().max(4096).optional(), + creator: z.string().optional(), + }) + .strict() + .openapi("CopyBuiltinTaxonomyRevisionInput"); + +export const TaxonomyResponseSchema = z + .object({ + _id: z.string(), + projectId: z.string(), + id: TaxonomyBusinessIdSchema, + name: z.string(), + description: z.string().optional(), + sourceType: TaxonomySourceTypeSchema, + revisionCounter: z.number().int().nonnegative(), + latestRevisionId: z.string().optional(), + latestRevision: z.number().int().positive().optional(), + creator: z.string().optional(), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date().optional(), + deletedAt: z.coerce.date().optional(), + }) + .openapi("TaxonomyResponse"); + +export const TaxonomyElementResponseSchema = TaxonomyResponseSchema.openapi( + "TaxonomyElementResponse", +); + +export const TaxonomyRevisionResponseSchema = PortableTaxonomySchema.extend({ + _id: z.string(), + taxonomyId: z.string(), + projectId: z.string(), + sourceType: TaxonomySourceTypeSchema, + revision: z.number().int().positive(), + ref: TaxonomyRevisionRefSchema, + contentSha256: z.string().length(64), + creator: z.string().optional(), + createdAt: z.coerce.date(), +}).openapi("TaxonomyRevisionResponse"); + +export const TaxonomyElementRevisionResponseSchema = + PortableTaxonomyElementSchema.extend({ + _id: z.string(), + elementId: z.string(), + projectId: z.string(), + sourceType: TaxonomySourceTypeSchema, + revision: z.number().int().positive(), + ref: TaxonomyRevisionRefSchema, + contentSha256: z.string().length(64), + creator: z.string().optional(), + createdAt: z.coerce.date(), + }).openapi("TaxonomyElementRevisionResponse"); diff --git a/packages/shared/src/storage/blob-storage.ts b/packages/shared/src/storage/blob-storage.ts index 635518776..b07eb73e9 100644 --- a/packages/shared/src/storage/blob-storage.ts +++ b/packages/shared/src/storage/blob-storage.ts @@ -460,6 +460,24 @@ export class BlobStorage { return blockBlobClient.url; } + /** Uploads an arbitrary in-memory artifact and returns its stable blob URL. */ + async uploadBuffer( + blobName: string, + body: Buffer, + contentType: string, + contentEncoding?: string, + ): Promise { + await this.ensureContainer(); + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName); + await blockBlobClient.uploadData(body, { + blobHTTPHeaders: { + blobContentType: contentType, + ...(contentEncoding ? { blobContentEncoding: contentEncoding } : {}), + }, + }); + return blockBlobClient.url; + } + /** * Downloads a snapshot from blob storage and extracts it to the target directory. */ @@ -530,4 +548,27 @@ export class BlobStorage { const buf = await this.downloadBlobToBuffer(blobName); return JSON.parse(buf.toString("utf-8")) as T; } + + /** Downloads arbitrary snapshot-container bytes by stable URL or blob name. */ + async downloadBuffer(urlOrBlobName: string): Promise { + await this.ensureContainer(); + let blobName = urlOrBlobName; + if ( + urlOrBlobName.startsWith("http://") || + urlOrBlobName.startsWith("https://") + ) { + const url = new URL(urlOrBlobName); + const containerPrefix = `/${SNAPSHOTS_CONTAINER}/`; + const containerIndex = url.pathname.indexOf(containerPrefix); + if (containerIndex === -1) { + throw new Error( + `Blob URL does not contain container '${SNAPSHOTS_CONTAINER}': ${urlOrBlobName}`, + ); + } + blobName = decodeURIComponent( + url.pathname.substring(containerIndex + containerPrefix.length), + ); + } + return this.downloadBlobToBuffer(blobName); + } } diff --git a/packages/shared/src/taxonomies/index.ts b/packages/shared/src/taxonomies/index.ts new file mode 100644 index 000000000..1983cb0e0 --- /dev/null +++ b/packages/shared/src/taxonomies/index.ts @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export * from "./taxonomy-store.js"; +export * from "./taxonomy-revision-store.js"; diff --git a/packages/shared/src/taxonomies/taxonomy-contract.test.ts b/packages/shared/src/taxonomies/taxonomy-contract.test.ts new file mode 100644 index 000000000..1c267cbe5 --- /dev/null +++ b/packages/shared/src/taxonomies/taxonomy-contract.test.ts @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + PortableTaxonomySchema, + TaxonomyRevisionRefSchema, +} from "../schemas/taxonomy.js"; +import { taxonomyContentSha256 } from "./taxonomy-revision-store.js"; + +const taxonomy = { + schema_version: "scope/taxonomy/v1" as const, + id: "example", + version: "1.0.0", + surfaces: [], + elements: [{ ref: "element-a@r1" }], +}; + +describe("taxonomy contracts", () => { + it("requires canonical exact revision refs", () => { + expect(TaxonomyRevisionRefSchema.parse("element-a@r12")).toBe( + "element-a@r12", + ); + for (const invalid of ["element-a", "element-a@latest", "@r1", "a@r0"]) { + expect(() => TaxonomyRevisionRefSchema.parse(invalid)).toThrow(); + } + }); + + it("rejects ambiguous and unsafe element references", () => { + for (const element of [ + { path: "../element.yaml" }, + { path: "/tmp/element.yaml" }, + { path: "element.yaml", ref: "element-a@r1" }, + {}, + ]) { + expect(() => + PortableTaxonomySchema.parse({ ...taxonomy, elements: [element] }), + ).toThrow(); + } + }); + + it("hashes semantic content independently of object key order", () => { + const reordered = { + version: "1.0.0", + id: "example", + elements: [{ ref: "element-a@r1" }], + surfaces: [], + schema_version: "scope/taxonomy/v1" as const, + }; + expect(taxonomyContentSha256(taxonomy)).toBe( + taxonomyContentSha256(reordered), + ); + }); +}); diff --git a/packages/shared/src/taxonomies/taxonomy-revision-store.ts b/packages/shared/src/taxonomies/taxonomy-revision-store.ts new file mode 100644 index 000000000..c5e6ed821 --- /dev/null +++ b/packages/shared/src/taxonomies/taxonomy-revision-store.ts @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID } from "node:crypto"; +import type { Collection } from "mongodb"; +import type { + PortableTaxonomyDocument, + PortableTaxonomyElementDocument, + TaxonomyDocument, + TaxonomyElementDocument, + TaxonomyElementRevisionDocument, + TaxonomyRevisionDocument, +} from "../types/taxonomy.js"; +import type { TaxonomyRootStore } from "./taxonomy-store.js"; + +type RevisionDocument = + | TaxonomyRevisionDocument + | TaxonomyElementRevisionDocument; +type PortableDocument = + | PortableTaxonomyDocument + | PortableTaxonomyElementDocument; +type RootDocument = TaxonomyDocument | TaxonomyElementDocument; + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, child]) => child !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, child]) => [key, canonicalize(child)]), + ); + } + return value; +} + +export function taxonomyContentSha256(content: PortableDocument): string { + return createHash("sha256") + .update(JSON.stringify(canonicalize(content))) + .digest("hex"); +} + +/** Store for one immutable taxonomy revision collection. */ +export class TaxonomyRevisionStore< + TRevision extends RevisionDocument, + TRoot extends RootDocument, + TContent extends PortableDocument, +> { + constructor( + private readonly collection: Collection, + private readonly rootStore: TaxonomyRootStore, + private readonly rootKey: "taxonomyId" | "elementId", + ) {} + + async get(id: string): Promise { + return this.collection.findOne({ _id: id } as never) as Promise; + } + + async getByRef(projectId: string, ref: string): Promise { + return this.collection.findOne({ projectId, ref } as never) as Promise; + } + + async getLatest(rootId: string): Promise { + return this.collection + .find({ [this.rootKey]: rootId } as never) + .sort({ revision: -1 }) + .limit(1) + .next() as Promise; + } + + async list(rootId: string): Promise { + return this.collection + .find({ [this.rootKey]: rootId } as never) + .sort({ revision: -1 }) + .toArray() as Promise; + } + + async create( + rootId: string, + content: TContent, + creator?: string, + ): Promise<{ revision: TRevision; deduplicated: boolean }> { + const root = await this.rootStore.get(rootId); + if (!root) throw new Error(`Taxonomy root '${rootId}' not found`); + if (root.sourceType === "builtin" && root.revisionCounter > 0) { + throw new Error("Built-in taxonomy content is read-only"); + } + + const contentSha256 = taxonomyContentSha256(content); + const latest = await this.getLatest(rootId); + if (latest?.contentSha256 === contentSha256) { + return { revision: latest, deduplicated: true }; + } + + const allocated = await this.rootStore.allocateRevision(rootId); + if (!allocated) throw new Error(`Taxonomy root '${rootId}' not found`); + const revision = allocated.revisionCounter; + const document = { + ...content, + _id: randomUUID(), + [this.rootKey]: rootId, + projectId: root.projectId, + sourceType: root.sourceType, + revision, + ref: `${content.id}@r${revision}`, + contentSha256, + ...(creator ? { creator } : {}), + createdAt: new Date(), + } as unknown as TRevision; + await this.collection.insertOne(document as never); + await this.rootStore.setLatest(rootId, document._id, revision); + return { revision: document, deduplicated: false }; + } +} diff --git a/packages/shared/src/taxonomies/taxonomy-store.ts b/packages/shared/src/taxonomies/taxonomy-store.ts new file mode 100644 index 000000000..0f4c6958b --- /dev/null +++ b/packages/shared/src/taxonomies/taxonomy-store.ts @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { randomUUID } from "node:crypto"; +import type { Collection } from "mongodb"; +import type { + TaxonomyDocument, + TaxonomyElementDocument, + TaxonomySourceType, +} from "../types/taxonomy.js"; + +type RootDocument = TaxonomyDocument | TaxonomyElementDocument; + +export interface CreateTaxonomyRootInput { + projectId: string; + id: string; + name: string; + description?: string; + sourceType: TaxonomySourceType; + creator?: string; +} + +/** Store for one mutable taxonomy root collection. */ +export class TaxonomyRootStore { + constructor(private readonly collection: Collection) {} + + async get(id: string, includeDeleted = false): Promise { + return this.collection.findOne({ + _id: id, + ...(includeDeleted ? {} : { deletedAt: { $exists: false } }), + } as never) as Promise; + } + + async list(projectId: string, includeDeleted = false): Promise { + return this.collection + .find({ + projectId, + ...(includeDeleted ? {} : { deletedAt: { $exists: false } }), + } as never) + .sort({ createdAt: -1 }) + .toArray() as Promise; + } + + async getByBusinessId( + projectId: string, + id: string, + includeDeleted = false, + ): Promise { + return this.collection.findOne({ + projectId, + id, + ...(includeDeleted ? {} : { deletedAt: { $exists: false } }), + } as never) as Promise; + } + + async create(input: CreateTaxonomyRootInput): Promise { + const now = new Date(); + const document = { + _id: randomUUID(), + projectId: input.projectId, + id: input.id, + name: input.name, + ...(input.description ? { description: input.description } : {}), + sourceType: input.sourceType, + revisionCounter: 0, + ...(input.creator ? { creator: input.creator } : {}), + createdAt: now, + } as T; + await this.collection.insertOne(document as never); + return document; + } + + async updateMetadata( + id: string, + patch: { name?: string; description?: string }, + ): Promise { + return this.collection.findOneAndUpdate( + { _id: id, sourceType: "user", deletedAt: { $exists: false } } as never, + { + $set: { + ...(patch.name !== undefined ? { name: patch.name } : {}), + ...(patch.description !== undefined + ? { description: patch.description } + : {}), + updatedAt: new Date(), + }, + } as never, + { returnDocument: "after" }, + ) as Promise; + } + + async allocateRevision(id: string): Promise { + return this.collection.findOneAndUpdate( + { _id: id, deletedAt: { $exists: false } } as never, + { $inc: { revisionCounter: 1 }, $set: { updatedAt: new Date() } } as never, + { returnDocument: "after" }, + ) as Promise; + } + + async setLatest(id: string, revisionId: string, revision: number): Promise { + await this.collection.updateOne( + { + _id: id, + $or: [ + { latestRevision: { $exists: false } }, + { latestRevision: { $lt: revision } }, + ], + } as never, + { + $set: { + latestRevisionId: revisionId, + latestRevision: revision, + updatedAt: new Date(), + }, + } as never, + ); + } + + async softDelete(id: string): Promise { + const result = await this.collection.updateOne( + { _id: id, sourceType: "user", deletedAt: { $exists: false } } as never, + { $set: { deletedAt: new Date(), updatedAt: new Date() } } as never, + ); + return result.modifiedCount === 1; + } + + async restore(id: string): Promise { + return this.collection.findOneAndUpdate( + { _id: id, sourceType: "user", deletedAt: { $exists: true } } as never, + { $unset: { deletedAt: "" }, $set: { updatedAt: new Date() } } as never, + { returnDocument: "after" }, + ) as Promise; + } +} diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 12c41bccf..8b3552d8d 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -9,3 +9,4 @@ export * from "./resource.js"; export * from "./extension.js"; export * from "./profile.js"; export * from "./project.js"; +export * from "./taxonomy.js"; diff --git a/packages/shared/src/types/taxonomy.ts b/packages/shared/src/types/taxonomy.ts new file mode 100644 index 000000000..8a232e64e --- /dev/null +++ b/packages/shared/src/types/taxonomy.ts @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export type TaxonomySourceType = "builtin" | "user"; + +export interface TaxonomyPathElementReference { + path: string; +} + +export interface TaxonomyRevisionElementReference { + ref: string; +} + +export type TaxonomyElementReference = + | TaxonomyPathElementReference + | TaxonomyRevisionElementReference; + +export interface PortableTaxonomyDocument { + schema_version: "scope/taxonomy/v1"; + id: string; + version: string; + surfaces: Array<{ + id: string; + label: string; + description?: string; + }>; + elements: TaxonomyElementReference[]; +} + +export interface PortableTaxonomyElementDocument { + schema_version?: "scope/taxonomy/v1"; + id: string; + label: string; + description?: string; + category_id?: string; + implementation: + | { type: "prompt"; prompt: string } + | { type: "deterministic"; handler: string; config?: unknown }; + preconditions: Record; + output: { + evidence_roles: Record; + constraints?: unknown[]; + }; +} + +export interface TaxonomyDocument { + _id: string; + projectId: string; + id: string; + name: string; + description?: string; + sourceType: TaxonomySourceType; + revisionCounter: number; + latestRevisionId?: string; + latestRevision?: number; + creator?: string; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +export interface TaxonomyElementDocument { + _id: string; + projectId: string; + id: string; + name: string; + description?: string; + sourceType: TaxonomySourceType; + revisionCounter: number; + latestRevisionId?: string; + latestRevision?: number; + creator?: string; + createdAt: Date; + updatedAt?: Date; + deletedAt?: Date; +} + +export interface TaxonomyRevisionDocument extends PortableTaxonomyDocument { + _id: string; + taxonomyId: string; + projectId: string; + sourceType: TaxonomySourceType; + revision: number; + ref: string; + contentSha256: string; + creator?: string; + createdAt: Date; +} + +export interface TaxonomyElementRevisionDocument + extends PortableTaxonomyElementDocument { + _id: string; + elementId: string; + projectId: string; + sourceType: TaxonomySourceType; + revision: number; + ref: string; + contentSha256: string; + creator?: string; + createdAt: Date; +} diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index dec69b01d..1d5394121 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -554,6 +554,13 @@ export interface RunState { reportTemplateIds?: string[]; /** Per-handler post-processing status. Keys are handler IDs (e.g. "pp-atif"). */ handlerStatus?: Record; + /** Immutable compressed Taxonome SDK result for this exact run attempt. */ + taxonomyEvaluation?: { + url: string; + schemaVersion: string; + contentSha256: string; + sizeBytes: number; + }; /** Whole-run observation results — one entry per selected `subject:"run"` * observation criterion, evaluated once against the final snapshot + the * merged trajectory across all iterations by pp-taxonomy. Per-iteration diff --git a/packages/taxonomy-engine/src/taxonomy.ts b/packages/taxonomy-engine/src/taxonomy.ts index 397f822ab..43a6a7cf7 100644 --- a/packages/taxonomy-engine/src/taxonomy.ts +++ b/packages/taxonomy-engine/src/taxonomy.ts @@ -405,54 +405,20 @@ export function serializableTaxonomy(taxonomy: Omit { - const limits: Required = { - maxFileBytes: options.limits?.maxFileBytes ?? 1024 * 1024, - maxTotalBytes: options.limits?.maxTotalBytes ?? 4 * 1024 * 1024, - maxReferences: options.limits?.maxReferences ?? 100, - }; - const unresolvedManifestPath = resolve(path); - if ((await lstat(unresolvedManifestPath)).isSymbolicLink()) { - throw new Error(`Taxonomy manifest cannot be a symlink: ${path}`); - } - const manifestPath = await realpath(unresolvedManifestPath); - const root = dirname(manifestPath); - const byteState = { totalBytes: 0 }; - const document = taxonomyDocumentSchema.parse( - await readYamlFile(manifestPath, limits, byteState), +): NormalizedTaxonomy { + const document = taxonomyDocumentSchema.parse(taxonomy); + const authoredElements = elementDocuments.map((element, index) => + parseAuthoredElement(element, `stored element ${index + 1}`), ); - - const authoredElements: z.infer[] = []; - const loadedReferences = new Set(); - for (const entry of document.elements) { - const reference = elementReferenceSchema.safeParse(entry); - if (reference.success) { - if (loadedReferences.size >= limits.maxReferences) { - throw new Error(`Taxonomy exceeds maximum reference count ${limits.maxReferences}`); - } - const elementPath = await resolveElementPath(root, reference.data.path); - if (loadedReferences.has(elementPath)) { - throw new Error(`Duplicate or cyclic taxonomy element reference '${reference.data.path}'`); - } - loadedReferences.add(elementPath); - authoredElements.push( - parseAuthoredElement( - await readYamlFile(elementPath, limits, byteState), - reference.data.path, - ), - ); - } else { - authoredElements.push(parseAuthoredElement(entry, "inline element")); - } + if (document.elements.length !== authoredElements.length) { + throw new Error( + `Taxonomy declares ${document.elements.length} elements but ${authoredElements.length} were resolved`, + ); } const surfaceIds = new Set(); @@ -475,13 +441,8 @@ export async function loadTaxonomy( const output = validateOutputContract(authored.id, authored.output); if (authored.implementation.type === "prompt") { - return { - ...authored, - implementation: authored.implementation, - output, - }; + return { ...authored, implementation: authored.implementation, output }; } - const registered = options.handlers.resolve(authored.implementation.handler); const config = registered.handler.configSchema.parse(authored.implementation.config ?? {}); return { @@ -510,9 +471,65 @@ export async function loadTaxonomy( ...withoutDigest, digest: taxonomyDigest(serializableTaxonomy(withoutDigest)), }; - Object.freeze(normalized.surfaces); Object.freeze(normalized.elements); Object.freeze(normalized); return normalized; } + +/** + * Loads embedded/referenced YAML forms into one immutable normalized taxonomy. + * + * @param path - Taxonomy manifest path. + * @param options - Combined deterministic handler registry and optional limits. + */ +export async function loadTaxonomy( + path: string, + options: TaxonomyLoadOptions, +): Promise { + const limits: Required = { + maxFileBytes: options.limits?.maxFileBytes ?? 1024 * 1024, + maxTotalBytes: options.limits?.maxTotalBytes ?? 4 * 1024 * 1024, + maxReferences: options.limits?.maxReferences ?? 100, + }; + const unresolvedManifestPath = resolve(path); + if ((await lstat(unresolvedManifestPath)).isSymbolicLink()) { + throw new Error(`Taxonomy manifest cannot be a symlink: ${path}`); + } + const manifestPath = await realpath(unresolvedManifestPath); + const root = dirname(manifestPath); + const byteState = { totalBytes: 0 }; + const document = taxonomyDocumentSchema.parse( + await readYamlFile(manifestPath, limits, byteState), + ); + + const authoredElements: z.infer[] = []; + const loadedReferences = new Set(); + for (const entry of document.elements) { + const reference = elementReferenceSchema.safeParse(entry); + if (reference.success) { + if (loadedReferences.size >= limits.maxReferences) { + throw new Error(`Taxonomy exceeds maximum reference count ${limits.maxReferences}`); + } + const elementPath = await resolveElementPath(root, reference.data.path); + if (loadedReferences.has(elementPath)) { + throw new Error(`Duplicate or cyclic taxonomy element reference '${reference.data.path}'`); + } + loadedReferences.add(elementPath); + authoredElements.push( + parseAuthoredElement( + await readYamlFile(elementPath, limits, byteState), + reference.data.path, + ), + ); + } else { + authoredElements.push(parseAuthoredElement(entry, "inline element")); + } + } + + return normalizeTaxonomyDocuments( + { ...document, elements: authoredElements }, + authoredElements, + options, + ); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23710c24b..aa029f2c5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -761,6 +761,9 @@ importers: atifact: specifier: ^0.10.1 version: 0.10.1 + ax-taxonomy: + specifier: workspace:* + version: link:../../../packages/ax-taxonomy dotenv: specifier: 16.6.1 version: 16.6.1 @@ -770,6 +773,9 @@ importers: shared: specifier: workspace:* version: link:../../../packages/shared + taxonomy-engine: + specifier: workspace:* + version: link:../../../packages/taxonomy-engine telemetry: specifier: workspace:* version: link:../../../packages/telemetry From c0da1b881683fb483122019f81072fc5eae33406 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Wed, 28 Oct 2026 15:26:28 -0700 Subject: [PATCH 02/32] feat(taxonome): add portal and CLI taxonomy workflows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../__snapshots__/cli-commands.test.ts.snap | 98 +++++ apps/cli/src/commands/experiment.ts | 13 + apps/cli/src/commands/taxonomy.ts | 403 ++++++++++++++++++ apps/cli/src/index.ts | 2 + apps/portal/src/App.tsx | 2 + apps/portal/src/components/FeatureRoute.tsx | 10 +- apps/portal/src/components/Layout.stories.tsx | 26 ++ apps/portal/src/components/Layout.tsx | 12 +- .../src/contexts/FeatureFlagContext.tsx | 10 +- apps/portal/src/lib/api.ts | 93 +++- apps/portal/src/pages/ExperimentDetail.tsx | 70 ++- apps/portal/src/pages/RunDetail.tsx | 25 +- apps/portal/src/pages/SubmitRun.tsx | 43 ++ apps/portal/src/pages/TaxonomyList.tsx | 258 +++++++++++ apps/portal/src/types.ts | 57 +++ 15 files changed, 1111 insertions(+), 11 deletions(-) create mode 100644 apps/cli/src/commands/taxonomy.ts create mode 100644 apps/portal/src/pages/TaxonomyList.tsx diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index ba858e01b..0b82b65a2 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -240,6 +240,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--resources", "--scenario", "--skills", + "--taxonomy-revision", "--traits", "--url", "--worker", @@ -1211,6 +1212,103 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` }, }, }, + "taxonomy": { + "subcommands": { + "copy": { + "options": [ + "--id", + "--name", + "--project", + "--url", + ], + }, + "create": { + "options": [ + "--description", + "--file", + "--name", + "--project", + "--url", + ], + }, + "delete": { + "options": [ + "--url", + ], + }, + "element": { + "subcommands": { + "create": { + "options": [ + "--file", + "--name", + "--project", + "--url", + ], + }, + "list": { + "options": [ + "--project", + "--url", + ], + }, + "revise": { + "options": [ + "--file", + "--url", + ], + }, + "revisions": { + "options": [ + "--url", + ], + }, + }, + }, + "export": { + "options": [ + "--output", + "--url", + ], + }, + "import": { + "options": [ + "--element", + "--name", + "--project", + "--taxonomy", + "--url", + ], + }, + "list": { + "options": [ + "--project", + "--url", + ], + }, + "restore": { + "options": [ + "--url", + ], + }, + "revise": { + "options": [ + "--file", + "--url", + ], + }, + "revisions": { + "options": [ + "--url", + ], + }, + "show": { + "options": [ + "--url", + ], + }, + }, + }, "update": { "options": [ "--force", diff --git a/apps/cli/src/commands/experiment.ts b/apps/cli/src/commands/experiment.ts index 29e60ffee..8f2210204 100644 --- a/apps/cli/src/commands/experiment.ts +++ b/apps/cli/src/commands/experiment.ts @@ -691,6 +691,10 @@ export function registerExperimentCommands(program: Command): void { collectRepeatable, [], ) + .option( + "--taxonomy-revision ", + "Pin one exact taxonomy revision database ID", + ) .option("--empty", "Create an explicit empty experiment") .option( "--request-id ", @@ -763,6 +767,9 @@ export function registerExperimentCommands(program: Command): void { requests: [], requestIds: [], reportTemplateIds: options.reportTemplate as string[], + ...(options.taxonomyRevision + ? { taxonomyRevisionId: options.taxonomyRevision as string } + : {}), } : requestIds.length > 0 ? { @@ -771,6 +778,9 @@ export function registerExperimentCommands(program: Command): void { requests: [], requestIds, reportTemplateIds: options.reportTemplate as string[], + ...(options.taxonomyRevision + ? { taxonomyRevisionId: options.taxonomyRevision as string } + : {}), } : { mode: "compose" as const, @@ -778,6 +788,9 @@ export function registerExperimentCommands(program: Command): void { requests: [built!.request], requestIds: [], reportTemplateIds: options.reportTemplate as string[], + ...(options.taxonomyRevision + ? { taxonomyRevisionId: options.taxonomyRevision as string } + : {}), }; const expandedRunCount = built ? built.profileCount * built.occurrences diff --git a/apps/cli/src/commands/taxonomy.ts b/apps/cli/src/commands/taxonomy.ts new file mode 100644 index 000000000..eddf3db06 --- /dev/null +++ b/apps/cli/src/commands/taxonomy.ts @@ -0,0 +1,403 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { Command } from "commander"; +import { parse as parseYaml } from "yaml"; +import { configureHelp } from "../utils/helpFormatter.js"; +import { + getDefaultApiUrl, + normalizeUrl, + withProjectOption, +} from "../utils/shared.js"; +import { requireProjectId } from "../utils/config.js"; +import { errorText, successText } from "../utils/style.js"; + +interface ApiError { + error?: string; +} + +async function responseBody(response: Response): Promise { + const text = await response.text(); + if (!text) return undefined; + try { + return JSON.parse(text) as unknown; + } catch { + return text; + } +} + +async function request( + baseUrl: string, + path: string, + init?: RequestInit, +): Promise { + const response = await fetch(`${normalizeUrl(baseUrl)}/api/v1${path}`, init); + const body = await responseBody(response); + if (!response.ok) { + const error = body as ApiError | undefined; + throw new Error(error?.error ?? `HTTP ${response.status}`); + } + return body; +} + +function projectPath(path: string, projectId: string): string { + const separator = path.includes("?") ? "&" : "?"; + return `${path}${separator}projectId=${encodeURIComponent(projectId)}`; +} + +function print(value: unknown): void { + console.log(JSON.stringify(value, null, 2)); +} + +function contentFromFile(path: string): unknown { + return parseYaml(readFileSync(resolve(path), "utf8")) as unknown; +} + +/** Registers API-backed taxonomy and taxonomy-element commands. */ +export function registerTaxonomyCommands(program: Command): void { + const taxonomy = program + .command("taxonomy") + .description("Manage Taxonome taxonomies and immutable revisions") + .action(() => taxonomy.help()); + configureHelp(taxonomy); + + withProjectOption( + taxonomy + .command("list") + .description("List project taxonomies") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + print( + await request( + options.url, + projectPath("/taxonomies", requireProjectId(options.project)), + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("show") + .description("Get a taxonomy root by database ID") + .argument("", "Taxonomy root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print(await request(options.url, `/taxonomies/${encodeURIComponent(id)}`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("create") + .description("Create a user taxonomy with revision 1") + .requiredOption("--name ", "Display name") + .requiredOption("--file ", "Portable taxonomy YAML") + .option("--description ", "Description") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + const body = { + name: options.name, + ...(options.description ? { description: options.description } : {}), + sourceType: "user", + content: contentFromFile(options.file), + }; + print( + await request( + options.url, + projectPath("/taxonomies", requireProjectId(options.project)), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("import") + .description("Import taxonomy YAML and referenced element YAML") + .requiredOption("--name ", "Display name") + .requiredOption("--taxonomy ", "Taxonomy YAML path") + .option( + "--element ", + "Map a manifest path to an element YAML file (repeatable)", + (value, previous: string[]) => [...previous, value], + [], + ) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + const elements = Object.fromEntries( + (options.element as string[]).map((mapping) => { + const separator = mapping.indexOf("="); + if (separator <= 0) throw new Error(`Invalid --element mapping '${mapping}'`); + return [ + mapping.slice(0, separator), + readFileSync(resolve(mapping.slice(separator + 1)), "utf8"), + ]; + }), + ); + print( + await request( + options.url, + projectPath("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/taxonomies/import", requireProjectId(options.project)), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: options.name, + taxonomyYaml: readFileSync(resolve(options.taxonomy), "utf8"), + elements, + }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("revisions") + .description("List immutable revisions") + .argument("", "Taxonomy root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomies/${encodeURIComponent(id)}/revisions`, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("revise") + .description("Append an immutable user taxonomy revision") + .argument("", "Taxonomy root database ID") + .requiredOption("--file ", "Portable taxonomy YAML") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomies/${encodeURIComponent(id)}/revisions`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: contentFromFile(options.file) }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("copy") + .description("Copy an exact built-in revision to a user taxonomy") + .argument("", "Built-in taxonomy revision database ID") + .requiredOption("--name ", "New taxonomy name") + .option("--id ", "Portable taxonomy business ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (revisionId, options) => { + try { + print( + await request( + options.url, + projectPath( + `/taxonomy-revisions/${encodeURIComponent(revisionId)}/copy`, + requireProjectId(options.project), + ), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: options.name, + ...(options.id ? { id: options.id } : {}), + }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("export") + .description("Export an exact taxonomy revision as YAML") + .argument("", "Taxonomy revision database ID") + .requiredOption("-o, --output ", "Destination YAML file") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (revisionId, options) => { + try { + const response = await fetch( + `${normalizeUrl(options.url)}/api/v1/taxonomy-revisions/${encodeURIComponent(revisionId)}/export`, + ); + if (!response.ok) throw new Error(String(await responseBody(response))); + writeFileSync(resolve(options.output), await response.text(), "utf8"); + console.log(successText(`Exported taxonomy revision to ${options.output}`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + for (const action of ["delete", "restore"] as const) { + taxonomy + .command(action) + .description(`${action === "delete" ? "Soft-delete" : "Restore"} a user taxonomy`) + .argument("", "Taxonomy root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + const path = + action === "delete" + ? `/taxonomies/${encodeURIComponent(id)}` + : `/taxonomies/${encodeURIComponent(id)}/restore`; + const result = await request(options.url, path, { + method: action === "delete" ? "DELETE" : "POST", + }); + if (result !== undefined) print(result); + else console.log(successText(`Taxonomy ${action}d.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + } + + const element = taxonomy + .command("element") + .description("Manage taxonomy elements") + .action(() => element.help()); + configureHelp(element); + + withProjectOption( + element + .command("list") + .description("List project taxonomy elements") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + print( + await request( + options.url, + projectPath( + "/taxonomy-elements", + requireProjectId(options.project), + ), + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + element + .command("create") + .description("Create a user taxonomy element with revision 1") + .requiredOption("--name ", "Display name") + .requiredOption("--file ", "Portable element YAML") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + print( + await request( + options.url, + projectPath( + "/taxonomy-elements", + requireProjectId(options.project), + ), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: options.name, + sourceType: "user", + content: contentFromFile(options.file), + }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + element + .command("revisions") + .description("List immutable taxonomy element revisions") + .argument("", "Element root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomy-elements/${encodeURIComponent(id)}/revisions`, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + element + .command("revise") + .description("Append an immutable user element revision") + .argument("", "Element root database ID") + .requiredOption("--file ", "Portable element YAML") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomy-elements/${encodeURIComponent(id)}/revisions`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: contentFromFile(options.file) }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index a9278391a..c235fdfc0 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -26,6 +26,7 @@ import { registerProfileCommands } from "./commands/profile.js"; import { registerProjectCommands } from "./commands/project.js"; import { registerSecretCommands } from "./commands/secret.js"; import { registerExperimentCommands } from "./commands/experiment.js"; +import { registerTaxonomyCommands } from "./commands/taxonomy.js"; import { registerUpdateCommand } from "./commands/update.js"; import { checkForUpdates } from "./utils/update-check.js"; import { errorText } from "./utils/style.js"; @@ -72,6 +73,7 @@ registerSecretCommands(program); registerProjectCommands(program); registerRunCommands(program); registerExperimentCommands(program); +registerTaxonomyCommands(program); registerCriteriaCommands(program); registerPromptFeatureCommands(program); registerReportCommands(program); diff --git a/apps/portal/src/App.tsx b/apps/portal/src/App.tsx index 195a75400..6612b3121 100644 --- a/apps/portal/src/App.tsx +++ b/apps/portal/src/App.tsx @@ -67,6 +67,7 @@ import { FeatureRoute } from "@/components/FeatureRoute"; import { ProjectGate } from "@/components/ProjectGate"; import { HomeRoute } from "@/components/HomeRoute"; import { useFavicon } from "@/hooks/useFavicon"; +import { TaxonomyList } from "@/pages/TaxonomyList"; export function App() { useFavicon(); @@ -146,6 +147,7 @@ export function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/apps/portal/src/components/FeatureRoute.tsx b/apps/portal/src/components/FeatureRoute.tsx index 387d73e86..e2cd2aff2 100644 --- a/apps/portal/src/components/FeatureRoute.tsx +++ b/apps/portal/src/components/FeatureRoute.tsx @@ -7,14 +7,18 @@ import type { ReactNode } from "react"; interface FeatureRouteProps { featureKey: string; + failClosed?: boolean; children: ReactNode; } /** Route guard that redirects to /statistics when a feature flag is disabled */ -export function FeatureRoute({ featureKey, children }: FeatureRouteProps) { - const { isFeatureEnabled } = useFeatureFlags(); +export function FeatureRoute({ featureKey, failClosed = false, children }: FeatureRouteProps) { + const { isFeatureEnabled, isFeatureEnabledFailClosed } = useFeatureFlags(); - if (!isFeatureEnabled(featureKey)) { + const enabled = failClosed + ? isFeatureEnabledFailClosed(featureKey) + : isFeatureEnabled(featureKey); + if (!enabled) { return ; } diff --git a/apps/portal/src/components/Layout.stories.tsx b/apps/portal/src/components/Layout.stories.tsx index d48655a8b..d97b158b5 100644 --- a/apps/portal/src/components/Layout.stories.tsx +++ b/apps/portal/src/components/Layout.stories.tsx @@ -123,6 +123,32 @@ export const Expanded: Story = { }, }; +export const TaxonomeEnabled: Story = { + parameters: { + msw: { + handlers: [ + ...mswHandlers, + http.get("/api/v1/feature-flags", () => + HttpResponse.json([ + { + key: "taxonome", + label: "Taxonome and Taxonomies", + enabled: true, + updatedAt: "2026-10-28T00:00:00.000Z", + }, + ]), + ), + ], + }, + }, + play: async ({ canvas }) => { + localStorage.setItem("scope:layout:sidebar-expanded", "true"); + await expect( + await canvas.findByRole("link", { name: "Taxonomies" }), + ).toHaveAttribute("href", "/taxonomies"); + }, +}; + // No project selected: only global entries (Projects + Platform) remain; the // scoped groups and the New Run CTA are hidden until a project is picked. export const NoProjectSelected: Story = { diff --git a/apps/portal/src/components/Layout.tsx b/apps/portal/src/components/Layout.tsx index ba51d5f19..d295c221b 100644 --- a/apps/portal/src/components/Layout.tsx +++ b/apps/portal/src/components/Layout.tsx @@ -32,6 +32,7 @@ import { SlidersHorizontal, PanelLeftClose, PanelLeftOpen, + Network, } from "lucide-react"; import { Button } from "@/components/ui/button"; import { @@ -59,6 +60,7 @@ interface NavItem { icon: React.ComponentType<{ className?: string }>; /** When set, this nav item is only shown if the corresponding feature flag is enabled */ featureKey?: string; + failClosed?: boolean; } interface NavGroup { @@ -114,6 +116,7 @@ const navGroups: NavGroup[] = [ { to: "/profiles", label: "Profiles", icon: SlidersHorizontal, featureKey: "profiles" }, { to: "/skills", label: "Skills", icon: BookOpen, featureKey: "skills" }, { to: "/codebases", label: "Codebases", icon: FolderGit2 }, + { to: "/taxonomies", label: "Taxonomies", icon: Network, featureKey: "taxonome", failClosed: true }, ], }, { @@ -579,7 +582,7 @@ function NewWorkflowLauncher({ export function Layout() { const location = useLocation(); - const { isFeatureEnabled } = useFeatureFlags(); + const { isFeatureEnabled, isFeatureEnabledFailClosed } = useFeatureFlags(); const { hasProject } = useProjectContext(); const [mobileOpen, setMobileOpen] = useState(false); const [sidebarExpanded, setSidebarExpanded] = useState(() => { @@ -599,7 +602,12 @@ export function Layout() { }, [sidebarExpanded]); const filterByFeature = (items: NavItem[]) => - items.filter((item) => !item.featureKey || isFeatureEnabled(item.featureKey)); + items.filter((item) => { + if (!item.featureKey) return true; + return item.failClosed + ? isFeatureEnabledFailClosed(item.featureKey) + : isFeatureEnabled(item.featureKey); + }); const visibleGroups = useMemo( () => diff --git a/apps/portal/src/contexts/FeatureFlagContext.tsx b/apps/portal/src/contexts/FeatureFlagContext.tsx index 640248c1e..49d334da3 100644 --- a/apps/portal/src/contexts/FeatureFlagContext.tsx +++ b/apps/portal/src/contexts/FeatureFlagContext.tsx @@ -15,12 +15,15 @@ interface FeatureFlagContextValue { isLoading: boolean; /** Check if a feature is enabled. Returns true by default (fail-open) if flags haven't loaded. */ isFeatureEnabled: (key: string) => boolean; + /** Fail-closed check for sensitive preview features. */ + isFeatureEnabledFailClosed: (key: string) => boolean; } const FeatureFlagContext = createContext({ flags: [], isLoading: true, isFeatureEnabled: () => true, + isFeatureEnabledFailClosed: () => false, }); export function FeatureFlagProvider({ children }: { children: ReactNode }) { @@ -41,8 +44,13 @@ export function FeatureFlagProvider({ children }: { children: ReactNode }) { return flag?.enabled ?? true; } + function isFeatureEnabledFailClosed(key: string): boolean { + if (isLoading) return false; + return flags.find((flag) => flag.key === key)?.enabled === true; + } + return ( - + {children} ); diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index f42d89082..00ee7e732 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTemplateVersion, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject } from "@/types"; +import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTemplateVersion, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, TaxonomyDocument, TaxonomyRevisionDocument, TaxonomyElementDocument, TaxonomyElementRevisionDocument, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject } from "@/types"; import type { Project, CreateProjectRequest, UpdateProjectRequest } from "@/types"; import { qs } from "./url"; @@ -277,7 +277,7 @@ export const api = { }, createExperiment: async ( - body: { metadata: ExperimentMetadataView; reportTemplateIds?: string[] } & ( + body: { metadata: ExperimentMetadataView; reportTemplateIds?: string[]; taxonomyRevisionId?: string } & ( | { request: RunCreateInput; empty?: never } | { empty: true; request?: never } ), @@ -299,6 +299,9 @@ export const api = { metadata: body.metadata, requests: requestBody ? [requestBody] : [], reportTemplateIds: body.reportTemplateIds ?? [], + ...(body.taxonomyRevisionId + ? { taxonomyRevisionId: body.taxonomyRevisionId } + : {}), }), }, { scoped: true }, @@ -313,6 +316,7 @@ export const api = { metadata: ExperimentMetadataView; requestIds: string[]; reportTemplateIds?: string[]; + taxonomyRevisionId?: string; }): Promise => { const value = await request( "/experiments", @@ -324,6 +328,9 @@ export const api = { requests: [], requestIds: body.requestIds, reportTemplateIds: body.reportTemplateIds ?? [], + ...(body.taxonomyRevisionId + ? { taxonomyRevisionId: body.taxonomyRevisionId } + : {}), }), }, { scoped: true }, @@ -1395,6 +1402,88 @@ export const api = { }); }, + // ─── Taxonomies ──────────────────────────────────────────────────────────── + + listTaxonomies: (): Promise => { + return request("/taxonomies", undefined, { scoped: true }); + }, + + getTaxonomy: (id: string): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}`); + }, + + listTaxonomyRevisions: (id: string): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}/revisions`); + }, + + createTaxonomy: (input: { + name: string; + description?: string; + content: unknown; + }): Promise => { + return request("/taxonomies", { + method: "POST", + body: JSON.stringify({ ...input, sourceType: "user" }), + }, { scoped: true }); + }, + + copyTaxonomyRevision: ( + revisionId: string, + input: { name: string; id?: string }, + ): Promise => { + return request( + `/taxonomy-revisions/${encodeURIComponent(revisionId)}/copy`, + { method: "POST", body: JSON.stringify(input) }, + { scoped: true }, + ); + }, + + deleteTaxonomy: (id: string): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}`, { + method: "DELETE", + }); + }, + + listTaxonomyElements: (): Promise => { + return request("/taxonomy-elements", undefined, { scoped: true }); + }, + + listTaxonomyElementRevisions: ( + id: string, + ): Promise => { + return request(`/taxonomy-elements/${encodeURIComponent(id)}/revisions`); + }, + + createTaxonomyElementRevision: ( + id: string, + content: unknown, + ): Promise => { + return request(`/taxonomy-elements/${encodeURIComponent(id)}/revisions`, { + method: "POST", + body: JSON.stringify({ content }), + }); + }, + + getTaxonomyReadout: (experimentId: string): Promise<{ + status: "processing" | "done" | "failed"; + jsonUrl?: string; + markdownUrl?: string; + error?: string; + }> => { + return request(`/experiments/${encodeURIComponent(experimentId)}/taxonomy-readout`, undefined, { scoped: true }); + }, + + retryTaxonomyReadout: (experimentId: string): Promise<{ + status: "processing" | "done" | "failed"; + jsonUrl?: string; + markdownUrl?: string; + error?: string; + }> => { + return request(`/experiments/${encodeURIComponent(experimentId)}/taxonomy-readout/retry`, { + method: "POST", + }, { scoped: true }); + }, + // ─── Skills ─────────────────────────────────────────────────────────────── /** List all imported skills */ diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx index a7b1c3c6e..ddc17b749 100644 --- a/apps/portal/src/pages/ExperimentDetail.tsx +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -64,6 +64,7 @@ import type { } from "@/lib/experiments"; import { formatDate, truncate } from "@/lib/utils"; import type { ReportTemplate } from "@/types"; +import { useFeatureFlags } from "@/contexts/FeatureFlagContext"; const RUN_COLUMN_OPTIONS: CustomizeColumnsOption[] = [ { id: "requestId", label: "ID", required: true }, @@ -278,6 +279,8 @@ export function ExperimentDetail() { const { id } = useParams<{ id: string }>(); const navigate = useNavigate(); const queryClient = useQueryClient(); + const { isFeatureEnabledFailClosed } = useFeatureFlags(); + const taxonomeEnabled = isFeatureEnabledFailClosed("taxonome"); const [searchParams, setSearchParams] = useSearchParams(); const [selectedRequestIds, setSelectedRequestIds] = useState>(new Set()); const [removeDialogOpen, setRemoveDialogOpen] = useState(false); @@ -373,6 +376,23 @@ export function ExperimentDetail() { return 5_000; }, }); + const taxonomyReadoutQuery = useQuery({ + queryKey: ["experiment-taxonomy-readout", id], + queryFn: () => api.getTaxonomyReadout(id!), + enabled: Boolean(id && taxonomeEnabled && !isViewingHistorical), + retry: false, + }); + const retryTaxonomyReadout = useMutation({ + mutationFn: () => api.retryTaxonomyReadout(id!), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: ["experiment-taxonomy-readout", id], + }); + toast.success("Taxonome readout generated"); + }, + onError: (error) => + toast.error(error instanceof Error ? error.message : "Readout failed"), + }); const retry = useMutation({ mutationFn: () => api.retryExperiment( @@ -1188,7 +1208,55 @@ export function ExperimentDetail() { - + + {taxonomeEnabled && !isViewingHistorical && ( + + + + Taxonome readout + + + + + {taxonomyReadoutQuery.data ? ( + <> + + {taxonomyReadoutQuery.data.status} + + {taxonomyReadoutQuery.data.error && ( +

{taxonomyReadoutQuery.data.error}

+ )} + {taxonomyReadoutQuery.data.status === "done" && ( + + )} + + ) : ( +

+ Generate a typed readout after exact-run taxonomy evaluations finish. +

+ )} +
+
+ )} (turn.observationResults?.length ?? 0) > 0) || (activeRun?.observationResults?.length ?? 0) > 0; const hasConfiguredObservations = (run?.observations?.length ?? 0) > 0; + const hasTaxonomyEvaluation = Boolean(activeRun?.taxonomyEvaluation); // Aggregate enrichment status across the whole post-processing DAG so the // "Enriched" badge waits for pp-observations instead of flipping after pp-atif. const enrichmentStatus = useMemo( @@ -1178,7 +1179,7 @@ export function RunDetail() { {hasHarData && Network} {hasHarData && Tool Calls} {hasVideoData && } - {(hasConfiguredObservations || hasObservationResults) && ( + {(hasConfiguredObservations || hasObservationResults || hasTaxonomyEvaluation) && ( Taxonomy {hasObservationResults ? `(${((activeRun?.turns?.reduce((count, turn) => count + (turn.observationResults?.length ?? 0), 0) ?? 0) + (activeRun?.observationResults?.length ?? 0))})` : ""} @@ -1278,8 +1279,28 @@ export function RunDetail() {
)} - {(hasConfiguredObservations || hasObservationResults) && ( + {(hasConfiguredObservations || hasObservationResults || hasTaxonomyEvaluation) && ( + {activeRun?.taxonomyEvaluation && ( +
+
+

Taxonome evaluation

+

+ Full SDK result for this exact run attempt. +

+
+
+ {activeRun.taxonomyEvaluation.schemaVersion} + {activeRun.taxonomyEvaluation.sizeBytes.toLocaleString()} bytes + {activeRun.taxonomyEvaluation.contentSha256.slice(0, 12)} + +
+
+ )}

Observations

diff --git a/apps/portal/src/pages/SubmitRun.tsx b/apps/portal/src/pages/SubmitRun.tsx index c352bbf57..3f9202898 100644 --- a/apps/portal/src/pages/SubmitRun.tsx +++ b/apps/portal/src/pages/SubmitRun.tsx @@ -43,6 +43,7 @@ import { } from "@/components/ReasoningEffortSelect"; import { TaskPromptPicker } from "@/components/TaskPromptPicker"; import { useCommandEnter } from "@/hooks/useCommandEnter"; +import { useFeatureFlags } from "@/contexts/FeatureFlagContext"; import { useVisibleGates } from "@/hooks/useVisibleGates"; import { useStrictAgentCapabilities } from "@/hooks/useStrictAgentCapabilities"; import { KbdBadge } from "@/components/KbdBadge"; @@ -224,6 +225,7 @@ export function SubmitRun({ const [experimentDescription, setExperimentDescription] = useState(""); const [experimentReportTemplateIds, setExperimentReportTemplateIds] = useState([]); + const [taxonomyRevisionId, setTaxonomyRevisionId] = useState(""); const [experimentDescriptionOpen, setExperimentDescriptionOpen] = useState(false); @@ -270,6 +272,8 @@ export function SubmitRun({ const [generateDescription, setGenerateDescription] = useState(""); // ─── Queries ──────────────────────────────────────────────────────────── + const { isFeatureEnabledFailClosed } = useFeatureFlags(); + const taxonomeEnabled = isFeatureEnabledFailClosed("taxonome"); const { data: agents = [], isSuccess: agentsLoaded } = useQuery({ queryKey: ["agents", "include-deleted"], queryFn: () => api.listAgents({ includeDeleted: true }), @@ -292,6 +296,11 @@ export function SubmitRun({ const manualExperimentReportTemplates = experimentReportTemplates.filter( (template) => !template.trigger, ); + const { data: taxonomies = [] } = useQuery({ + queryKey: ["taxonomies"], + queryFn: api.listTaxonomies, + enabled: mode === "experiment" && taxonomeEnabled, + }); const { data: profileVersions = [] } = useQuery({ queryKey: ["profile-versions", selectedProfileId], @@ -704,6 +713,7 @@ export function SubmitRun({ : {}), }, reportTemplateIds: experimentReportTemplateIds, + ...(taxonomyRevisionId ? { taxonomyRevisionId } : {}), ...(request ? { request } : { empty: true }), }) : api.submitRun(request!), @@ -927,6 +937,39 @@ export function SubmitRun({ +
+ {taxonomeEnabled && ( + <> + + +

+ The exact immutable revision is pinned to this Experiment. +

+ + )} +
(); + const [editingElement, setEditingElement] = useState<{ + root: TaxonomyElementDocument; + revision: TaxonomyElementRevisionDocument; + }>(); + const [prompt, setPrompt] = useState(""); + const { data: taxonomies = [], isLoading } = useQuery({ + queryKey: ["taxonomies"], + queryFn: api.listTaxonomies, + }); + const { data: revisions = [] } = useQuery({ + queryKey: ["taxonomies", expanded, "revisions"], + queryFn: () => api.listTaxonomyRevisions(expanded!), + enabled: Boolean(expanded), + }); + const { data: elements = [] } = useQuery({ + queryKey: ["taxonomy-elements"], + queryFn: api.listTaxonomyElements, + }); + + const create = useMutation({ + mutationFn: () => + api.createTaxonomy({ name, content: parseYaml(yaml) as unknown }), + onSuccess: () => { + setCreateOpen(false); + setName(""); + queryClient.invalidateQueries({ queryKey: ["taxonomies"] }); + toast.success("Taxonomy created"); + }, + onError: (error) => + toast.error(error instanceof Error ? error.message : "Creation failed"), + }); + const remove = useMutation({ + mutationFn: api.deleteTaxonomy, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["taxonomies"] }); + toast.success("Taxonomy deleted"); + }, + }); + const copy = useMutation({ + mutationFn: async (taxonomy: TaxonomyDocument) => { + const exact = await api.listTaxonomyRevisions(taxonomy._id); + const latest = exact[0]; + if (!latest) throw new Error("Built-in taxonomy has no revision"); + return api.copyTaxonomyRevision(latest._id, { + name: `${taxonomy.name} copy`, + id: `${taxonomy.id}-copy`, + }); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["taxonomies"] }); + toast.success("Built-in taxonomy copied"); + }, + }); + const reviseElement = useMutation({ + mutationFn: async () => { + if (!editingElement) throw new Error("No element selected"); + return api.createTaxonomyElementRevision(editingElement.root._id, { + id: editingElement.revision.id, + label: editingElement.revision.label, + ...(editingElement.revision.description + ? { description: editingElement.revision.description } + : {}), + implementation: { type: "prompt", prompt }, + preconditions: editingElement.revision.preconditions, + output: editingElement.revision.output, + }); + }, + onSuccess: () => { + setEditingElement(undefined); + queryClient.invalidateQueries({ queryKey: ["taxonomy-elements"] }); + toast.success("Prompt element revision created"); + }, + }); + + async function editPromptElement(root: TaxonomyElementDocument) { + const exact = await api.listTaxonomyElementRevisions(root._id); + const revision = exact[0]; + if (!revision || revision.implementation.type !== "prompt") { + toast.info("Deterministic code elements are read-only"); + return; + } + setPrompt(revision.implementation.prompt); + setEditingElement({ root, revision }); + } + + return ( +
+
+
+

+ Taxonomies +

+

+ Manage project-scoped Taxonome definitions and exact immutable revisions. +

+
+ +
+ + {isLoading ? ( +

Loading taxonomies…

+ ) : ( +
+ {taxonomies.map((taxonomy) => ( + + +
+
+ + {taxonomy.name} + + {taxonomy.sourceType === "builtin" ? "Built-in · read-only" : "User"} + + +

+ {taxonomy.id}{taxonomy.latestRevision ? `@r${taxonomy.latestRevision}` : ""} +

+
+
+ + {taxonomy.sourceType === "builtin" ? ( + + ) : ( + + )} + +
+
+

Elements

+

+ Prompt elements can append revisions. Built-in and deterministic code elements are read-only. +

+
+
+ {elements.map((element) => ( + + +
+

{element.name}

+

+ {element.id}@r{element.latestRevision} +

+
+
+ + {element.sourceType === "builtin" ? "Built-in · read-only" : "User"} + + {element.sourceType === "user" && ( + + )} +
+
+
+ ))} +
+
+
+
+
+ {expanded === taxonomy._id && ( + + {revisions.map((revision) => ( +
+ {revision.ref} + {revision.contentSha256.slice(0, 12)} +
+ ))} +
+ )} +
+ ))} +
+ )} + + + + Create taxonomy +
+
+ + setName(event.target.value)} /> +
+
+ +