diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index b75d8727f..97fce3541 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -393,6 +393,117 @@ describe("API Endpoints", () => { expect(Array.isArray(res.body)).toBe(true); expect(res.body[0]).toHaveProperty("id", "r1"); }); + + it("returns grouped results when groupBy=task", async () => { + const groupedDocs = [ + { key: "tp-1", label: "Build a calculator", aggregates: { count: 3, turns: { min: 1, max: 3, mean: 2, stdDev: 0.8 }, duration: null, promptTokens: null, completionTokens: null }, uniform: { workerType: "coder-acp-copilot" } }, + ]; + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue(groupedDocs), + }); + + const res = await request(app).get("/api/v1/requests?groupBy=task"); + expect(res.status).toBe(200); + expect(Array.isArray(res.body)).toBe(true); + expect(res.body[0]).toHaveProperty("key", "tp-1"); + expect(res.body[0]).toHaveProperty("aggregates"); + expect(res.body[0].aggregates).toHaveProperty("count", 3); + expect(res.body[0]).toHaveProperty("uniform"); + }); + + it("returns grouped results when groupBy=submissionId", async () => { + const groupedDocs = [ + { key: "sub-1", label: "sub-1", aggregates: { count: 2, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, + ]; + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue(groupedDocs), + }); + + const res = await request(app).get("/api/v1/requests?groupBy=submissionId"); + expect(res.status).toBe(200); + expect(res.body[0]).toHaveProperty("key", "sub-1"); + }); + + it("calls aggregate pipeline when groupBy is provided", async () => { + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }); + + await request(app).get("/api/v1/requests?groupBy=task"); + expect(mocks.collection.aggregate).toHaveBeenCalled(); + const pipeline = (mocks.collection.aggregate as any).mock.calls[0][0]; + expect(pipeline[0]).toHaveProperty("$match"); + // buildGroupingPipeline adds 4 stages ($addFields, $group, $project, $sort) + expect(pipeline).toHaveLength(5); // $match + 4 from buildGroupingPipeline + }); + + it("does not call aggregate when groupBy is absent", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests"); + expect(mocks.collection.aggregate).not.toHaveBeenCalled(); + }); + + it("passes status filter to find query", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests?status=done"); + expect(mocks.collection.find).toHaveBeenCalledWith( + expect.objectContaining({ status: "done" }), + ); + }); + + it("passes outcome filter to find query", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests?outcome=succeeded"); + expect(mocks.collection.find).toHaveBeenCalledWith( + expect.objectContaining({ outcome: "succeeded" }), + ); + }); + + it("passes status and outcome filters to aggregate pipeline $match", async () => { + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }); + + await request(app).get("/api/v1/requests?groupBy=task&status=done&outcome=failed"); + const pipeline = (mocks.collection.aggregate as any).mock.calls[0][0]; + expect(pipeline[0]).toEqual( + expect.objectContaining({ + $match: expect.objectContaining({ status: "done", outcome: "failed" }), + }), + ); + }); + + it("combines status, outcome, and worker filters", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests?status=processing&outcome=succeeded&worker=coder-acp-copilot"); + expect(mocks.collection.find).toHaveBeenCalledWith( + expect.objectContaining({ + status: "processing", + outcome: "succeeded", + workerType: "coder-acp-copilot", + }), + ); + }); }); describe("GET /api/v1/requests/:id", () => { diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts new file mode 100644 index 000000000..088070a4c --- /dev/null +++ b/apps/api/src/grouping.test.ts @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect } from "vitest"; +import { buildGroupingPipeline } from "./grouping.js"; + +describe("buildGroupingPipeline", () => { + it("returns 4 pipeline stages for task grouping", () => { + const pipeline = buildGroupingPipeline("task"); + expect(pipeline).toHaveLength(4); + expect(pipeline[0]).toHaveProperty("$addFields"); + expect(pipeline[1]).toHaveProperty("$group"); + expect(pipeline[2]).toHaveProperty("$project"); + expect(pipeline[3]).toHaveProperty("$sort"); + }); + + it("returns 4 pipeline stages for submissionId grouping", () => { + const pipeline = buildGroupingPipeline("submissionId"); + expect(pipeline).toHaveLength(4); + }); + + it("$sort stage orders by key ascending", () => { + const pipeline = buildGroupingPipeline("task"); + expect(pipeline[3]).toEqual({ $sort: { key: 1 } }); + }); + + it("groups by taskPromptId for task grouping", () => { + const pipeline = buildGroupingPipeline("task"); + const addFields = pipeline[0].$addFields as Record; + // _groupKey should reference $taskPromptId with fallback to scenario.task + expect(addFields._groupKey).toBeDefined(); + }); + + it("groups by submissionId for submissionId grouping", () => { + const pipeline = buildGroupingPipeline("submissionId"); + const addFields = pipeline[0].$addFields as Record; + expect(addFields._groupKey).toBeDefined(); + }); + + it("$group stage uses _groupKey as _id", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._id).toBe("$_groupKey"); + }); + + it("$group stage accumulates count and stat arrays", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group.count).toEqual({ $sum: 1 }); + expect(group._turnCounts).toBeDefined(); + expect(group._durations).toBeDefined(); + expect(group._promptTokensList).toBeDefined(); + expect(group._completionTokensList).toBeDefined(); + }); + + it("$group stage accumulates status and outcome counts", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._statusPending).toBeDefined(); + expect(group._statusProcessing).toBeDefined(); + expect(group._statusDone).toBeDefined(); + expect(group._outcomeSucceeded).toBeDefined(); + expect(group._outcomeFailed).toBeDefined(); + expect(group._outcomeFinished).toBeDefined(); + }); + + it("$group stage collects distinct values for uniform detection", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._workerTypes).toBeDefined(); + expect(group._models).toBeDefined(); + expect(group._agentVersions).toBeDefined(); + expect(group._platforms).toBeDefined(); + expect(group._statuses).toBeDefined(); + expect(group._submissionIds).toBeDefined(); + expect(group._tasks).toBeDefined(); + expect(group._mcpKeys).toBeDefined(); + expect(group._skillKeys).toBeDefined(); + }); + + it("$group stage collects run IDs", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._runIds).toEqual({ $push: "$_id" }); + }); + + it("$project stage outputs key, label, runIds, aggregates, uniform", () => { + const pipeline = buildGroupingPipeline("task"); + const project = (pipeline[2] as { $project: Record }).$project; + expect(project._id).toBe(0); + expect(project.key).toBeDefined(); + expect(project.runIds).toBe("$_runIds"); + expect(project.label).toBeDefined(); + expect(project.aggregates).toBeDefined(); + expect(project.uniform).toBeDefined(); + }); + + it("$project aggregates include statusCounts and outcomeCounts", () => { + const pipeline = buildGroupingPipeline("task"); + const project = pipeline[2].$project as Record; + const aggregates = project.aggregates as Record; + expect(aggregates.statusCounts).toEqual({ pending: "$_statusPending", processing: "$_statusProcessing", done: "$_statusDone" }); + expect(aggregates.outcomeCounts).toEqual({ succeeded: "$_outcomeSucceeded", failed: "$_outcomeFailed", finished: "$_outcomeFinished" }); + }); +}); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts new file mode 100644 index 000000000..bb3d3add8 --- /dev/null +++ b/apps/api/src/grouping.ts @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Document } from "mongodb"; +import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "shared"; + +/** + * Build a MongoDB aggregation pipeline that groups runs and computes + * per-group aggregates (count, stats on turns/duration/tokens) and + * uniform-value detection. + * + * The pipeline is appended after a $match stage with the caller's filter. + */ +export function buildGroupingPipeline( + groupBy: "task" | "submissionId", +): Document[] { + const groupField = + groupBy === "task" ? "$taskPromptId" : "$submissionId"; + const fallbackGroupField = + groupBy === "task" ? "$scenario.task" : { $literal: "no-submission" }; + + return [ + // 1. Compute per-run derived values + { + $addFields: { + _groupKey: { $ifNull: [groupField, fallbackGroupField] }, + _turnCount: { $cond: { if: { $isArray: "$turns" }, then: { $size: "$turns" }, else: null } }, + _duration: { + $cond: { + if: { $and: [{ $isArray: "$turns" }, { $gt: [{ $size: "$turns" }, 0] }] }, + then: { + $reduce: { + input: "$turns", + initialValue: 0, + in: { $add: ["$$value", { $ifNull: ["$$this.durationMs", 0] }] }, + }, + }, + else: null, + }, + }, + _promptTokens: { + $cond: { + if: "$tokenUsage", + then: "$tokenUsage.promptTokens", + else: { + $cond: { + if: { $and: [{ $isArray: "$turns" }, { $gt: [{ $size: { $filter: { input: { $ifNull: ["$turns", []] }, cond: { $ne: ["$$this.tokenUsage", null] } } } }, 0] }] }, + then: { + $reduce: { + input: "$turns", + initialValue: 0, + in: { $add: ["$$value", { $ifNull: ["$$this.tokenUsage.promptTokens", 0] }] }, + }, + }, + else: null, + }, + }, + }, + }, + _completionTokens: { + $cond: { + if: "$tokenUsage", + then: "$tokenUsage.completionTokens", + else: { + $cond: { + if: { $and: [{ $isArray: "$turns" }, { $gt: [{ $size: { $filter: { input: { $ifNull: ["$turns", []] }, cond: { $ne: ["$$this.tokenUsage", null] } } } }, 0] }] }, + then: { + $reduce: { + input: "$turns", + initialValue: 0, + in: { $add: ["$$value", { $ifNull: ["$$this.tokenUsage.completionTokens", 0] }] }, + }, + }, + else: null, + }, + }, + }, + }, + // Sorted-joined array keys for uniform comparison + _mcpKey: { + $reduce: { + input: { $sortArray: { input: { $ifNull: ["$mcpServers", []] }, sortBy: 1 } }, + initialValue: "", + in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, + }, + }, + _skillKey: { + $reduce: { + input: { $sortArray: { input: { $ifNull: ["$skillRevisions", []] }, sortBy: 1 } }, + initialValue: "", + in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, + }, + }, + _extensionKey: { + $reduce: { + input: { $sortArray: { input: { $ifNull: ["$extensions", []] }, sortBy: 1 } }, + initialValue: "", + in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, + }, + }, + }, + }, + // 2. Group and accumulate per-run values + { + $group: { + _id: "$_groupKey", + count: { $sum: 1 }, + // Collect run IDs for report summaries and bulk actions + _runIds: { $push: "$_id" }, + // Label: take first task name for display + _firstTask: { $first: "$scenario.task" }, + // Push arrays for stats computation + _turnCounts: { $push: "$_turnCount" }, + _durations: { $push: "$_duration" }, + _promptTokensList: { $push: "$_promptTokens" }, + _completionTokensList: { $push: "$_completionTokens" }, + // Uniform detection: collect distinct values + _workerTypes: { $addToSet: "$workerType" }, + _models: { $addToSet: { $ifNull: ["$model", ""] } }, + _agentVersions: { $addToSet: { $ifNull: ["$agentVersion", ""] } }, + _platforms: { $addToSet: { $ifNull: ["$os.platform", ""] } }, + _statuses: { $addToSet: "$status" }, + _submissionIds: { $addToSet: { $ifNull: ["$submissionId", ""] } }, + _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, + _mcpKeys: { $addToSet: "$_mcpKey" }, + _skillKeys: { $addToSet: "$_skillKey" }, + _extensionKeys: { $addToSet: "$_extensionKey" }, + // Status/outcome distribution counts + _statusPending: { $sum: { $cond: [{ $eq: ["$status", "pending"] }, 1, 0] } }, + _statusProcessing: { $sum: { $cond: [{ $eq: ["$status", "processing"] }, 1, 0] } }, + _statusDone: { $sum: { $cond: [{ $eq: ["$status", "done"] }, 1, 0] } }, + _outcomeSucceeded: { $sum: { $cond: [{ $eq: ["$outcome", "succeeded"] }, 1, 0] } }, + _outcomeFailed: { $sum: { $cond: [{ $eq: ["$outcome", "failed"] }, 1, 0] } }, + _outcomeFinished: { $sum: { $cond: [{ $eq: ["$outcome", "finished"] }, 1, 0] } }, + // Keep first run's array values for uniform fields + _firstMcpServers: { $first: "$mcpServers" }, + _firstSkillRevisions: { $first: "$skillRevisions" }, + _firstExtensions: { $first: "$extensions" }, + }, + }, + // 3. Project into final shape + { + $project: { + _id: 0, + key: "$_id", + runIds: "$_runIds", + label: { + $cond: { + if: { $eq: ["$_id", "no-submission"] }, + then: "No submission ID", + else: { $ifNull: ["$_firstTask", "$_id"] }, + }, + }, + aggregates: { + count: "$count", + turns: { $let: { vars: { vals: { $filter: { input: "$_turnCounts", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + duration: { $let: { vars: { vals: { $filter: { input: "$_durations", cond: { $and: [{ $ne: ["$$this", null] }, { $gt: ["$$this", 0] }] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + promptTokens: { $let: { vars: { vals: { $filter: { input: "$_promptTokensList", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + completionTokens: { $let: { vars: { vals: { $filter: { input: "$_completionTokensList", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + statusCounts: { pending: "$_statusPending", processing: "$_statusProcessing", done: "$_statusDone" }, + outcomeCounts: { succeeded: "$_outcomeSucceeded", failed: "$_outcomeFailed", finished: "$_outcomeFinished" }, + }, + uniform: { + workerType: { $cond: { if: { $eq: [{ $size: "$_workerTypes" }, 1] }, then: { $arrayElemAt: ["$_workerTypes", 0] }, else: "$$REMOVE" } }, + model: { $cond: { if: { $and: [{ $eq: [{ $size: "$_models" }, 1] }, { $ne: [{ $arrayElemAt: ["$_models", 0] }, ""] }] }, then: { $arrayElemAt: ["$_models", 0] }, else: "$$REMOVE" } }, + agentVersion: { $cond: { if: { $and: [{ $eq: [{ $size: "$_agentVersions" }, 1] }, { $ne: [{ $arrayElemAt: ["$_agentVersions", 0] }, ""] }] }, then: { $arrayElemAt: ["$_agentVersions", 0] }, else: "$$REMOVE" } }, + platform: { $cond: { if: { $and: [{ $eq: [{ $size: "$_platforms" }, 1] }, { $ne: [{ $arrayElemAt: ["$_platforms", 0] }, ""] }] }, then: { $arrayElemAt: ["$_platforms", 0] }, else: "$$REMOVE" } }, + status: { $cond: { if: { $eq: [{ $size: "$_statuses" }, 1] }, then: { $arrayElemAt: ["$_statuses", 0] }, else: "$$REMOVE" } }, + submissionId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_submissionIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_submissionIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_submissionIds", 0] }, else: "$$REMOVE" } }, + task: { $cond: { if: { $and: [{ $eq: [{ $size: "$_tasks" }, 1] }, { $ne: [{ $arrayElemAt: ["$_tasks", 0] }, ""] }] }, then: { $arrayElemAt: ["$_tasks", 0] }, else: "$$REMOVE" } }, + mcpServers: { $cond: { if: { $and: [{ $eq: [{ $size: "$_mcpKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_mcpKeys", 0] }, ""] }] }, then: "$_firstMcpServers", else: "$$REMOVE" } }, + skillRevisions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_skillKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_skillKeys", 0] }, ""] }] }, then: "$_firstSkillRevisions", else: "$$REMOVE" } }, + extensions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_extensionKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_extensionKeys", 0] }, ""] }] }, then: "$_firstExtensions", else: "$$REMOVE" } }, + }, + }, + }, + // 4. Sort by grouping key for deterministic ordering + { $sort: { key: 1 } }, + ]; +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 6a4960744..57cd4cddd 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -90,7 +90,9 @@ import { CreateRequestInputSchema, ListRequestsQuerySchema, BulkResubmitInputSchema, + RunGroupSchema, } from "shared"; +import { buildGroupingPipeline } from "./grouping.js"; import { checkMigrations } from "db-migrations/check-migrations"; import { generateOpenAPIDocument, registry } from "./openapi/index.js"; import swaggerUi from "swagger-ui-express"; @@ -1036,19 +1038,23 @@ apiRoute(app, registry, { }); // List all requests (excludes soft-deleted by default) +// When groupBy is provided, returns RunGroup[] instead of flat RequestDocument[] apiRoute(app, registry, { method: "get", path: "/api/v1/requests", tags: ["Requests"], summary: "List requests", query: ListRequestsQuerySchema, - response: z.array(RequestResponseSchema), + response: z.union([z.array(RequestResponseSchema), z.array(RunGroupSchema)]), handler: async (req, res) => { const workerFilter = req.query.worker as string; const taskPromptIdFilter = req.query.taskPromptId as string; const criteriaFilter = req.query.criteria as string; const submissionIdFilter = req.query.submissionId as string; + const statusFilter = req.query.status as string; + const outcomeFilter = req.query.outcome as string; const includeDeleted = req.query.includeDeleted === "true"; + const groupByParam = req.query.groupBy as "task" | "submissionId" | undefined; const filter: Record = {}; if (workerFilter && VALID_WORKERS.includes(workerFilter as WorkerType)) { @@ -1057,6 +1063,12 @@ apiRoute(app, registry, { if (taskPromptIdFilter) { filter.taskPromptId = taskPromptIdFilter; } + if (statusFilter) { + filter.status = statusFilter; + } + if (outcomeFilter) { + filter.outcome = outcomeFilter; + } if (submissionIdFilter) { // Prefix-based matching: allow filtering by partial submission ID filter.submissionId = { $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}` }; @@ -1086,6 +1098,17 @@ apiRoute(app, registry, { } } + // Grouped mode: return RunGroup[] via aggregation pipeline + if (groupByParam) { + const pipeline = [ + { $match: filter }, + ...buildGroupingPipeline(groupByParam), + ]; + const groups = await collection.aggregate(pipeline).toArray(); + res.json(groups); + return; + } + const resources = await collection .find(filter) .sort({ createdAt: -1 }) diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 77d07693d..1ec05f253 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillRevisionDocument, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest } from "@/types"; +import type { Run, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillRevisionDocument, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, RunGroup } from "@/types"; const BASE = "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/api/v1"; @@ -19,17 +19,32 @@ async function request(path: string, init?: RequestInit): Promise { } export const api = { - /** List all runs, optionally filtered by worker, task prompt, and/or MDP criteria state */ - listRuns: (opts?: { worker?: string; taskPromptId?: string; criteria?: string; submissionId?: string }): Promise => { + /** List all runs, optionally filtered by worker, task prompt, status, outcome, and/or MDP criteria state */ + listRuns: (opts?: { worker?: string; taskPromptId?: string; status?: string; outcome?: string; criteria?: string; submissionId?: string }): Promise => { const params = new URLSearchParams(); if (opts?.worker) params.set("worker", opts.worker); if (opts?.taskPromptId) params.set("taskPromptId", opts.taskPromptId); + if (opts?.status) params.set("status", opts.status); + if (opts?.outcome) params.set("outcome", opts.outcome); if (opts?.criteria) params.set("criteria", opts.criteria); if (opts?.submissionId) params.set("submissionId", opts.submissionId); const qs = params.toString(); return request(`/requests${qs ? `?${qs}` : ""}`); }, + /** List runs grouped by task or submissionId, with server-computed aggregates */ + listRunGroups: (opts: { groupBy: "task" | "submissionId"; worker?: string; taskPromptId?: string; status?: string; outcome?: string; criteria?: string; submissionId?: string }): Promise => { + const params = new URLSearchParams(); + params.set("groupBy", opts.groupBy); + if (opts.worker) params.set("worker", opts.worker); + if (opts.taskPromptId) params.set("taskPromptId", opts.taskPromptId); + if (opts.status) params.set("status", opts.status); + if (opts.outcome) params.set("outcome", opts.outcome); + if (opts.criteria) params.set("criteria", opts.criteria); + if (opts.submissionId) params.set("submissionId", opts.submissionId); + return request(`/requests?${params.toString()}`); + }, + /** Get a single run by ID */ getRun: (id: string): Promise => { return request(`/requests/${id}`); diff --git a/apps/portal/src/lib/grouping.test.ts b/apps/portal/src/lib/grouping.test.ts index 5579ba754..edf044aab 100644 --- a/apps/portal/src/lib/grouping.test.ts +++ b/apps/portal/src/lib/grouping.test.ts @@ -2,164 +2,7 @@ // Licensed under the MIT License. import { describe, it, expect } from "vitest"; -import { groupRuns, formatStatRange } from "./grouping"; -import type { Run } from "../types"; - -// Re-export internals for testing via a test-only helper -// Since computeStats is private, we test it indirectly through groupRuns - -function makeRun(overrides: Partial = {}): Run { - return { - _id: Math.random().toString(36).slice(2), - id: "", - workerType: "coder-acp-copilot", - status: "done", - createdAt: new Date().toISOString(), - scenario: { task: "Default task", criteria: [] }, - ...overrides, - }; -} - -describe("groupRuns", () => { - it("returns empty array when groupBy is 'none'", () => { - const runs = [makeRun(), makeRun()]; - expect(groupRuns(runs, "none")).toEqual([]); - }); - - it("groups by task using taskPromptId", () => { - const runs = [ - makeRun({ taskPromptId: "t1", scenario: { task: "Task A", criteria: [] } }), - makeRun({ taskPromptId: "t1", scenario: { task: "Task A", criteria: [] } }), - makeRun({ taskPromptId: "t2", scenario: { task: "Task B", criteria: [] } }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups).toHaveLength(2); - expect(groups[0].key).toBe("t1"); - expect(groups[0].label).toBe("Task A"); - expect(groups[0].runs).toHaveLength(2); - expect(groups[1].key).toBe("t2"); - expect(groups[1].runs).toHaveLength(1); - }); - - it("falls back to scenario.task when taskPromptId is missing", () => { - const runs = [ - makeRun({ scenario: { task: "Same task", criteria: [] } }), - makeRun({ scenario: { task: "Same task", criteria: [] } }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups).toHaveLength(1); - expect(groups[0].key).toBe("Same task"); - expect(groups[0].runs).toHaveLength(2); - }); - - it("groups by submissionId", () => { - const runs = [ - makeRun({ submissionId: "sub-1" }), - makeRun({ submissionId: "sub-1" }), - makeRun({ submissionId: "sub-2" }), - makeRun({}), // no submissionId - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups).toHaveLength(3); - - const sub1 = groups.find((g) => g.key === "sub-1"); - expect(sub1?.runs).toHaveLength(2); - expect(sub1?.label).toBe("sub-1"); - - const noSub = groups.find((g) => g.key === "no-submission"); - expect(noSub?.runs).toHaveLength(1); - expect(noSub?.label).toBe("No submission ID"); - }); - - it("computes turn aggregates correctly", () => { - const runs = [ - makeRun({ turns: [{ iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }] }), - makeRun({ turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - { iteration: 2, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - { iteration: 3, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - ] }), - makeRun({ turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - { iteration: 2, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - ] }), - ]; - // All same taskPromptId so they group together - runs.forEach((r) => (r.taskPromptId = "t1")); - const groups = groupRuns(runs, "task"); - expect(groups).toHaveLength(1); - const agg = groups[0].aggregates; - expect(agg.count).toBe(3); - expect(agg.turns).not.toBeNull(); - expect(agg.turns!.min).toBe(1); - expect(agg.turns!.max).toBe(3); - expect(agg.turns!.mean).toBe(2); // (1+3+2)/3 - }); - - it("computes duration aggregates from turn durationMs", () => { - const runs = [ - makeRun({ - taskPromptId: "t1", - turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "", durationMs: 10000 }, - { iteration: 2, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "", durationMs: 20000 }, - ], - }), - makeRun({ - taskPromptId: "t1", - turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "", durationMs: 5000 }, - ], - }), - ]; - const groups = groupRuns(runs, "task"); - const agg = groups[0].aggregates; - expect(agg.duration).not.toBeNull(); - expect(agg.duration!.min).toBe(5000); // 5s - expect(agg.duration!.max).toBe(30000); // 10+20=30s - expect(agg.duration!.mean).toBe(17500); // (30000+5000)/2 - }); - - it("computes token aggregates", () => { - const runs = [ - makeRun({ - taskPromptId: "t1", - tokenUsage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }, - }), - makeRun({ - taskPromptId: "t1", - tokenUsage: { promptTokens: 200, completionTokens: 100, totalTokens: 300 }, - }), - ]; - const groups = groupRuns(runs, "task"); - const agg = groups[0].aggregates; - expect(agg.promptTokens).not.toBeNull(); - expect(agg.promptTokens!.min).toBe(100); - expect(agg.promptTokens!.max).toBe(200); - expect(agg.promptTokens!.mean).toBe(150); - expect(agg.completionTokens!.min).toBe(50); - expect(agg.completionTokens!.max).toBe(100); - }); - - it("handles runs with no turns or tokens gracefully", () => { - const runs = [ - makeRun({ taskPromptId: "t1" }), // no turns, no tokens - makeRun({ taskPromptId: "t1" }), - ]; - const groups = groupRuns(runs, "task"); - const agg = groups[0].aggregates; - expect(agg.count).toBe(2); - expect(agg.turns).toBeNull(); - expect(agg.duration).toBeNull(); - expect(agg.promptTokens).toBeNull(); - expect(agg.completionTokens).toBeNull(); - }); - - it("handles empty runs array", () => { - expect(groupRuns([], "task")).toEqual([]); - expect(groupRuns([], "submissionId")).toEqual([]); - }); -}); +import { formatStatRange } from "./grouping"; describe("formatStatRange", () => { it("returns dash for null stats", () => { @@ -185,162 +28,3 @@ describe("formatStatRange", () => { expect(result).toContain("200ms"); }); }); - -describe("uniform values", () => { - it("detects uniform workerType", () => { - const runs = [ - makeRun({ taskPromptId: "t1", workerType: "coder-acp-copilot" }), - makeRun({ taskPromptId: "t1", workerType: "coder-acp-copilot" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.workerType).toBe("coder-acp-copilot"); - }); - - it("returns undefined for mixed workerType", () => { - const runs = [ - makeRun({ taskPromptId: "t1", workerType: "coder-acp-copilot" }), - makeRun({ taskPromptId: "t1", workerType: "coder-vscode-web" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.workerType).toBeUndefined(); - }); - - it("detects uniform model", () => { - const runs = [ - makeRun({ taskPromptId: "t1", model: "gpt-4o" }), - makeRun({ taskPromptId: "t1", model: "gpt-4o" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.model).toBe("gpt-4o"); - }); - - it("omits model when all undefined", () => { - const runs = [ - makeRun({ taskPromptId: "t1" }), - makeRun({ taskPromptId: "t1" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.model).toBeUndefined(); - }); - - it("detects uniform status", () => { - const runs = [ - makeRun({ taskPromptId: "t1", status: "done" }), - makeRun({ taskPromptId: "t1", status: "done" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.status).toBe("done"); - }); - - it("returns undefined for mixed status", () => { - const runs = [ - makeRun({ taskPromptId: "t1", status: "done" }), - makeRun({ taskPromptId: "t1", status: "pending" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.status).toBeUndefined(); - }); - - it("detects uniform mcpServers (order-independent)", () => { - const runs = [ - makeRun({ taskPromptId: "t1", mcpServers: ["b", "a"] }), - makeRun({ taskPromptId: "t1", mcpServers: ["a", "b"] }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.mcpServers).toEqual(["b", "a"]); - }); - - it("returns undefined for mixed mcpServers", () => { - const runs = [ - makeRun({ taskPromptId: "t1", mcpServers: ["a"] }), - makeRun({ taskPromptId: "t1", mcpServers: ["b"] }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.mcpServers).toBeUndefined(); - }); - - it("detects uniform skillRevisions", () => { - const runs = [ - makeRun({ taskPromptId: "t1", skillRevisions: ["org/repo/skill@abc"] }), - makeRun({ taskPromptId: "t1", skillRevisions: ["org/repo/skill@abc"] }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.skillRevisions).toEqual(["org/repo/skill@abc"]); - }); - - it("detects uniform agentVersion", () => { - const runs = [ - makeRun({ taskPromptId: "t1", agentVersion: "copilot-0.0.415" }), - makeRun({ taskPromptId: "t1", agentVersion: "copilot-0.0.415" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.agentVersion).toBe("copilot-0.0.415"); - }); - - it("single-run group always shows uniform values", () => { - const runs = [ - makeRun({ - taskPromptId: "t1", - workerType: "coder-vscode-web", - model: "gpt-4o", - status: "done", - mcpServers: ["fs"], - agentVersion: "v1", - }), - ]; - const groups = groupRuns(runs, "task"); - const u = groups[0].uniform; - expect(u.workerType).toBe("coder-vscode-web"); - expect(u.model).toBe("gpt-4o"); - expect(u.status).toBe("done"); - expect(u.mcpServers).toEqual(["fs"]); - expect(u.agentVersion).toBe("v1"); - }); - - it("sets uniform extensions when all runs share the same extensions", () => { - const ext = ["ms-azuretools.vscode-cosmosdb@0.32.0"]; - const runs = [ - makeRun({ submissionId: "s1", extensions: ext }), - makeRun({ submissionId: "s1", extensions: ext }), - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups[0].uniform.extensions).toEqual(ext); - }); - - it("does not set uniform extensions when runs have different extensions", () => { - const runs = [ - makeRun({ submissionId: "s1", extensions: ["ms-azuretools.vscode-cosmosdb@0.32.0"] }), - makeRun({ submissionId: "s1", extensions: ["ms-python.python@2024.1.0"] }), - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups[0].uniform.extensions).toBeUndefined(); - }); - - it("does not set uniform extensions when no runs have extensions", () => { - const runs = [ - makeRun({ submissionId: "s1" }), - makeRun({ submissionId: "s1" }), - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups[0].uniform.extensions).toBeUndefined(); - }); - - it("treats same extensions in different order as uniform", () => { - const runs = [ - makeRun({ submissionId: "s1", extensions: ["ext-b@1.0", "ext-a@2.0"] }), - makeRun({ submissionId: "s1", extensions: ["ext-a@2.0", "ext-b@1.0"] }), - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups[0].uniform.extensions).toBeDefined(); - expect(groups[0].uniform.extensions).toHaveLength(2); - }); - - it("does not set uniform extensions when only some runs have them", () => { - const runs = [ - makeRun({ submissionId: "s1", extensions: ["ms-azuretools.vscode-cosmosdb@0.32.0"] }), - makeRun({ submissionId: "s1" }), - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups[0].uniform.extensions).toBeUndefined(); - }); -}); diff --git a/apps/portal/src/lib/grouping.ts b/apps/portal/src/lib/grouping.ts index 53b973b27..9b07af8e8 100644 --- a/apps/portal/src/lib/grouping.ts +++ b/apps/portal/src/lib/grouping.ts @@ -1,188 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, RunStatus } from "../types"; - -export type GroupByKey = "none" | "task" | "submissionId"; - -export interface AggregateStats { - min: number; - max: number; - mean: number; - stdDev: number; -} - -export interface GroupUniformValues { - workerType?: string; - model?: string; - agentVersion?: string; - platform?: string; - mcpServers?: string[]; - skillRevisions?: string[]; - extensions?: string[]; - status?: RunStatus; - submissionId?: string; - task?: string; -} - -export interface GroupAggregates { - count: number; - turns: AggregateStats | null; - duration: AggregateStats | null; - promptTokens: AggregateStats | null; - completionTokens: AggregateStats | null; -} - -export interface RunGroup { - key: string; - label: string; - runs: Run[]; - aggregates: GroupAggregates; - uniform: GroupUniformValues; -} - -function computeStats(values: number[]): AggregateStats | null { - if (values.length === 0) return null; - const min = Math.min(...values); - const max = Math.max(...values); - const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); - return { min, max, mean, stdDev }; -} - -function getRunTurns(run: Run): number | null { - return run.turns?.length ?? null; -} - -function getRunDuration(run: Run): number | null { - if (!run.turns || run.turns.length === 0) return null; - const total = run.turns.reduce((sum, t) => sum + (t.durationMs ?? 0), 0); - return total > 0 ? total : null; -} - -function getRunTokens(run: Run): { prompt: number; completion: number } | null { - const usage = - run.tokenUsage ?? - (run.turns?.some((t) => t.tokenUsage) - ? run.turns!.reduce( - (acc, t) => { - if (!t.tokenUsage) return acc; - return { - promptTokens: acc.promptTokens + t.tokenUsage.promptTokens, - completionTokens: acc.completionTokens + t.tokenUsage.completionTokens, - totalTokens: acc.totalTokens + t.tokenUsage.totalTokens, - }; - }, - { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, - ) - : undefined); - if (!usage) return null; - return { prompt: usage.promptTokens, completion: usage.completionTokens }; -} - -function computeAggregates(runs: Run[]): GroupAggregates { - const turnValues = runs.map(getRunTurns).filter((v): v is number => v !== null); - const durationValues = runs.map(getRunDuration).filter((v): v is number => v !== null); - const tokenData = runs.map(getRunTokens).filter((v): v is { prompt: number; completion: number } => v !== null); - - return { - count: runs.length, - turns: computeStats(turnValues), - duration: computeStats(durationValues), - promptTokens: computeStats(tokenData.map((t) => t.prompt)), - completionTokens: computeStats(tokenData.map((t) => t.completion)), - }; -} - -function arrKey(arr: string[] | undefined): string { - return (arr ?? []).slice().sort().join(","); -} - -function uniform(runs: Run[], getter: (r: Run) => T): T | undefined { - if (runs.length === 0) return undefined; - const first = getter(runs[0]); - return runs.every((r) => getter(r) === first) ? first : undefined; -} - -function computeUniformValues(runs: Run[]): GroupUniformValues { - if (runs.length === 0) return {}; - - const result: GroupUniformValues = {}; - - const wt = uniform(runs, (r) => r.workerType); - if (wt !== undefined) result.workerType = wt; - - const model = uniform(runs, (r) => r.model ?? ""); - if (model !== undefined && model !== "") result.model = model; - - const ver = uniform(runs, (r) => r.agentVersion ?? ""); - if (ver !== undefined && ver !== "") result.agentVersion = ver; - - const plat = uniform(runs, (r) => r.os?.platform ?? ""); - if (plat !== undefined && plat !== "") result.platform = plat; - - const status = uniform(runs, (r) => r.status); - if (status !== undefined) result.status = status; - - const sub = uniform(runs, (r) => r.submissionId ?? ""); - if (sub !== undefined && sub !== "") result.submissionId = sub; - - const task = uniform(runs, (r) => r.scenario?.task ?? ""); - if (task !== undefined && task !== "") result.task = task; - - // Array fields: compare by sorted join - const mcpKey = uniform(runs, (r) => arrKey(r.mcpServers)); - if (mcpKey !== undefined && mcpKey !== "") result.mcpServers = runs[0].mcpServers; - - const skillKey = uniform(runs, (r) => arrKey(r.skillRevisions)); - if (skillKey !== undefined && skillKey !== "") result.skillRevisions = runs[0].skillRevisions; - - const extKey = uniform(runs, (r) => arrKey(r.extensions)); - if (extKey !== undefined && extKey !== "") result.extensions = runs[0].extensions; - - return result; -} - -export function groupRuns(runs: Run[], groupBy: GroupByKey): RunGroup[] { - if (groupBy === "none") return []; - - const groups = new Map(); - - for (const run of runs) { - let key: string; - if (groupBy === "task") { - key = run.taskPromptId ?? run.scenario?.task ?? "unknown"; - } else { - key = run.submissionId ?? "no-submission"; - } - const list = groups.get(key); - if (list) { - list.push(run); - } else { - groups.set(key, [run]); - } - } - - const result: RunGroup[] = []; - for (const [key, groupRuns] of groups) { - let label: string; - if (groupBy === "task") { - label = groupRuns[0]?.scenario?.task ?? key; - } else { - label = key === "no-submission" ? "No submission ID" : key; - } - result.push({ - key, - label, - runs: groupRuns, - aggregates: computeAggregates(groupRuns), - uniform: computeUniformValues(groupRuns), - }); - } - - return result; -} +import type { AggregateStats } from "../types"; export function formatStatRange( stat: AggregateStats | null, diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index efa85a207..e3bc5f5c3 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -23,8 +23,8 @@ import { StatusBadge, OutcomeBadge } from "@/components/StatusBadge"; import { Trash2, Eye, Plus, RefreshCw, Repeat, FileText, X, Download, ChevronRight, ChevronDown } from "lucide-react"; import { formatDate, formatId, truncate, formatDuration } from "@/lib/utils"; import { WORKER_TYPES, STATUS_LIST, OUTCOME_LIST } from "@/types"; -import type { Run, BulkResubmitOverrides, McpServerDocument, CodingAgent, BulkReportSummary } from "@/types"; -import { groupRuns, formatStatRange, type GroupByKey, type RunGroup } from "@/lib/grouping"; +import type { Run, BulkResubmitOverrides, McpServerDocument, CodingAgent, BulkReportSummary, RunGroup, GroupByKey } from "@/types"; +import { formatStatRange } from "@/lib/grouping"; export function RunsList() { const [searchParams, setSearchParams] = useSearchParams(); @@ -43,14 +43,38 @@ export function RunsList() { const [resubmitOverrides, setResubmitOverrides] = useState({}); const queryClient = useQueryClient(); + // Merge URL taskPromptId with dropdown taskFilter (dropdown takes precedence) + const effectiveTaskPromptId = taskFilter !== "all" ? taskFilter : taskPromptId; + const effectiveStatus = statusFilter !== "all" ? statusFilter : undefined; + const effectiveOutcome = outcomeFilter !== "all" ? outcomeFilter : undefined; + const { data: runs = [], isLoading, isRefetching } = useQuery({ - queryKey: ["runs", workerFilter, taskPromptId, criteriaState, submissionId], + queryKey: ["runs", workerFilter, effectiveTaskPromptId, statusFilter, outcomeFilter, criteriaState, submissionId], queryFn: () => api.listRuns({ worker: workerFilter === "all" ? undefined : workerFilter, - taskPromptId, + taskPromptId: effectiveTaskPromptId, + status: effectiveStatus, + outcome: effectiveOutcome, + criteria: criteriaState, + submissionId, + }), + enabled: groupBy === "none", + refetchInterval: 10_000, + }); + + // Fetch server-side groups when groupBy is active + const { data: serverGroups = [], isLoading: isGroupsLoading, isRefetching: isGroupsRefetching } = useQuery({ + queryKey: ["run-groups", groupBy, workerFilter, effectiveTaskPromptId, statusFilter, outcomeFilter, criteriaState, submissionId], + queryFn: () => api.listRunGroups({ + groupBy: groupBy as "task" | "submissionId", + worker: workerFilter === "all" ? undefined : workerFilter, + taskPromptId: effectiveTaskPromptId, + status: effectiveStatus, + outcome: effectiveOutcome, criteria: criteriaState, submissionId, }), + enabled: groupBy !== "none", refetchInterval: 10_000, }); @@ -76,10 +100,24 @@ export function RunsList() { refetchInterval: 10_000, }); - const uniqueTasks = useMemo( - () => [...new Set(runs.map((r) => r.scenario?.task).filter(Boolean) as string[])].sort(), - [runs], - ); + // Derive unique task options (name + taskPromptId) from runs or server groups + const taskOptions = useMemo(() => { + const seen = new Map(); // taskPromptId → task name + if (groupBy === "none") { + for (const r of runs) { + if (r.taskPromptId && r.scenario?.task && !seen.has(r.taskPromptId)) { + seen.set(r.taskPromptId, r.scenario.task); + } + } + } else if (groupBy === "task") { + for (const g of serverGroups) { + if (g.key && g.label && !seen.has(g.key)) { + seen.set(g.key, g.label); + } + } + } + return [...seen.entries()].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)); + }, [runs, serverGroups, groupBy]); // Compute summary of selected runs' values for the resubmit dialog const selectedRunsSummary = useMemo(() => { @@ -174,14 +212,10 @@ export function RunsList() { }, }); - const filteredRuns = runs.filter((r) => { - if (statusFilter !== "all" && r.status !== statusFilter) return false; - if (outcomeFilter !== "all" && r.outcome !== outcomeFilter) return false; - if (taskFilter !== "all" && r.scenario?.task !== taskFilter) return false; - return true; - }); + // Filtering is now server-side via status, outcome, and taskPromptId query params + const filteredRuns = runs; - const runGroups = useMemo(() => groupRuns(filteredRuns, groupBy), [filteredRuns, groupBy]); + const runGroups = serverGroups; const toggleGroup = (key: string) => { setExpandedGroups((prev) => { @@ -192,20 +226,6 @@ export function RunsList() { }); }; - const toggleGroupSelect = (group: RunGroup) => { - setSelectedIds((prev) => { - const next = new Set(prev); - const groupIds = group.runs.map((r) => r._id); - const allInGroupSelected = groupIds.every((id) => prev.has(id)); - if (allInGroupSelected) { - groupIds.forEach((id) => next.delete(id)); - } else { - groupIds.forEach((id) => next.add(id)); - } - return next; - }); - }; - const allSelected = filteredRuns.length > 0 && filteredRuns.every((r) => selectedIds.has(r._id)); const someSelected = filteredRuns.some((r) => selectedIds.has(r._id)); @@ -296,9 +316,9 @@ export function RunsList() { All tasks - {uniqueTasks.map((t) => ( - - {truncate(t, 50)} + {taskOptions.map((t) => ( + + {truncate(t.name, 50)} ))} @@ -318,10 +338,11 @@ export function RunsList() {
- {isRefetching && } + {(isRefetching || isGroupsRefetching) && } - {filteredRuns.length} run{filteredRuns.length !== 1 ? "s" : ""} - {groupBy !== "none" && ` in ${runGroups.length} group${runGroups.length !== 1 ? "s" : ""}`} + {groupBy !== "none" + ? `${runGroups.length} group${runGroups.length !== 1 ? "s" : ""}` + : `${filteredRuns.length} run${filteredRuns.length !== 1 ? "s" : ""}`}
@@ -840,13 +861,13 @@ export function RunsList() { )} {/* Table */} - {isLoading ? ( + {(groupBy === "none" ? isLoading : isGroupsLoading) ? (
{Array.from({ length: 5 }).map((_, i) => ( ))}
- ) : filteredRuns.length === 0 ? ( + ) : (groupBy === "none" && filteredRuns.length === 0) || (groupBy !== "none" && runGroups.length === 0) ? (
No runs found. Submit one?
@@ -855,11 +876,13 @@ export function RunsList() { - + {groupBy === "none" && ( + + )} ID Submission @@ -885,25 +908,21 @@ export function RunsList() { {groupBy !== "none" ? ( runGroups.map((group) => { const isExpanded = expandedGroups.has(group.key); - const groupIds = group.runs.map((r) => r._id); - const allGroupSelected = groupIds.length > 0 && groupIds.every((id) => selectedIds.has(id)); - const someGroupSelected = groupIds.some((id) => selectedIds.has(id)); return ( toggleGroup(group.key)} - allGroupSelected={allGroupSelected} - someGroupSelected={someGroupSelected} - onToggleGroupSelect={() => toggleGroupSelect(group)} selectedIds={selectedIds} onToggleSelect={toggleSelect} reportSummaries={reportSummaries} deleteMutation={deleteMutation} - bulkDeleteMutation={bulkDeleteMutation} - bulkReportMutation={bulkReportMutation} groupBy={groupBy} + workerFilter={workerFilter === "all" ? undefined : workerFilter} + statusFilter={effectiveStatus} + outcomeFilter={effectiveOutcome} + criteriaState={criteriaState} /> ); }) @@ -1120,35 +1139,77 @@ function GroupRows({ group, isExpanded, onToggleExpand, - allGroupSelected, - someGroupSelected, - onToggleGroupSelect, selectedIds, onToggleSelect, reportSummaries, deleteMutation, - bulkDeleteMutation, - bulkReportMutation, groupBy, + workerFilter, + statusFilter, + outcomeFilter, + criteriaState, }: { group: RunGroup; isExpanded: boolean; onToggleExpand: () => void; - allGroupSelected: boolean; - someGroupSelected: boolean; - onToggleGroupSelect: () => void; selectedIds: Set; onToggleSelect: (id: string) => void; reportSummaries: BulkReportSummary | undefined; deleteMutation: { mutate: (id: string) => void; isPending: boolean }; - bulkDeleteMutation: { mutate: (ids: string[]) => void; isPending: boolean }; - bulkReportMutation: { mutate: (ids: string[]) => void; isPending: boolean }; groupBy: GroupByKey; + workerFilter?: string; + statusFilter?: string; + outcomeFilter?: string; + criteriaState?: string; }) { const { aggregates, uniform } = group; const fmtDur = (v: number) => formatDuration(Math.round(v)); const fmtNum = (v: number) => Math.round(v).toLocaleString(); + // Fetch runs for this group on expand + const expandFilter = useMemo(() => { + const opts: Record = { worker: workerFilter, status: statusFilter, outcome: outcomeFilter, criteria: criteriaState }; + if (groupBy === "task") { + // group.key is taskPromptId (or scenario.task fallback) + opts.taskPromptId = group.key; + } else { + opts.submissionId = group.key === "no-submission" ? undefined : group.key; + } + return opts; + }, [group.key, groupBy, workerFilter, statusFilter, outcomeFilter, criteriaState]); + + const { data: expandedRuns = [], isLoading: isExpandLoading } = useQuery({ + queryKey: ["group-runs", group.key, groupBy, workerFilter, statusFilter, outcomeFilter, criteriaState], + queryFn: () => api.listRuns(expandFilter), + enabled: isExpanded, + refetchInterval: 10_000, + }); + + // Group-level checkbox: select/deselect all runs in this group (runIds from server) + const groupRunIds = group.runIds; + const allGroupSelected = groupRunIds.length > 0 && groupRunIds.every((id) => selectedIds.has(id)); + const someGroupSelected = groupRunIds.some((id) => selectedIds.has(id)); + const handleToggleGroupSelect = () => { + if (allGroupSelected) { + groupRunIds.forEach((id) => onToggleSelect(id)); + } else { + groupRunIds.filter((id) => !selectedIds.has(id)).forEach((id) => onToggleSelect(id)); + } + }; + + // Fetch report summaries for all runs in this group (always available via runIds) + const { data: groupReportSummaries } = useQuery({ + queryKey: ["report-summaries", groupRunIds], + queryFn: () => api.bulkReportSummary(groupRunIds), + enabled: groupRunIds.length > 0, + refetchInterval: 10_000, + }); + // Merge parent-level and group-level summaries + const mergedReportSummaries = useMemo(() => { + if (!reportSummaries && !groupReportSummaries) return undefined; + return { ...reportSummaries, ...groupReportSummaries }; + }, [reportSummaries, groupReportSummaries]); + return ( <> e.stopPropagation()}> @@ -1277,17 +1338,12 @@ function GroupRows({ processing: "bg-blue-500", done: "bg-green-500", }; - const completed = group.runs.filter((r) => r.status === "done").length; - const total = group.runs.length; - const segments = Object.entries( - group.runs.reduce>((acc, r) => { - acc[r.status] = (acc[r.status] ?? 0) + 1; - return acc; - }, {}), - ); + const total = aggregates.count; + const done = aggregates.statusCounts?.done ?? 0; + const segments = Object.entries(aggregates.statusCounts ?? {}).filter(([, c]) => c > 0); return (
- {completed}/{total} done + {done}/{total} done
{segments.map(([status, count]) => (
= { succeeded: "bg-green-500", failed: "bg-red-500", - exhausted: "bg-yellow-500", + finished: "bg-yellow-500", }; - const doneRuns = group.runs.filter((r) => r.status === "done" && r.outcome); - if (doneRuns.length === 0) return –; - const segments = Object.entries( - doneRuns.reduce>((acc, r) => { - acc[r.outcome!] = (acc[r.outcome!] ?? 0) + 1; - return acc; - }, {}), - ); - const succeeded = doneRuns.filter((r) => r.outcome === "succeeded").length; + const doneCount = aggregates.statusCounts?.done ?? 0; + const segments = Object.entries(aggregates.outcomeCounts ?? {}).filter(([, c]) => c > 0); + if (doneCount === 0 || segments.length === 0) return –; + const succeeded = aggregates.outcomeCounts?.succeeded ?? 0; return (
- {succeeded}/{doneRuns.length} pass + {succeeded}/{doneCount} pass
{segments.map(([outcome, count]) => (
))} @@ -1339,11 +1390,10 @@ function GroupRows({ {/* Report */} {(() => { - const runsWithReports = group.runs.filter((r) => reportSummaries?.[r._id]); - if (runsWithReports.length === 0) return –; + if (!mergedReportSummaries || groupRunIds.length === 0) return –; let total = 0, completed = 0, pending = 0, generating = 0, failed = 0; - for (const r of group.runs) { - const s = reportSummaries?.[r._id]; + for (const id of groupRunIds) { + const s = mergedReportSummaries[id]; if (!s) continue; total += s.total; completed += s.completed; @@ -1351,6 +1401,7 @@ function GroupRows({ generating += s.generating; failed += s.failed; } + if (total === 0) return –; return ; })()} @@ -1358,6 +1409,8 @@ function GroupRows({ {formatStatRange(aggregates.turns, fmtNum)} + {/* LLM Calls */} + {/* Duration */} {formatStatRange(aggregates.duration, fmtDur)} @@ -1371,70 +1424,29 @@ function GroupRows({ {/* Created */} {/* Actions */} - e.stopPropagation()}> -
- - - - - - - Generate reports for {group.aggregates.count} run{group.aggregates.count !== 1 ? "s" : ""}? - - This will queue report generation for all runs in this group. - - - - Cancel - bulkReportMutation.mutate(group.runs.map((r) => r._id))} - disabled={bulkReportMutation.isPending} - > - {bulkReportMutation.isPending ? "Generating…" : "Generate"} - - - - - - - - - - - Delete {group.aggregates.count} run{group.aggregates.count !== 1 ? "s" : ""} in this group? - - This will soft-delete all runs in this group. They can be recovered later if needed. - - - - Cancel - bulkDeleteMutation.mutate(group.runs.map((r) => r._id))} - disabled={bulkDeleteMutation.isPending} - className="bg-destructive text-destructive-foreground hover:bg-destructive/90" - > - {bulkDeleteMutation.isPending ? "Deleting…" : "Delete"} - - - - -
-
+ - {isExpanded && group.runs.map((run) => ( - - ))} + {isExpanded && ( + isExpandLoading ? ( + + + + Loading runs… + + + ) : ( + expandedRuns.map((run) => ( + + )) + ) + )} ); } diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index c81754c96..0cfc44354 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -798,3 +798,45 @@ export interface FeatureFlag { enabled: boolean; updatedAt: string; } + +// --- Runs grouping types (mirrored from shared) --- + +export type GroupByKey = "none" | "task" | "submissionId"; + +export interface AggregateStats { + min: number; + max: number; + mean: number; + stdDev: number; +} + +export interface GroupUniformValues { + workerType?: string; + model?: string; + agentVersion?: string; + platform?: string; + mcpServers?: string[]; + skillRevisions?: string[]; + extensions?: string[]; + status?: RunStatus; + submissionId?: string; + task?: string; +} + +export interface GroupAggregates { + count: number; + turns: AggregateStats | null; + duration: AggregateStats | null; + promptTokens: AggregateStats | null; + completionTokens: AggregateStats | null; + statusCounts: Record; + outcomeCounts: Record; +} + +export interface RunGroup { + key: string; + label: string; + runIds: string[]; + aggregates: GroupAggregates; + uniform: GroupUniformValues; +} diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index a34e40c50..9cfd9540e 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -94,9 +94,17 @@ When you run a command: | Migration | Description | |-----------|-------------| | `001-backfill-task-prompts` | Creates `task-prompts` collection from existing `requests.scenario.task` values, links runs via `taskPromptId`, migrates prompt feature extraction data | +| `002-create-indexes` | Creates single-field indexes on all collections (see [db.md](db.md)) | +| `003-create-skill-indexes` | Adds indexes for `skills` and `skill-revisions` collections | +| `004-add-submission-id-index` | Sparse index on `requests.submissionId` for efficient filtering | +| `005-backfill-iteration-durations` | Estimates `startedAt`/`durationMs` on existing turns from timestamps | +| `006-split-status-outcome` | Splits run status into `status` (pending/processing/done) + `outcome` (succeeded/failed/exhausted) | +| `007-rename-exhausted-to-finished` | Renames outcome `exhausted` → `finished` | +| `008-backfill-ai-call-count` | Downloads HARs from blob storage to count AI completion calls per turn | +| `009-add-requests-filter-indexes` | Adds indexes on `taskPromptId`, `status`, `outcome`, `workerType`, `deletedAt` for server-side filtering/grouping | ## CI/CD The `db-migrations` package is compiled as part of the CI `build` job (`pnpm -r build` → `tsc`). This catches TypeScript errors in migrations before deployment. -Migrations themselves are **not** run automatically in CI — they require a live database connection and are run manually as part of the deployment process. +In Kubernetes, migrations are run automatically by a Job before the API pods start. Locally, Docker Compose runs migrations automatically as well. The API readiness probe checks the `_migrations` collection against the `REQUIRED_MIGRATIONS` list — pods report "not ready" until all required migrations have been applied. diff --git a/docs/architecture/db.md b/docs/architecture/db.md new file mode 100644 index 000000000..efff555cd --- /dev/null +++ b/docs/architecture/db.md @@ -0,0 +1,156 @@ +# Database Schema & Indexes + +MongoDB database (Cosmos DB for MongoDB RU in production, MongoDB 7.0 locally) storing all platform data. + +- **Database**: `requests-db` (configurable via `MONGO_DATABASE`) +- **Connection**: `MONGO_CONNECTION_STRING` or `MONGO_URI` + +## Collections + +### `requests` + +Benchmark runs — the core entity. Each document represents a single coding agent run against a task. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | +| `submissionId` | `{ submissionId: 1 }` | sparse | 004 | +| `taskPromptId` | `{ taskPromptId: 1 }` | | 009 | +| `status` | `{ status: 1 }` | | 009 | +| `outcome` | `{ outcome: 1 }` | | 009 | +| `workerType` | `{ workerType: 1 }` | | 009 | +| `deletedAt` | `{ deletedAt: 1 }` | | 009 | + +Key fields: `status`, `outcome`, `workerType`, `taskPromptId`, `submissionId`, `scenario.task`, `turns[]`, `tokenUsage`, `model`, `agentVersion`, `os.platform`, `mcpServers[]`, `skillRevisions[]`, `deletedAt`, `createdAt`. + +### `task-prompts` + +Canonical task definitions with prompt text. Runs link to a task prompt via `taskPromptId`. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `reports` + +Evaluation reports generated by the judge for individual runs. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | +| `requestId` | `{ requestId: 1 }` | | 002 | + +### `criteria` + +Evaluation criteria definitions used by the judge. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `id` | `{ id: 1 }` | unique | 002 | + +### `agents` + +Registered coding agent configurations. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `models` + +LLM model records associated with agents. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `agentId` | `{ agentId: 1 }` | | 002 | +| `provider` | `{ provider: 1 }` | | 002 | +| `agentId_provider` | `{ agentId: 1, provider: 1 }` | | 002 | +| `modelId` | `{ modelId: 1 }` | | 002 | + +### `skills` + +Agent skill definitions (source + name). + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 003 | +| `source_skillName` | `{ source: 1, skillName: 1 }` | | 003 | + +### `skill-revisions` + +Immutable revisions of agent skills, referenced by runs. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `ref` | `{ ref: 1 }` | unique | 003 | +| `source_skillName` | `{ source: 1, skillName: 1 }` | | 003 | +| `resolvedAt` | `{ resolvedAt: -1 }` | | 003 | + +### `prompt-features` + +Feature definitions extracted from task prompts. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `id` | `{ id: 1 }` | unique | 002 | + +### `prompt-feature-extractions` + +Cached results of prompt feature extraction (keyed by task text hash). + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `taskTextHash` | `{ taskTextHash: 1 }` | unique | 002 | + +### `insights` + +Generated analysis insights. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `mcp-servers` + +MCP (Model Context Protocol) server configurations. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `feature-flags` + +Runtime feature flags. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `key` | `{ key: 1 }` | unique | 002 | + +### Other collections + +- **`extensions`** — VS Code extension metadata +- **`report-templates`** — Report generation templates +- **`accounts`** — User/service accounts +- **`_migrations`** — Migration tracking (managed by `mongo-migrate-ts`) + +## Indexing Strategy + +Production uses **Cosmos DB for MongoDB (RU-based)**. Key considerations: + +- **No wildcard index** — unlike the default Cosmos DB policy, the `requests` collection uses explicit single-field indexes only. +- **Single-field indexes over compound** — Cosmos DB for MongoDB RU uses index intersection for multi-property `$match` filters. Compound indexes are only needed for sort optimization. +- **Sparse indexes** — used for optional fields (`submissionId`) to avoid indexing documents that lack the field. +- **All index changes go through migrations** — see [db-migrations.md](db-migrations.md) for the migration framework. diff --git a/packages/db-migrations/src/check-migrations.test.ts b/packages/db-migrations/src/check-migrations.test.ts index d9179c0a0..a83b91b8a 100644 --- a/packages/db-migrations/src/check-migrations.test.ts +++ b/packages/db-migrations/src/check-migrations.test.ts @@ -39,6 +39,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]); const result = await checkMigrations(db); expect(result.ready).toBe(true); @@ -52,6 +53,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]); }); @@ -59,7 +61,7 @@ describe("checkMigrations", () => { const db = makeMockDb(["001-backfill-task-prompts.ts"]); const result = await checkMigrations(db); expect(result.ready).toBe(false); - expect(result.pending).toEqual(["002-create-indexes.ts", "003-create-skill-indexes.ts", "004-add-submission-id-index.ts", "005-backfill-iteration-durations.ts", "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts"]); + expect(result.pending).toEqual(["002-create-indexes.ts", "003-create-skill-indexes.ts", "004-add-submission-id-index.ts", "005-backfill-iteration-durations.ts", "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", "009-add-requests-filter-indexes.ts"]); expect(result.applied).toEqual(["001-backfill-task-prompts.ts"]); }); @@ -76,6 +78,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]); expect(result.applied).toEqual([]); }); @@ -90,6 +93,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", "999-future-migration.ts", ]); const result = await checkMigrations(db); diff --git a/packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts b/packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts new file mode 100644 index 000000000..dd6c215d9 --- /dev/null +++ b/packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Migration: Add single-field indexes for commonly filtered fields on the + * requests collection. + * + * The server-side grouping and filtering pipeline uses $match on these fields. + * Cosmos DB for MongoDB (RU) recommends single-field indexes over compound + * indexes for multi-property filters — it uses index intersection internally. + */ + +import type { Db } from "mongodb"; +import type { MigrationInterface } from "mongo-migrate-ts"; + +const INDEXES: Array<{ key: Record; name: string }> = [ + { key: { taskPromptId: 1 }, name: "taskPromptId_1" }, + { key: { status: 1 }, name: "status_1" }, + { key: { outcome: 1 }, name: "outcome_1" }, + { key: { workerType: 1 }, name: "workerType_1" }, + { key: { deletedAt: 1 }, name: "deletedAt_1" }, +]; + +export class AddRequestsFilterIndexes implements MigrationInterface { + async up(db: Db): Promise { + const col = db.collection("requests"); + for (const { key, name } of INDEXES) { + try { + await col.createIndex(key); + console.log(` Created index ${name} on requests`); + } catch (err: any) { + console.log( + ` Index ${name} already exists or couldn't be created: ${err.message ?? err}`, + ); + } + } + } + + async down(db: Db): Promise { + console.log( + " Skipping index drop — indexes should be dropped manually if needed", + ); + } +} diff --git a/packages/db-migrations/src/required-migrations.ts b/packages/db-migrations/src/required-migrations.ts index 4935a8ac4..ad31c2830 100644 --- a/packages/db-migrations/src/required-migrations.ts +++ b/packages/db-migrations/src/required-migrations.ts @@ -21,4 +21,5 @@ export const REQUIRED_MIGRATIONS: readonly string[] = [ "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]; diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index c96e7fc6c..e4e9b36d1 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -136,11 +136,58 @@ export const ListRequestsQuerySchema = z submissionId: z.string().optional(), status: RequestStatusSchema.optional(), outcome: RequestOutcomeSchema.optional(), + groupBy: z.enum(["task", "submissionId"]).optional(), page: z.coerce.number().int().min(1).optional(), limit: z.coerce.number().int().min(1).max(100).optional(), }) .openapi("ListRequestsQuery"); +export const AggregateStatsSchema = z + .object({ + min: z.number(), + max: z.number(), + mean: z.number(), + stdDev: z.number(), + }) + .openapi("AggregateStats"); + +export const GroupUniformValuesSchema = z + .object({ + workerType: z.string().optional(), + model: z.string().optional(), + agentVersion: z.string().optional(), + platform: z.string().optional(), + mcpServers: z.array(z.string()).optional(), + skillRevisions: z.array(z.string()).optional(), + extensions: z.array(z.string()).optional(), + status: RequestStatusSchema.optional(), + submissionId: z.string().optional(), + task: z.string().optional(), + }) + .openapi("GroupUniformValues"); + +export const GroupAggregatesSchema = z + .object({ + count: z.number(), + turns: AggregateStatsSchema.nullable(), + duration: AggregateStatsSchema.nullable(), + promptTokens: AggregateStatsSchema.nullable(), + completionTokens: AggregateStatsSchema.nullable(), + statusCounts: z.record(z.string(), z.number()), + outcomeCounts: z.record(z.string(), z.number()), + }) + .openapi("GroupAggregates"); + +export const RunGroupSchema = z + .object({ + key: z.string(), + label: z.string(), + runIds: z.array(z.string()), + aggregates: GroupAggregatesSchema, + uniform: GroupUniformValuesSchema, + }) + .openapi("RunGroup"); + export const BulkResubmitInputSchema = z .object({ ids: z.array(z.string()).min(1), diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 35f0402dc..511f9d9b1 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -241,6 +241,48 @@ export interface QueueProcessorConfig extends BaseQueueProcessorConfig { // --- Enhanced Criteria System types --- +// --- Runs grouping types (shared between API and portal) --- + +export type GroupByKey = "none" | "task" | "submissionId"; + +export interface AggregateStats { + min: number; + max: number; + mean: number; + stdDev: number; +} + +export interface GroupUniformValues { + workerType?: string; + model?: string; + agentVersion?: string; + platform?: string; + mcpServers?: string[]; + skillRevisions?: string[]; + extensions?: string[]; + status?: "pending" | "processing" | "done"; + submissionId?: string; + task?: string; +} + +export interface GroupAggregates { + count: number; + turns: AggregateStats | null; + duration: AggregateStats | null; + promptTokens: AggregateStats | null; + completionTokens: AggregateStats | null; + statusCounts: Record; + outcomeCounts: Record; +} + +export interface RunGroup { + key: string; + label: string; + runIds: string[]; + aggregates: GroupAggregates; + uniform: GroupUniformValues; +} + // Criteria definition (loaded from config/criteria/*.yaml for v2 scenarios) export interface CriteriaConfig { id: string;