From 65dfdb7d4b8ebb6f2d584a8817289e7fad5f3e26 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 18:50:14 -0700 Subject: [PATCH 01/28] docs: add plan for server-side runs grouping (#568) --- plan.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 plan.md diff --git a/plan.md b/plan.md new file mode 100644 index 000000000..78bbd920f --- /dev/null +++ b/plan.md @@ -0,0 +1,104 @@ +# Plan: Move Runs Grouping Server Side + +**Issue:** https://github.com/growth-ecosystems/scope-core/issues/568 + +## Problem + +Runs grouping is currently implemented client-side in [`apps/portal/src/lib/grouping.ts`](apps/portal/src/lib/grouping.ts). The portal fetches **all** runs from `GET /api/v1/requests`, then groups and aggregates them in the browser. This blocks server-side pagination because: + +1. Pagination returns a page of runs, but grouping needs visibility across ALL runs to compute correct groups and aggregates. +2. A paginated response can split a group across pages, producing wrong counts, stats, and uniform-value detection. + +## Solution + +Add a new **server-side grouping endpoint** that performs the `GROUP BY` and aggregation in CosmosDB/MongoDB, returning pre-computed group metadata. The portal then fetches groups first and drills into individual runs per group on demand. + +## Steps + +### Step 1 — New `GET /api/v1/requests/groups` endpoint + +Add a new API route that accepts: +- All existing filters: `worker`, `taskPromptId`, `criteria`, `submissionId`, `includeDeleted` +- `groupBy`: `"task"` | `"submissionId"` (required) + +Returns an array of group summaries (no individual runs): + +```ts +interface GroupResponse { + key: string; // taskPromptId or submissionId + label: string; // human-readable label (task name, submission ID) + count: number; // number of runs in group + aggregates: { + turns: AggregateStats | null; + duration: AggregateStats | null; + promptTokens: AggregateStats | null; + completionTokens: AggregateStats | null; + }; + uniform: GroupUniformValues; +} +``` + +**Implementation:** Use MongoDB aggregation pipeline: +1. `$match` — apply the same filters as the existing list endpoint +2. `$group` — group by `taskPromptId` or `submissionId`, accumulate arrays of per-run values (turn count, duration, tokens) and collect candidate uniform fields +3. `$project` — compute min/max/mean/stdDev from accumulated arrays, detect uniform values + +This keeps all computation in the database and returns only lightweight group summaries. + +### Step 2 — Add pagination to `GET /api/v1/requests` + +Extend the existing list endpoint with: +- `groupKey`: value to filter by (e.g., a specific `taskPromptId` or `submissionId`) +- `groupBy`: which field the key refers to (`"task"` | `"submissionId"`) +- `limit` / `offset` (or `continuationToken` for CosmosDB-native pagination) + +When `groupKey` + `groupBy` are provided, the endpoint filters runs to that single group and paginates within it. This is used when the user expands a group row in the UI. + +### Step 3 — Move shared types to `packages/shared` + +Move the `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, and `RunGroup` interfaces from `apps/portal/src/lib/grouping.ts` to `packages/shared/src/types/` so both API and portal can import them. + +### Step 4 — Portal API client + +Add `api.listRunGroups(opts)` method in `apps/portal/src/lib/api.ts` that calls the new groups endpoint. + +Update `api.listRuns(opts)` to accept `groupKey`, `groupBy`, `limit`, `offset` params. + +### Step 5 — Update `RunsList.tsx` + +When `groupBy !== "none"`: +1. Fetch groups from `api.listRunGroups(...)` instead of fetching all runs. +2. Render group rows from the server response (no client-side `groupRuns()` call). +3. On group expand, fetch that group's runs via `api.listRuns({ groupBy, groupKey, limit, offset })`. +4. Support "load more" or pagination within an expanded group. + +When `groupBy === "none"`: +1. Fetch runs via `api.listRuns({ limit, offset })` with pagination. +2. Render flat rows, no client-side grouping. + +### Step 6 — Deprecate client-side grouping + +Keep `grouping.ts` and its tests intact for now (no breaking change), but the portal no longer calls `groupRuns()` in the main list view. It can be removed in a follow-up. + +### Step 7 — Tests + +- **API unit tests** (`endpoints.test.ts`): test the new `/requests/groups` endpoint with different `groupBy` values and filters. +- **API unit tests**: test pagination params on the existing `/requests` endpoint. +- **Portal tests**: update `RunsList` tests if any exist; test the new `api.listRunGroups` client method. + +## File Changes Summary + +| File | Change | +|------|--------| +| `packages/shared/src/types/types.ts` | Add `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, group response types | +| `apps/api/src/index.ts` | Add `GET /api/v1/requests/groups` route; add `groupKey`/`groupBy`/`limit`/`offset` to list route | +| `apps/api/src/endpoints.test.ts` | Tests for new groups endpoint and pagination | +| `apps/portal/src/lib/api.ts` | Add `listRunGroups()`; extend `listRuns()` with pagination + group filter params | +| `apps/portal/src/pages/RunsList.tsx` | Use server groups when `groupBy !== "none"`; paginate flat list when `groupBy === "none"` | +| `apps/portal/src/lib/grouping.ts` | Keep as-is (deprecated, not deleted) | + +## Risks & Notes + +- **CosmosDB aggregation pipeline support**: CosmosDB for MongoDB API supports `$group`, `$project`, `$match`, `$sort`, `$addFields`, `$unwind`, and the needed accumulator operators (`$min`, `$max`, `$avg`, `$push`, `$sum`). Standard deviation (`$stdDevPop`) is supported in MongoDB 3.2+ API — verify CosmosDB compatibility or compute it from the pushed array in `$project`. +- **Index coverage**: The `$group` stage operates after `$match`, so existing indexes on `workerType`, `taskPromptId`, `submissionId`, `deletedAt`, `createdAt` should be sufficient. +- **Breaking change**: None — the existing endpoint retains its current behavior when the new params are absent. From 260a8bd7fef8fd0cbde548827e7fd6d678bed9a9 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 18:55:48 -0700 Subject: [PATCH 02/28] =?UTF-8?q?docs:=20revise=20plan=20=E2=80=94=20singl?= =?UTF-8?q?e=20endpoint=20with=20groupBy=20param,=20server-side=20aggregat?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plan.md | 104 ++++++++++++++++++++++++++------------------------------ 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/plan.md b/plan.md index 78bbd920f..bade4480c 100644 --- a/plan.md +++ b/plan.md @@ -4,101 +4,93 @@ ## Problem -Runs grouping is currently implemented client-side in [`apps/portal/src/lib/grouping.ts`](apps/portal/src/lib/grouping.ts). The portal fetches **all** runs from `GET /api/v1/requests`, then groups and aggregates them in the browser. This blocks server-side pagination because: +Runs grouping is currently implemented client-side in `apps/portal/src/lib/grouping.ts`. The portal fetches **all** runs from `GET /api/v1/requests`, then groups and aggregates them in the browser. This blocks server-side pagination because: 1. Pagination returns a page of runs, but grouping needs visibility across ALL runs to compute correct groups and aggregates. 2. A paginated response can split a group across pages, producing wrong counts, stats, and uniform-value detection. ## Solution -Add a new **server-side grouping endpoint** that performs the `GROUP BY` and aggregation in CosmosDB/MongoDB, returning pre-computed group metadata. The portal then fetches groups first and drills into individual runs per group on demand. +Add a `groupBy` query parameter to the existing `GET /api/v1/requests` endpoint. When absent, the endpoint behaves exactly as today (flat array of runs). When set to `task` or `submissionId`, it runs a MongoDB aggregation pipeline server-side and returns pre-computed group summaries — including all aggregation (turns, duration, tokens stats) and uniform-value detection — so the portal renders them directly with zero client-side computation. ## Steps -### Step 1 — New `GET /api/v1/requests/groups` endpoint +### Step 1 — Move shared types to `packages/shared` -Add a new API route that accepts: -- All existing filters: `worker`, `taskPromptId`, `criteria`, `submissionId`, `includeDeleted` -- `groupBy`: `"task"` | `"submissionId"` (required) +Move `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, and `RunGroup` interfaces from `apps/portal/src/lib/grouping.ts` to `packages/shared/src/types/` so both API and portal can import them. -Returns an array of group summaries (no individual runs): +### Step 2 — Add `groupBy` param to `GET /api/v1/requests` -```ts -interface GroupResponse { - key: string; // taskPromptId or submissionId - label: string; // human-readable label (task name, submission ID) - count: number; // number of runs in group - aggregates: { - turns: AggregateStats | null; - duration: AggregateStats | null; - promptTokens: AggregateStats | null; - completionTokens: AggregateStats | null; - }; - uniform: GroupUniformValues; -} -``` - -**Implementation:** Use MongoDB aggregation pipeline: -1. `$match` — apply the same filters as the existing list endpoint -2. `$group` — group by `taskPromptId` or `submissionId`, accumulate arrays of per-run values (turn count, duration, tokens) and collect candidate uniform fields -3. `$project` — compute min/max/mean/stdDev from accumulated arrays, detect uniform values +Add optional `groupBy` query param (`"task"` | `"submissionId"`) to the existing list endpoint. -This keeps all computation in the database and returns only lightweight group summaries. +When `groupBy` is provided, run a MongoDB aggregation pipeline: +1. `$match` — same filters as today (worker, taskPromptId, criteria, submissionId, deletedAt) +2. `$addFields` — compute per-run derived values: turn count, total duration (sum of `turn.durationMs`), prompt/completion tokens +3. `$group` — group by `taskPromptId` (or `submissionId`), accumulate min/max/avg/count for turns, duration, tokens; collect first values and distinct-count for uniform-value detection; grab first `scenario.task` for label +4. `$project` — reshape into `RunGroup` response format with `AggregateStats` and `GroupUniformValues` -### Step 2 — Add pagination to `GET /api/v1/requests` +Response shape when grouped: +```ts +{ + key: string; + label: string; + count: number; + aggregates: GroupAggregates; // turns, duration, promptTokens, completionTokens stats + uniform: GroupUniformValues; // fields identical across all runs in group +}[] +``` -Extend the existing list endpoint with: -- `groupKey`: value to filter by (e.g., a specific `taskPromptId` or `submissionId`) -- `groupBy`: which field the key refers to (`"task"` | `"submissionId"`) -- `limit` / `offset` (or `continuationToken` for CosmosDB-native pagination) +When `groupBy` is absent, behavior is unchanged (flat `RequestDocument[]`). -When `groupKey` + `groupBy` are provided, the endpoint filters runs to that single group and paginates within it. This is used when the user expands a group row in the UI. +### Step 3 — Server-side aggregation logic -### Step 3 — Move shared types to `packages/shared` +Implement the aggregation computation (stats + uniform values) in a dedicated module `apps/api/src/grouping.ts` that builds the MongoDB pipeline stages. This keeps `index.ts` clean and the logic unit-testable. -Move the `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, and `RunGroup` interfaces from `apps/portal/src/lib/grouping.ts` to `packages/shared/src/types/` so both API and portal can import them. +The aggregation computes **all** the same metrics the client currently does: +- **Counts**: number of runs per group +- **Stats** (min/max/mean/stdDev): turn count, total duration, prompt tokens, completion tokens +- **Uniform values**: workerType, model, agentVersion, platform, mcpServers, skillRevisions, status, submissionId, task — included only when all runs in the group share the same value ### Step 4 — Portal API client -Add `api.listRunGroups(opts)` method in `apps/portal/src/lib/api.ts` that calls the new groups endpoint. - -Update `api.listRuns(opts)` to accept `groupKey`, `groupBy`, `limit`, `offset` params. +Update `api.listRuns()` in `apps/portal/src/lib/api.ts` to accept an optional `groupBy` param. Add a separate `api.listRunGroups(opts)` method (or overload) that returns `RunGroup[]` when grouping is requested. ### Step 5 — Update `RunsList.tsx` When `groupBy !== "none"`: -1. Fetch groups from `api.listRunGroups(...)` instead of fetching all runs. -2. Render group rows from the server response (no client-side `groupRuns()` call). -3. On group expand, fetch that group's runs via `api.listRuns({ groupBy, groupKey, limit, offset })`. -4. Support "load more" or pagination within an expanded group. +1. Call `api.listRunGroups(...)` → get `RunGroup[]` from server with pre-computed aggregates. +2. Render group rows directly from server response — no client-side `groupRuns()` call. +3. On group expand, fetch that group's runs via `api.listRuns({ taskPromptId })` or `api.listRuns({ submissionId })`. When `groupBy === "none"`: -1. Fetch runs via `api.listRuns({ limit, offset })` with pagination. -2. Render flat rows, no client-side grouping. +1. Fetch runs via `api.listRuns()` as today. +2. Render flat rows, no grouping. ### Step 6 — Deprecate client-side grouping -Keep `grouping.ts` and its tests intact for now (no breaking change), but the portal no longer calls `groupRuns()` in the main list view. It can be removed in a follow-up. +Keep `grouping.ts` and its tests intact (no breaking change), but the portal no longer calls `groupRuns()` in the main list view. Can be removed in a follow-up. ### Step 7 — Tests -- **API unit tests** (`endpoints.test.ts`): test the new `/requests/groups` endpoint with different `groupBy` values and filters. -- **API unit tests**: test pagination params on the existing `/requests` endpoint. -- **Portal tests**: update `RunsList` tests if any exist; test the new `api.listRunGroups` client method. +- **API unit tests** (`apps/api/src/grouping.test.ts`): test the aggregation pipeline builder with different `groupBy` values and filters. +- **API endpoint tests** (`endpoints.test.ts`): test `GET /api/v1/requests?groupBy=task` returns grouped response. +- **Portal tests**: update any `RunsList` tests to use server-side groups. ## File Changes Summary | File | Change | |------|--------| -| `packages/shared/src/types/types.ts` | Add `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, group response types | -| `apps/api/src/index.ts` | Add `GET /api/v1/requests/groups` route; add `groupKey`/`groupBy`/`limit`/`offset` to list route | -| `apps/api/src/endpoints.test.ts` | Tests for new groups endpoint and pagination | -| `apps/portal/src/lib/api.ts` | Add `listRunGroups()`; extend `listRuns()` with pagination + group filter params | -| `apps/portal/src/pages/RunsList.tsx` | Use server groups when `groupBy !== "none"`; paginate flat list when `groupBy === "none"` | +| `packages/shared/src/types/types.ts` | Add `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, `RunGroup` types | +| `apps/api/src/grouping.ts` | New — builds MongoDB aggregation pipeline for grouping + stats + uniform values | +| `apps/api/src/grouping.test.ts` | New — tests for aggregation pipeline builder | +| `apps/api/src/index.ts` | Add `groupBy` query param to list endpoint; call pipeline when set | +| `apps/api/src/endpoints.test.ts` | Test grouped response from list endpoint | +| `apps/portal/src/lib/api.ts` | Add `listRunGroups()` method | +| `apps/portal/src/pages/RunsList.tsx` | Use server groups when grouped; stop calling client-side `groupRuns()` | | `apps/portal/src/lib/grouping.ts` | Keep as-is (deprecated, not deleted) | ## Risks & Notes -- **CosmosDB aggregation pipeline support**: CosmosDB for MongoDB API supports `$group`, `$project`, `$match`, `$sort`, `$addFields`, `$unwind`, and the needed accumulator operators (`$min`, `$max`, `$avg`, `$push`, `$sum`). Standard deviation (`$stdDevPop`) is supported in MongoDB 3.2+ API — verify CosmosDB compatibility or compute it from the pushed array in `$project`. -- **Index coverage**: The `$group` stage operates after `$match`, so existing indexes on `workerType`, `taskPromptId`, `submissionId`, `deletedAt`, `createdAt` should be sufficient. -- **Breaking change**: None — the existing endpoint retains its current behavior when the new params are absent. +- **CosmosDB aggregation support**: CosmosDB for MongoDB API supports `$group`, `$project`, `$match`, `$sort`, `$addFields`, `$unwind`, `$min`, `$max`, `$avg`, `$push`, `$sum`. `$stdDevPop` is supported in MongoDB 3.2+ — verify CosmosDB compatibility or compute stdDev from pushed arrays in `$project`. +- **Index coverage**: `$group` runs after `$match`, so existing indexes on `workerType`, `taskPromptId`, `submissionId`, `deletedAt`, `createdAt` remain sufficient. +- **Breaking change**: None — `groupBy` is optional; when absent the endpoint returns the same flat array as before. From db42ad9d9b4363f50fbdf9c8e3495fb639464b7e Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 18:59:44 -0700 Subject: [PATCH 03/28] feat: add shared grouping types and schemas (#568) --- packages/shared/src/schemas/request.ts | 43 ++++++++++++++++++++++++++ packages/shared/src/types/types.ts | 38 +++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index c96e7fc6c..35ec16c61 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -136,11 +136,54 @@ export const ListRequestsQuerySchema = z submissionId: z.string().optional(), status: RequestStatusSchema.optional(), outcome: RequestOutcomeSchema.optional(), + groupBy: z.enum(["task", "submissionId"]).optional(), page: z.coerce.number().int().min(1).optional(), limit: z.coerce.number().int().min(1).max(100).optional(), }) .openapi("ListRequestsQuery"); +export const AggregateStatsSchema = z + .object({ + min: z.number(), + max: z.number(), + mean: z.number(), + stdDev: z.number(), + }) + .openapi("AggregateStats"); + +export const GroupUniformValuesSchema = z + .object({ + workerType: z.string().optional(), + model: z.string().optional(), + agentVersion: z.string().optional(), + platform: z.string().optional(), + mcpServers: z.array(z.string()).optional(), + skillRevisions: z.array(z.string()).optional(), + status: RequestStatusSchema.optional(), + submissionId: z.string().optional(), + task: z.string().optional(), + }) + .openapi("GroupUniformValues"); + +export const GroupAggregatesSchema = z + .object({ + count: z.number(), + turns: AggregateStatsSchema.nullable(), + duration: AggregateStatsSchema.nullable(), + promptTokens: AggregateStatsSchema.nullable(), + completionTokens: AggregateStatsSchema.nullable(), + }) + .openapi("GroupAggregates"); + +export const RunGroupSchema = z + .object({ + key: z.string(), + label: z.string(), + aggregates: GroupAggregatesSchema, + uniform: GroupUniformValuesSchema, + }) + .openapi("RunGroup"); + export const BulkResubmitInputSchema = z .object({ ids: z.array(z.string()).min(1), diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 35f0402dc..82159733b 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -241,6 +241,44 @@ export interface QueueProcessorConfig extends BaseQueueProcessorConfig { // --- Enhanced Criteria System types --- +// --- Runs grouping types (shared between API and portal) --- + +export type GroupByKey = "none" | "task" | "submissionId"; + +export interface AggregateStats { + min: number; + max: number; + mean: number; + stdDev: number; +} + +export interface GroupUniformValues { + workerType?: string; + model?: string; + agentVersion?: string; + platform?: string; + mcpServers?: string[]; + skillRevisions?: string[]; + status?: "pending" | "processing" | "done"; + submissionId?: string; + task?: string; +} + +export interface GroupAggregates { + count: number; + turns: AggregateStats | null; + duration: AggregateStats | null; + promptTokens: AggregateStats | null; + completionTokens: AggregateStats | null; +} + +export interface RunGroup { + key: string; + label: string; + aggregates: GroupAggregates; + uniform: GroupUniformValues; +} + // Criteria definition (loaded from config/criteria/*.yaml for v2 scenarios) export interface CriteriaConfig { id: string; From 9809df8a07a2d9024f03de01ca9d4938fa9b4436 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 19:01:13 -0700 Subject: [PATCH 04/28] feat: add groupBy param to list endpoint with aggregation pipeline (#568) --- apps/api/src/grouping.ts | 172 +++++++++++++++++++++++++++++++++++++++ apps/api/src/index.ts | 17 +++- 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/grouping.ts diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts new file mode 100644 index 000000000..a526e121d --- /dev/null +++ b/apps/api/src/grouping.ts @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Document } from "mongodb"; +import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "shared"; + +/** + * Build a MongoDB aggregation pipeline that groups runs and computes + * per-group aggregates (count, stats on turns/duration/tokens) and + * uniform-value detection. + * + * The pipeline is appended after a $match stage with the caller's filter. + */ +export function buildGroupingPipeline( + groupBy: "task" | "submissionId", +): Document[] { + const groupField = + groupBy === "task" ? "$taskPromptId" : "$submissionId"; + const fallbackGroupField = + groupBy === "task" ? "$scenario.task" : { $literal: "no-submission" }; + + return [ + // 1. Compute per-run derived values + { + $addFields: { + _groupKey: { $ifNull: [groupField, fallbackGroupField] }, + _turnCount: { $cond: { if: { $isArray: "$turns" }, then: { $size: "$turns" }, else: null } }, + _duration: { + $cond: { + if: { $and: [{ $isArray: "$turns" }, { $gt: [{ $size: "$turns" }, 0] }] }, + then: { + $reduce: { + input: "$turns", + initialValue: 0, + in: { $add: ["$$value", { $ifNull: ["$$this.durationMs", 0] }] }, + }, + }, + else: null, + }, + }, + _promptTokens: { + $cond: { + if: "$tokenUsage", + then: "$tokenUsage.promptTokens", + else: { + $cond: { + if: { $and: [{ $isArray: "$turns" }, { $gt: [{ $size: { $filter: { input: { $ifNull: ["$turns", []] }, cond: { $ne: ["$$this.tokenUsage", null] } } } }, 0] }] }, + then: { + $reduce: { + input: "$turns", + initialValue: 0, + in: { $add: ["$$value", { $ifNull: ["$$this.tokenUsage.promptTokens", 0] }] }, + }, + }, + else: null, + }, + }, + }, + }, + _completionTokens: { + $cond: { + if: "$tokenUsage", + then: "$tokenUsage.completionTokens", + else: { + $cond: { + if: { $and: [{ $isArray: "$turns" }, { $gt: [{ $size: { $filter: { input: { $ifNull: ["$turns", []] }, cond: { $ne: ["$$this.tokenUsage", null] } } } }, 0] }] }, + then: { + $reduce: { + input: "$turns", + initialValue: 0, + in: { $add: ["$$value", { $ifNull: ["$$this.tokenUsage.completionTokens", 0] }] }, + }, + }, + else: null, + }, + }, + }, + }, + // Sorted-joined array keys for uniform comparison + _mcpKey: { + $reduce: { + input: { $sortArray: { input: { $ifNull: ["$mcpServers", []] }, sortBy: 1 } }, + initialValue: "", + in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, + }, + }, + _skillKey: { + $reduce: { + input: { $sortArray: { input: { $ifNull: ["$skillRevisions", []] }, sortBy: 1 } }, + initialValue: "", + in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, + }, + }, + }, + }, + // 2. Group and accumulate per-run values + { + $group: { + _id: "$_groupKey", + count: { $sum: 1 }, + // Label: take first task name for display + _firstTask: { $first: "$scenario.task" }, + // Push arrays for stats computation + _turnCounts: { $push: "$_turnCount" }, + _durations: { $push: "$_duration" }, + _promptTokensList: { $push: "$_promptTokens" }, + _completionTokensList: { $push: "$_completionTokens" }, + // Uniform detection: collect distinct values + _workerTypes: { $addToSet: "$workerType" }, + _models: { $addToSet: { $ifNull: ["$model", ""] } }, + _agentVersions: { $addToSet: { $ifNull: ["$agentVersion", ""] } }, + _platforms: { $addToSet: { $ifNull: ["$os.platform", ""] } }, + _statuses: { $addToSet: "$status" }, + _submissionIds: { $addToSet: { $ifNull: ["$submissionId", ""] } }, + _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, + _mcpKeys: { $addToSet: "$_mcpKey" }, + _skillKeys: { $addToSet: "$_skillKey" }, + // Keep first run's array values for uniform fields + _firstMcpServers: { $first: "$mcpServers" }, + _firstSkillRevisions: { $first: "$skillRevisions" }, + }, + }, + // 3. Project into final shape + { + $project: { + _id: 0, + key: "$_id", + label: { + $cond: { + if: { $eq: ["$_id", "no-submission"] }, + then: "No submission ID", + else: { $ifNull: ["$_firstTask", "$_id"] }, + }, + }, + aggregates: { + count: "$count", + turns: { $let: { vars: { vals: { $filter: { input: "$_turnCounts", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + duration: { $let: { vars: { vals: { $filter: { input: "$_durations", cond: { $and: [{ $ne: ["$$this", null] }, { $gt: ["$$this", 0] }] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + promptTokens: { $let: { vars: { vals: { $filter: { input: "$_promptTokensList", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + completionTokens: { $let: { vars: { vals: { $filter: { input: "$_completionTokensList", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + }, + uniform: { + workerType: { $cond: { if: { $eq: [{ $size: "$_workerTypes" }, 1] }, then: { $arrayElemAt: ["$_workerTypes", 0] }, else: "$$REMOVE" } }, + model: { $cond: { if: { $and: [{ $eq: [{ $size: "$_models" }, 1] }, { $ne: [{ $arrayElemAt: ["$_models", 0] }, ""] }] }, then: { $arrayElemAt: ["$_models", 0] }, else: "$$REMOVE" } }, + agentVersion: { $cond: { if: { $and: [{ $eq: [{ $size: "$_agentVersions" }, 1] }, { $ne: [{ $arrayElemAt: ["$_agentVersions", 0] }, ""] }] }, then: { $arrayElemAt: ["$_agentVersions", 0] }, else: "$$REMOVE" } }, + platform: { $cond: { if: { $and: [{ $eq: [{ $size: "$_platforms" }, 1] }, { $ne: [{ $arrayElemAt: ["$_platforms", 0] }, ""] }] }, then: { $arrayElemAt: ["$_platforms", 0] }, else: "$$REMOVE" } }, + status: { $cond: { if: { $eq: [{ $size: "$_statuses" }, 1] }, then: { $arrayElemAt: ["$_statuses", 0] }, else: "$$REMOVE" } }, + submissionId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_submissionIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_submissionIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_submissionIds", 0] }, else: "$$REMOVE" } }, + task: { $cond: { if: { $and: [{ $eq: [{ $size: "$_tasks" }, 1] }, { $ne: [{ $arrayElemAt: ["$_tasks", 0] }, ""] }] }, then: { $arrayElemAt: ["$_tasks", 0] }, else: "$$REMOVE" } }, + mcpServers: { $cond: { if: { $and: [{ $eq: [{ $size: "$_mcpKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_mcpKeys", 0] }, ""] }] }, then: "$_firstMcpServers", else: "$$REMOVE" } }, + skillRevisions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_skillKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_skillKeys", 0] }, ""] }] }, then: "$_firstSkillRevisions", else: "$$REMOVE" } }, + }, + }, + }, + ]; +} + +/** + * Compute aggregate stats from an array of numbers. + * Used for application-level grouping as a fallback when the MongoDB + * aggregation pipeline cannot be used. + */ +export function computeStats(values: number[]): AggregateStats | null { + if (values.length === 0) return null; + const min = Math.min(...values); + const max = Math.max(...values); + const mean = values.reduce((s, v) => s + v, 0) / values.length; + const variance = + values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; + const stdDev = Math.sqrt(variance); + return { min, max, mean, stdDev }; +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 6a4960744..1aa9074fa 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -90,7 +90,9 @@ import { CreateRequestInputSchema, ListRequestsQuerySchema, BulkResubmitInputSchema, + RunGroupSchema, } from "shared"; +import { buildGroupingPipeline } from "./grouping.js"; import { checkMigrations } from "db-migrations/check-migrations"; import { generateOpenAPIDocument, registry } from "./openapi/index.js"; import swaggerUi from "swagger-ui-express"; @@ -1036,19 +1038,21 @@ apiRoute(app, registry, { }); // List all requests (excludes soft-deleted by default) +// When groupBy is provided, returns RunGroup[] instead of flat RequestDocument[] apiRoute(app, registry, { method: "get", path: "/api/v1/requests", tags: ["Requests"], summary: "List requests", query: ListRequestsQuerySchema, - response: z.array(RequestResponseSchema), + response: z.union([z.array(RequestResponseSchema), z.array(RunGroupSchema)]), handler: async (req, res) => { const workerFilter = req.query.worker as string; const taskPromptIdFilter = req.query.taskPromptId as string; const criteriaFilter = req.query.criteria as string; const submissionIdFilter = req.query.submissionId as string; const includeDeleted = req.query.includeDeleted === "true"; + const groupByParam = req.query.groupBy as "task" | "submissionId" | undefined; const filter: Record = {}; if (workerFilter && VALID_WORKERS.includes(workerFilter as WorkerType)) { @@ -1086,6 +1090,17 @@ apiRoute(app, registry, { } } + // Grouped mode: return RunGroup[] via aggregation pipeline + if (groupByParam) { + const pipeline = [ + { $match: filter }, + ...buildGroupingPipeline(groupByParam), + ]; + const groups = await collection.aggregate(pipeline).toArray(); + res.json(groups); + return; + } + const resources = await collection .find(filter) .sort({ createdAt: -1 }) From ac10a3e5c87fa093f854dd378a0050ab8355a2f6 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 19:02:10 -0700 Subject: [PATCH 05/28] feat: add RunGroup types and api.listRunGroups() in portal (#568) --- apps/portal/src/lib/api.ts | 13 ++++++++++++- apps/portal/src/types.ts | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 77d07693d..fdbe71e8f 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillRevisionDocument, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest } from "@/types"; +import type { Run, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillRevisionDocument, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, RunGroup } from "@/types"; const BASE = "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/api/v1"; @@ -30,6 +30,17 @@ export const api = { return request(`/requests${qs ? `?${qs}` : ""}`); }, + /** List runs grouped by task or submissionId, with server-computed aggregates */ + listRunGroups: (opts: { groupBy: "task" | "submissionId"; worker?: string; taskPromptId?: string; criteria?: string; submissionId?: string }): Promise => { + const params = new URLSearchParams(); + params.set("groupBy", opts.groupBy); + if (opts.worker) params.set("worker", opts.worker); + if (opts.taskPromptId) params.set("taskPromptId", opts.taskPromptId); + if (opts.criteria) params.set("criteria", opts.criteria); + if (opts.submissionId) params.set("submissionId", opts.submissionId); + return request(`/requests?${params.toString()}`); + }, + /** Get a single run by ID */ getRun: (id: string): Promise => { return request(`/requests/${id}`); diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index c81754c96..fd987e431 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -798,3 +798,41 @@ export interface FeatureFlag { enabled: boolean; updatedAt: string; } + +// --- Runs grouping types (mirrored from shared) --- + +export type GroupByKey = "none" | "task" | "submissionId"; + +export interface AggregateStats { + min: number; + max: number; + mean: number; + stdDev: number; +} + +export interface GroupUniformValues { + workerType?: string; + model?: string; + agentVersion?: string; + platform?: string; + mcpServers?: string[]; + skillRevisions?: string[]; + status?: RunStatus; + submissionId?: string; + task?: string; +} + +export interface GroupAggregates { + count: number; + turns: AggregateStats | null; + duration: AggregateStats | null; + promptTokens: AggregateStats | null; + completionTokens: AggregateStats | null; +} + +export interface RunGroup { + key: string; + label: string; + aggregates: GroupAggregates; + uniform: GroupUniformValues; +} From 9c78ef31b8bead6994cc8c55549117306a23b3ed Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 19:05:20 -0700 Subject: [PATCH 06/28] feat: use server-side groups in RunsList, fetch runs on expand (#568) --- apps/portal/src/pages/RunsList.tsx | 282 ++++++++++------------------- 1 file changed, 91 insertions(+), 191 deletions(-) diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index c91bbf1df..2ee23dd85 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -23,8 +23,8 @@ import { StatusBadge, OutcomeBadge } from "@/components/StatusBadge"; import { Trash2, Eye, Plus, RefreshCw, Repeat, FileText, X, Download, ChevronRight, ChevronDown } from "lucide-react"; import { formatDate, formatId, truncate, formatDuration } from "@/lib/utils"; import { WORKER_TYPES, STATUS_LIST, OUTCOME_LIST } from "@/types"; -import type { Run, BulkResubmitOverrides, McpServerDocument, CodingAgent, BulkReportSummary } from "@/types"; -import { groupRuns, formatStatRange, type GroupByKey, type RunGroup } from "@/lib/grouping"; +import type { Run, BulkResubmitOverrides, McpServerDocument, CodingAgent, BulkReportSummary, RunGroup, GroupByKey } from "@/types"; +import { formatStatRange } from "@/lib/grouping"; export function RunsList() { const [searchParams, setSearchParams] = useSearchParams(); @@ -54,6 +54,20 @@ export function RunsList() { refetchInterval: 10_000, }); + // Fetch server-side groups when groupBy is active + const { data: serverGroups = [], isLoading: isGroupsLoading, isRefetching: isGroupsRefetching } = useQuery({ + queryKey: ["run-groups", groupBy, workerFilter, taskPromptId, criteriaState, submissionId], + queryFn: () => api.listRunGroups({ + groupBy: groupBy as "task" | "submissionId", + worker: workerFilter === "all" ? undefined : workerFilter, + taskPromptId, + criteria: criteriaState, + submissionId, + }), + enabled: groupBy !== "none", + refetchInterval: 10_000, + }); + // Fetch MCP servers for the resubmit dialog const { data: mcpServers = [] } = useQuery({ queryKey: ["mcp-servers"], @@ -181,7 +195,7 @@ export function RunsList() { return true; }); - const runGroups = useMemo(() => groupRuns(filteredRuns, groupBy), [filteredRuns, groupBy]); + const runGroups = serverGroups; const toggleGroup = (key: string) => { setExpandedGroups((prev) => { @@ -192,20 +206,6 @@ export function RunsList() { }); }; - const toggleGroupSelect = (group: RunGroup) => { - setSelectedIds((prev) => { - const next = new Set(prev); - const groupIds = group.runs.map((r) => r._id); - const allInGroupSelected = groupIds.every((id) => prev.has(id)); - if (allInGroupSelected) { - groupIds.forEach((id) => next.delete(id)); - } else { - groupIds.forEach((id) => next.add(id)); - } - return next; - }); - }; - const allSelected = filteredRuns.length > 0 && filteredRuns.every((r) => selectedIds.has(r._id)); const someSelected = filteredRuns.some((r) => selectedIds.has(r._id)); @@ -318,10 +318,11 @@ export function RunsList() {
- {isRefetching && } + {(isRefetching || isGroupsRefetching) && } - {filteredRuns.length} run{filteredRuns.length !== 1 ? "s" : ""} - {groupBy !== "none" && ` in ${runGroups.length} group${runGroups.length !== 1 ? "s" : ""}`} + {groupBy !== "none" + ? `${runGroups.length} group${runGroups.length !== 1 ? "s" : ""}` + : `${filteredRuns.length} run${filteredRuns.length !== 1 ? "s" : ""}`}
@@ -840,13 +841,13 @@ export function RunsList() { )} {/* Table */} - {isLoading ? ( + {(groupBy === "none" ? isLoading : isGroupsLoading) ? (
{Array.from({ length: 5 }).map((_, i) => ( ))}
- ) : filteredRuns.length === 0 ? ( + ) : (groupBy === "none" && filteredRuns.length === 0) || (groupBy !== "none" && runGroups.length === 0) ? (
No runs found. Submit one?
@@ -855,11 +856,13 @@ export function RunsList() { - + {groupBy === "none" && ( + + )} ID Submission @@ -884,25 +887,21 @@ export function RunsList() { {groupBy !== "none" ? ( runGroups.map((group) => { const isExpanded = expandedGroups.has(group.key); - const groupIds = group.runs.map((r) => r._id); - const allGroupSelected = groupIds.length > 0 && groupIds.every((id) => selectedIds.has(id)); - const someGroupSelected = groupIds.some((id) => selectedIds.has(id)); return ( toggleGroup(group.key)} - allGroupSelected={allGroupSelected} - someGroupSelected={someGroupSelected} - onToggleGroupSelect={() => toggleGroupSelect(group)} selectedIds={selectedIds} onToggleSelect={toggleSelect} reportSummaries={reportSummaries} deleteMutation={deleteMutation} - bulkDeleteMutation={bulkDeleteMutation} - bulkReportMutation={bulkReportMutation} groupBy={groupBy} + workerFilter={workerFilter === "all" ? undefined : workerFilter} + criteriaState={criteriaState} + submissionId={submissionId} + taskPromptId={taskPromptId} /> ); }) @@ -1102,35 +1101,52 @@ function GroupRows({ group, isExpanded, onToggleExpand, - allGroupSelected, - someGroupSelected, - onToggleGroupSelect, selectedIds, onToggleSelect, reportSummaries, deleteMutation, - bulkDeleteMutation, - bulkReportMutation, groupBy, + workerFilter, + criteriaState, + submissionId, + taskPromptId, }: { group: RunGroup; isExpanded: boolean; onToggleExpand: () => void; - allGroupSelected: boolean; - someGroupSelected: boolean; - onToggleGroupSelect: () => void; selectedIds: Set; onToggleSelect: (id: string) => void; reportSummaries: BulkReportSummary | undefined; deleteMutation: { mutate: (id: string) => void; isPending: boolean }; - bulkDeleteMutation: { mutate: (ids: string[]) => void; isPending: boolean }; - bulkReportMutation: { mutate: (ids: string[]) => void; isPending: boolean }; groupBy: GroupByKey; + workerFilter?: string; + criteriaState?: string; + submissionId?: string; + taskPromptId?: string; }) { const { aggregates, uniform } = group; const fmtDur = (v: number) => formatDuration(Math.round(v)); const fmtNum = (v: number) => Math.round(v).toLocaleString(); + // Fetch runs for this group on expand + const expandFilter = useMemo(() => { + const opts: Record = { worker: workerFilter, criteria: criteriaState }; + if (groupBy === "task") { + // group.key is taskPromptId (or scenario.task fallback) + opts.taskPromptId = group.key; + } else { + opts.submissionId = group.key === "no-submission" ? undefined : group.key; + } + return opts; + }, [group.key, groupBy, workerFilter, criteriaState]); + + const { data: expandedRuns = [], isLoading: isExpandLoading } = useQuery({ + queryKey: ["group-runs", group.key, groupBy, workerFilter, criteriaState], + queryFn: () => api.listRuns(expandFilter), + enabled: isExpanded, + refetchInterval: 10_000, + }); + return ( <> {/* Checkbox */} - e.stopPropagation()}> - - + {/* ID */}
@@ -1237,93 +1247,24 @@ function GroupRows({ {/* Status */} - {(() => { - const statusColors: Record = { - pending: "bg-gray-500", - processing: "bg-blue-500", - done: "bg-green-500", - }; - const completed = group.runs.filter((r) => r.status === "done").length; - const total = group.runs.length; - const segments = Object.entries( - group.runs.reduce>((acc, r) => { - acc[r.status] = (acc[r.status] ?? 0) + 1; - return acc; - }, {}), - ); - return ( -
- {completed}/{total} done -
- {segments.map(([status, count]) => ( -
- ))} -
-
- ); - })()} + {uniform.status ? ( + + ) : mixed} {/* Outcome */} - {(() => { - const outcomeColors: Record = { - succeeded: "bg-green-500", - failed: "bg-red-500", - exhausted: "bg-yellow-500", - }; - const doneRuns = group.runs.filter((r) => r.status === "done" && r.outcome); - if (doneRuns.length === 0) return –; - const segments = Object.entries( - doneRuns.reduce>((acc, r) => { - acc[r.outcome!] = (acc[r.outcome!] ?? 0) + 1; - return acc; - }, {}), - ); - const succeeded = doneRuns.filter((r) => r.outcome === "succeeded").length; - return ( -
- {succeeded}/{doneRuns.length} pass -
- {segments.map(([outcome, count]) => ( -
- ))} -
-
- ); - })()} + – {/* Report */} - {(() => { - const runsWithReports = group.runs.filter((r) => reportSummaries?.[r._id]); - if (runsWithReports.length === 0) return –; - let total = 0, completed = 0, pending = 0, generating = 0, failed = 0; - for (const r of group.runs) { - const s = reportSummaries?.[r._id]; - if (!s) continue; - total += s.total; - completed += s.completed; - pending += s.pending; - generating += s.generating; - failed += s.failed; - } - return ; - })()} + – {/* Turns */} {formatStatRange(aggregates.turns, fmtNum)} + {/* LLM Calls */} + {/* Duration */} {formatStatRange(aggregates.duration, fmtDur)} @@ -1337,70 +1278,29 @@ function GroupRows({ {/* Created */} {/* Actions */} - e.stopPropagation()}> -
- - - - - - - Generate reports for {group.aggregates.count} run{group.aggregates.count !== 1 ? "s" : ""}? - - This will queue report generation for all runs in this group. - - - - Cancel - bulkReportMutation.mutate(group.runs.map((r) => r._id))} - disabled={bulkReportMutation.isPending} - > - {bulkReportMutation.isPending ? "Generating…" : "Generate"} - - - - - - - - - - - Delete {group.aggregates.count} run{group.aggregates.count !== 1 ? "s" : ""} in this group? - - This will soft-delete all runs in this group. They can be recovered later if needed. - - - - Cancel - bulkDeleteMutation.mutate(group.runs.map((r) => r._id))} - disabled={bulkDeleteMutation.isPending} - className="bg-destructive text-destructive-foreground hover:bg-destructive/90" - > - {bulkDeleteMutation.isPending ? "Deleting…" : "Delete"} - - - - -
-
+ - {isExpanded && group.runs.map((run) => ( - - ))} + {isExpanded && ( + isExpandLoading ? ( + + + + Loading runs… + + + ) : ( + expandedRuns.map((run) => ( + + )) + ) + )} ); } From 2f8d208d923830766bab126d9e72e62bbcdbcef6 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 19:06:12 -0700 Subject: [PATCH 07/28] test: add grouping pipeline unit tests and endpoint tests (#568) --- apps/api/src/endpoints.test.ts | 54 +++++++++++++++++++ apps/api/src/grouping.test.ts | 98 ++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 apps/api/src/grouping.test.ts diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index b75d8727f..555fef90d 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -393,6 +393,60 @@ describe("API Endpoints", () => { expect(Array.isArray(res.body)).toBe(true); expect(res.body[0]).toHaveProperty("id", "r1"); }); + + it("returns grouped results when groupBy=task", async () => { + const groupedDocs = [ + { key: "tp-1", label: "Build a calculator", aggregates: { count: 3, turns: { min: 1, max: 3, mean: 2, stdDev: 0.8 }, duration: null, promptTokens: null, completionTokens: null }, uniform: { workerType: "coder-acp-copilot" } }, + ]; + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue(groupedDocs), + }); + + const res = await request(app).get("/api/v1/requests?groupBy=task"); + expect(res.status).toBe(200); + expect(Array.isArray(res.body)).toBe(true); + expect(res.body[0]).toHaveProperty("key", "tp-1"); + expect(res.body[0]).toHaveProperty("aggregates"); + expect(res.body[0].aggregates).toHaveProperty("count", 3); + expect(res.body[0]).toHaveProperty("uniform"); + }); + + it("returns grouped results when groupBy=submissionId", async () => { + const groupedDocs = [ + { key: "sub-1", label: "sub-1", aggregates: { count: 2, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, + ]; + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue(groupedDocs), + }); + + const res = await request(app).get("/api/v1/requests?groupBy=submissionId"); + expect(res.status).toBe(200); + expect(res.body[0]).toHaveProperty("key", "sub-1"); + }); + + it("calls aggregate pipeline when groupBy is provided", async () => { + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }); + + await request(app).get("/api/v1/requests?groupBy=task"); + expect(mocks.collection.aggregate).toHaveBeenCalled(); + const pipeline = (mocks.collection.aggregate as any).mock.calls[0][0]; + expect(pipeline[0]).toHaveProperty("$match"); + // buildGroupingPipeline adds 3 stages ($addFields, $group, $project) + expect(pipeline).toHaveLength(4); // $match + 3 from buildGroupingPipeline + }); + + it("does not call aggregate when groupBy is absent", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests"); + expect(mocks.collection.aggregate).not.toHaveBeenCalled(); + }); }); describe("GET /api/v1/requests/:id", () => { diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts new file mode 100644 index 000000000..96a012316 --- /dev/null +++ b/apps/api/src/grouping.test.ts @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect } from "vitest"; +import { buildGroupingPipeline, computeStats } from "./grouping.js"; + +describe("buildGroupingPipeline", () => { + it("returns 3 pipeline stages for task grouping", () => { + const pipeline = buildGroupingPipeline("task"); + expect(pipeline).toHaveLength(3); + expect(pipeline[0]).toHaveProperty("$addFields"); + expect(pipeline[1]).toHaveProperty("$group"); + expect(pipeline[2]).toHaveProperty("$project"); + }); + + it("returns 3 pipeline stages for submissionId grouping", () => { + const pipeline = buildGroupingPipeline("submissionId"); + expect(pipeline).toHaveLength(3); + }); + + it("groups by taskPromptId for task grouping", () => { + const pipeline = buildGroupingPipeline("task"); + const addFields = pipeline[0].$addFields as Record; + // _groupKey should reference $taskPromptId with fallback to scenario.task + expect(addFields._groupKey).toBeDefined(); + }); + + it("groups by submissionId for submissionId grouping", () => { + const pipeline = buildGroupingPipeline("submissionId"); + const addFields = pipeline[0].$addFields as Record; + expect(addFields._groupKey).toBeDefined(); + }); + + it("$group stage uses _groupKey as _id", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._id).toBe("$_groupKey"); + }); + + it("$group stage accumulates count and stat arrays", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group.count).toEqual({ $sum: 1 }); + expect(group._turnCounts).toBeDefined(); + expect(group._durations).toBeDefined(); + expect(group._promptTokensList).toBeDefined(); + expect(group._completionTokensList).toBeDefined(); + }); + + it("$group stage collects distinct values for uniform detection", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._workerTypes).toBeDefined(); + expect(group._models).toBeDefined(); + expect(group._agentVersions).toBeDefined(); + expect(group._platforms).toBeDefined(); + expect(group._statuses).toBeDefined(); + expect(group._submissionIds).toBeDefined(); + expect(group._tasks).toBeDefined(); + expect(group._mcpKeys).toBeDefined(); + expect(group._skillKeys).toBeDefined(); + }); + + it("$project stage outputs key, label, aggregates, uniform", () => { + const pipeline = buildGroupingPipeline("task"); + const project = pipeline[2].$project as Record; + expect(project._id).toBe(0); + expect(project.key).toBeDefined(); + expect(project.label).toBeDefined(); + expect(project.aggregates).toBeDefined(); + expect(project.uniform).toBeDefined(); + }); +}); + +describe("computeStats", () => { + it("returns null for empty array", () => { + expect(computeStats([])).toBeNull(); + }); + + it("computes stats for a single value", () => { + const result = computeStats([5]); + expect(result).toEqual({ min: 5, max: 5, mean: 5, stdDev: 0 }); + }); + + it("computes min/max/mean/stdDev for multiple values", () => { + const result = computeStats([2, 4, 4, 4, 5, 5, 7, 9]); + expect(result).not.toBeNull(); + expect(result!.min).toBe(2); + expect(result!.max).toBe(9); + expect(result!.mean).toBe(5); + expect(result!.stdDev).toBeCloseTo(2, 0); + }); + + it("handles identical values", () => { + const result = computeStats([3, 3, 3]); + expect(result).toEqual({ min: 3, max: 3, mean: 3, stdDev: 0 }); + }); +}); From 410f74bf28a03c3df6b69820d0a6c0fb3088aee3 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 22:07:20 -0700 Subject: [PATCH 08/28] fix: remove unused submissionId/taskPromptId props from GroupRows --- apps/portal/src/pages/RunsList.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 2ee23dd85..e247591d7 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -900,8 +900,6 @@ export function RunsList() { groupBy={groupBy} workerFilter={workerFilter === "all" ? undefined : workerFilter} criteriaState={criteriaState} - submissionId={submissionId} - taskPromptId={taskPromptId} /> ); }) @@ -1108,8 +1106,6 @@ function GroupRows({ groupBy, workerFilter, criteriaState, - submissionId, - taskPromptId, }: { group: RunGroup; isExpanded: boolean; @@ -1121,8 +1117,6 @@ function GroupRows({ groupBy: GroupByKey; workerFilter?: string; criteriaState?: string; - submissionId?: string; - taskPromptId?: string; }) { const { aggregates, uniform } = group; const fmtDur = (v: number) => formatDuration(Math.round(v)); From 27f7cf0a452f8dc762c5760936065febfb421b33 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 10 Apr 2026 23:28:10 -0700 Subject: [PATCH 09/28] fix: disable flat runs query when groupBy is active --- apps/portal/src/pages/RunsList.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index e247591d7..5a64c8a72 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -51,6 +51,7 @@ export function RunsList() { criteria: criteriaState, submissionId, }), + enabled: groupBy === "none", refetchInterval: 10_000, }); From 7f942140f678826250721b8f667a056e8e971055 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 08:59:41 -0700 Subject: [PATCH 10/28] fix: restore status/outcome progress bars in group rows --- apps/api/src/grouping.test.ts | 19 +++++++++ apps/api/src/grouping.ts | 9 +++++ apps/portal/src/pages/RunsList.tsx | 55 ++++++++++++++++++++++++-- apps/portal/src/types.ts | 2 + packages/shared/src/schemas/request.ts | 2 + packages/shared/src/types/types.ts | 2 + 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index 96a012316..ce2aacb3b 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -47,6 +47,17 @@ describe("buildGroupingPipeline", () => { expect(group._completionTokensList).toBeDefined(); }); + it("$group stage accumulates status and outcome counts", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._statusPending).toBeDefined(); + expect(group._statusProcessing).toBeDefined(); + expect(group._statusDone).toBeDefined(); + expect(group._outcomeSucceeded).toBeDefined(); + expect(group._outcomeFailed).toBeDefined(); + expect(group._outcomeFinished).toBeDefined(); + }); + it("$group stage collects distinct values for uniform detection", () => { const pipeline = buildGroupingPipeline("task"); const group = pipeline[1].$group as Record; @@ -70,6 +81,14 @@ describe("buildGroupingPipeline", () => { expect(project.aggregates).toBeDefined(); expect(project.uniform).toBeDefined(); }); + + it("$project aggregates include statusCounts and outcomeCounts", () => { + const pipeline = buildGroupingPipeline("task"); + const project = pipeline[2].$project as Record; + const aggregates = project.aggregates as Record; + expect(aggregates.statusCounts).toEqual({ pending: "$_statusPending", processing: "$_statusProcessing", done: "$_statusDone" }); + expect(aggregates.outcomeCounts).toEqual({ succeeded: "$_outcomeSucceeded", failed: "$_outcomeFailed", finished: "$_outcomeFinished" }); + }); }); describe("computeStats", () => { diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index a526e121d..f3d614fc1 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -115,6 +115,13 @@ export function buildGroupingPipeline( _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, _mcpKeys: { $addToSet: "$_mcpKey" }, _skillKeys: { $addToSet: "$_skillKey" }, + // Status/outcome distribution counts + _statusPending: { $sum: { $cond: [{ $eq: ["$status", "pending"] }, 1, 0] } }, + _statusProcessing: { $sum: { $cond: [{ $eq: ["$status", "processing"] }, 1, 0] } }, + _statusDone: { $sum: { $cond: [{ $eq: ["$status", "done"] }, 1, 0] } }, + _outcomeSucceeded: { $sum: { $cond: [{ $eq: ["$outcome", "succeeded"] }, 1, 0] } }, + _outcomeFailed: { $sum: { $cond: [{ $eq: ["$outcome", "failed"] }, 1, 0] } }, + _outcomeFinished: { $sum: { $cond: [{ $eq: ["$outcome", "finished"] }, 1, 0] } }, // Keep first run's array values for uniform fields _firstMcpServers: { $first: "$mcpServers" }, _firstSkillRevisions: { $first: "$skillRevisions" }, @@ -138,6 +145,8 @@ export function buildGroupingPipeline( duration: { $let: { vars: { vals: { $filter: { input: "$_durations", cond: { $and: [{ $ne: ["$$this", null] }, { $gt: ["$$this", 0] }] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, promptTokens: { $let: { vars: { vals: { $filter: { input: "$_promptTokensList", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, completionTokens: { $let: { vars: { vals: { $filter: { input: "$_completionTokensList", cond: { $ne: ["$$this", null] } } } }, in: { $cond: { if: { $eq: [{ $size: "$$vals" }, 0] }, then: null, else: { min: { $min: "$$vals" }, max: { $max: "$$vals" }, mean: { $avg: "$$vals" }, stdDev: { $cond: { if: { $lte: [{ $size: "$$vals" }, 1] }, then: 0, else: { $sqrt: { $avg: { $map: { input: "$$vals", in: { $pow: [{ $subtract: ["$$this", { $avg: "$$vals" }] }, 2] } } } } } } } } } } } }, + statusCounts: { pending: "$_statusPending", processing: "$_statusProcessing", done: "$_statusDone" }, + outcomeCounts: { succeeded: "$_outcomeSucceeded", failed: "$_outcomeFailed", finished: "$_outcomeFinished" }, }, uniform: { workerType: { $cond: { if: { $eq: [{ $size: "$_workerTypes" }, 1] }, then: { $arrayElemAt: ["$_workerTypes", 0] }, else: "$$REMOVE" } }, diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 5a64c8a72..436630e51 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -1242,13 +1242,60 @@ function GroupRows({ {/* Status */} - {uniform.status ? ( - - ) : mixed} + {(() => { + const statusColors: Record = { + pending: "bg-gray-500", + processing: "bg-blue-500", + done: "bg-green-500", + }; + const total = aggregates.count; + const done = aggregates.statusCounts?.done ?? 0; + const segments = Object.entries(aggregates.statusCounts ?? {}).filter(([, c]) => c > 0); + return ( +
+ {done}/{total} done +
+ {segments.map(([status, count]) => ( +
+ ))} +
+
+ ); + })()} {/* Outcome */} - – + {(() => { + const outcomeColors: Record = { + succeeded: "bg-green-500", + failed: "bg-red-500", + finished: "bg-yellow-500", + }; + const doneCount = aggregates.statusCounts?.done ?? 0; + const segments = Object.entries(aggregates.outcomeCounts ?? {}).filter(([, c]) => c > 0); + if (doneCount === 0 || segments.length === 0) return –; + const succeeded = aggregates.outcomeCounts?.succeeded ?? 0; + return ( +
+ {succeeded}/{doneCount} pass +
+ {segments.map(([outcome, count]) => ( +
+ ))} +
+
+ ); + })()} {/* Report */} diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index fd987e431..2e339b39b 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -828,6 +828,8 @@ export interface GroupAggregates { duration: AggregateStats | null; promptTokens: AggregateStats | null; completionTokens: AggregateStats | null; + statusCounts: Record; + outcomeCounts: Record; } export interface RunGroup { diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 35ec16c61..fcb709320 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -172,6 +172,8 @@ export const GroupAggregatesSchema = z duration: AggregateStatsSchema.nullable(), promptTokens: AggregateStatsSchema.nullable(), completionTokens: AggregateStatsSchema.nullable(), + statusCounts: z.record(z.string(), z.number()), + outcomeCounts: z.record(z.string(), z.number()), }) .openapi("GroupAggregates"); diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 82159733b..9b12b6674 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -270,6 +270,8 @@ export interface GroupAggregates { duration: AggregateStats | null; promptTokens: AggregateStats | null; completionTokens: AggregateStats | null; + statusCounts: Record; + outcomeCounts: Record; } export interface RunGroup { From 2d7bec7bf94a310fb8f5ec3b9fae58997ca60b5f Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 09:07:37 -0700 Subject: [PATCH 11/28] fix: restore group-level checkbox for selecting all runs in expanded group --- apps/portal/src/pages/RunsList.tsx | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 436630e51..1fca418bb 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -1142,6 +1142,18 @@ function GroupRows({ refetchInterval: 10_000, }); + // Group-level checkbox: select/deselect all runs in this group + const groupRunIds = useMemo(() => expandedRuns.map((r) => r._id), [expandedRuns]); + const allGroupSelected = groupRunIds.length > 0 && groupRunIds.every((id) => selectedIds.has(id)); + const someGroupSelected = groupRunIds.some((id) => selectedIds.has(id)); + const handleToggleGroupSelect = () => { + if (allGroupSelected) { + groupRunIds.forEach((id) => onToggleSelect(id)); + } else { + groupRunIds.filter((id) => !selectedIds.has(id)).forEach((id) => onToggleSelect(id)); + } + }; + return ( <> {/* Checkbox */} - + e.stopPropagation()}> + {groupRunIds.length > 0 ? ( + + ) : null} + {/* ID */}
From 8e31e086de210d8e0c6b6ca0273f4f014c4a1017 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 09:15:16 -0700 Subject: [PATCH 12/28] fix: restore report progress bar in group rows with per-group summary fetch --- apps/portal/src/pages/RunsList.tsx | 31 ++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 1fca418bb..0bac905b4 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -1154,6 +1154,19 @@ function GroupRows({ } }; + // Fetch report summaries for expanded runs + const { data: groupReportSummaries } = useQuery({ + queryKey: ["report-summaries", groupRunIds], + queryFn: () => api.bulkReportSummary(groupRunIds), + enabled: groupRunIds.length > 0, + refetchInterval: 10_000, + }); + // Merge parent-level and group-level summaries + const mergedReportSummaries = useMemo(() => { + if (!reportSummaries && !groupReportSummaries) return undefined; + return { ...reportSummaries, ...groupReportSummaries }; + }, [reportSummaries, groupReportSummaries]); + return ( <> {/* Report */} - – + {(() => { + if (expandedRuns.length === 0) return –; + let total = 0, completed = 0, pending = 0, generating = 0, failed = 0; + for (const r of expandedRuns) { + const s = mergedReportSummaries?.[r._id]; + if (!s) continue; + total += s.total; + completed += s.completed; + pending += s.pending; + generating += s.generating; + failed += s.failed; + } + if (total === 0) return –; + return ; + })()} {/* Turns */} @@ -1357,7 +1384,7 @@ function GroupRows({ run={run} selectedIds={selectedIds} onToggleSelect={onToggleSelect} - reportSummaries={reportSummaries} + reportSummaries={mergedReportSummaries} deleteMutation={deleteMutation} /> )) From 90c76125357728e8475eedb1c430aa04e4dc1821 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 09:18:08 -0700 Subject: [PATCH 13/28] feat: include runIds in RunGroup for report/checkbox without expanding Add runIds (string[]) to RunGroup type, schema, and aggregation pipeline so that report summaries, group checkbox, and bulk actions work even when a group row is collapsed. Previously these features required expanding the group to fetch individual runs. --- apps/api/src/grouping.test.ts | 9 ++++++++- apps/api/src/grouping.ts | 3 +++ apps/portal/src/pages/RunsList.tsx | 24 +++++++++++------------- apps/portal/src/types.ts | 2 ++ packages/shared/src/schemas/request.ts | 2 ++ packages/shared/src/types/types.ts | 2 ++ 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index ce2aacb3b..f29b7dac7 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -72,11 +72,18 @@ describe("buildGroupingPipeline", () => { expect(group._skillKeys).toBeDefined(); }); - it("$project stage outputs key, label, aggregates, uniform", () => { + it("$group stage collects run IDs", () => { + const pipeline = buildGroupingPipeline("task"); + const group = pipeline[1].$group as Record; + expect(group._runIds).toEqual({ $push: "$_id" }); + }); + + it("$project stage outputs key, label, runIds, aggregates, uniform", () => { const pipeline = buildGroupingPipeline("task"); const project = pipeline[2].$project as Record; expect(project._id).toBe(0); expect(project.key).toBeDefined(); + expect(project.runIds).toBe("$_runIds"); expect(project.label).toBeDefined(); expect(project.aggregates).toBeDefined(); expect(project.uniform).toBeDefined(); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index f3d614fc1..50f733382 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -98,6 +98,8 @@ export function buildGroupingPipeline( $group: { _id: "$_groupKey", count: { $sum: 1 }, + // Collect run IDs for report summaries and bulk actions + _runIds: { $push: "$_id" }, // Label: take first task name for display _firstTask: { $first: "$scenario.task" }, // Push arrays for stats computation @@ -132,6 +134,7 @@ export function buildGroupingPipeline( $project: { _id: 0, key: "$_id", + runIds: "$_runIds", label: { $cond: { if: { $eq: ["$_id", "no-submission"] }, diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 0bac905b4..33ead5557 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -1142,8 +1142,8 @@ function GroupRows({ refetchInterval: 10_000, }); - // Group-level checkbox: select/deselect all runs in this group - const groupRunIds = useMemo(() => expandedRuns.map((r) => r._id), [expandedRuns]); + // Group-level checkbox: select/deselect all runs in this group (runIds from server) + const groupRunIds = group.runIds; const allGroupSelected = groupRunIds.length > 0 && groupRunIds.every((id) => selectedIds.has(id)); const someGroupSelected = groupRunIds.some((id) => selectedIds.has(id)); const handleToggleGroupSelect = () => { @@ -1154,7 +1154,7 @@ function GroupRows({ } }; - // Fetch report summaries for expanded runs + // Fetch report summaries for all runs in this group (always available via runIds) const { data: groupReportSummaries } = useQuery({ queryKey: ["report-summaries", groupRunIds], queryFn: () => api.bulkReportSummary(groupRunIds), @@ -1175,13 +1175,11 @@ function GroupRows({ > {/* Checkbox */} e.stopPropagation()}> - {groupRunIds.length > 0 ? ( - - ) : null} + {/* ID */} @@ -1333,10 +1331,10 @@ function GroupRows({ {/* Report */} {(() => { - if (expandedRuns.length === 0) return –; + if (!mergedReportSummaries || groupRunIds.length === 0) return –; let total = 0, completed = 0, pending = 0, generating = 0, failed = 0; - for (const r of expandedRuns) { - const s = mergedReportSummaries?.[r._id]; + for (const id of groupRunIds) { + const s = mergedReportSummaries[id]; if (!s) continue; total += s.total; completed += s.completed; diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index 2e339b39b..151b33d78 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -835,6 +835,8 @@ export interface GroupAggregates { export interface RunGroup { key: string; label: string; + runIds: string[]; + runIds: string[]; aggregates: GroupAggregates; uniform: GroupUniformValues; } diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index fcb709320..6a05ae463 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -181,6 +181,8 @@ export const RunGroupSchema = z .object({ key: z.string(), label: z.string(), + runIds: z.array(z.string()), + runIds: z.array(z.string()), aggregates: GroupAggregatesSchema, uniform: GroupUniformValuesSchema, }) diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 9b12b6674..6a50ce32a 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -277,6 +277,8 @@ export interface GroupAggregates { export interface RunGroup { key: string; label: string; + runIds: string[]; + runIds: string[]; aggregates: GroupAggregates; uniform: GroupUniformValues; } From b5aa3b468a580cd92bd30c9c5f9130cecc1961a3 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 09:27:31 -0700 Subject: [PATCH 14/28] feat: sort groups by label for deterministic ordering Add $sort { label: 1 } as the 4th pipeline stage so groups are always returned in alphabetical order by task name or submission ID. --- apps/api/src/endpoints.test.ts | 4 ++-- apps/api/src/grouping.test.ts | 16 +++++++++++----- apps/api/src/grouping.ts | 2 ++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 555fef90d..67af793c3 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -433,8 +433,8 @@ describe("API Endpoints", () => { expect(mocks.collection.aggregate).toHaveBeenCalled(); const pipeline = (mocks.collection.aggregate as any).mock.calls[0][0]; expect(pipeline[0]).toHaveProperty("$match"); - // buildGroupingPipeline adds 3 stages ($addFields, $group, $project) - expect(pipeline).toHaveLength(4); // $match + 3 from buildGroupingPipeline + // buildGroupingPipeline adds 4 stages ($addFields, $group, $project, $sort) + expect(pipeline).toHaveLength(5); // $match + 4 from buildGroupingPipeline }); it("does not call aggregate when groupBy is absent", async () => { diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index f29b7dac7..986a58376 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -5,17 +5,23 @@ import { describe, it, expect } from "vitest"; import { buildGroupingPipeline, computeStats } from "./grouping.js"; describe("buildGroupingPipeline", () => { - it("returns 3 pipeline stages for task grouping", () => { + it("returns 4 pipeline stages for task grouping", () => { const pipeline = buildGroupingPipeline("task"); - expect(pipeline).toHaveLength(3); + expect(pipeline).toHaveLength(4); expect(pipeline[0]).toHaveProperty("$addFields"); expect(pipeline[1]).toHaveProperty("$group"); expect(pipeline[2]).toHaveProperty("$project"); + expect(pipeline[3]).toHaveProperty("$sort"); }); - it("returns 3 pipeline stages for submissionId grouping", () => { + it("returns 4 pipeline stages for submissionId grouping", () => { const pipeline = buildGroupingPipeline("submissionId"); - expect(pipeline).toHaveLength(3); + expect(pipeline).toHaveLength(4); + }); + + it("$sort stage orders by label ascending", () => { + const pipeline = buildGroupingPipeline("task"); + expect(pipeline[3]).toEqual({ $sort: { label: 1 } }); }); it("groups by taskPromptId for task grouping", () => { @@ -80,7 +86,7 @@ describe("buildGroupingPipeline", () => { it("$project stage outputs key, label, runIds, aggregates, uniform", () => { const pipeline = buildGroupingPipeline("task"); - const project = pipeline[2].$project as Record; + const project = (pipeline[2] as { $project: Record }).$project; expect(project._id).toBe(0); expect(project.key).toBeDefined(); expect(project.runIds).toBe("$_runIds"); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index 50f733382..c13f758a4 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -164,6 +164,8 @@ export function buildGroupingPipeline( }, }, }, + // 4. Sort by label for deterministic ordering + { $sort: { label: 1 } }, ]; } From c5797c1356aab2872fd6f7e6106eaec2106129d9 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 09:36:57 -0700 Subject: [PATCH 15/28] fix: remove duplicate runIds declarations --- packages/shared/src/schemas/request.ts | 1 - packages/shared/src/types/types.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 6a05ae463..6cd0bf083 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -182,7 +182,6 @@ export const RunGroupSchema = z key: z.string(), label: z.string(), runIds: z.array(z.string()), - runIds: z.array(z.string()), aggregates: GroupAggregatesSchema, uniform: GroupUniformValuesSchema, }) diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 6a50ce32a..344bc7e39 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -278,7 +278,6 @@ export interface RunGroup { key: string; label: string; runIds: string[]; - runIds: string[]; aggregates: GroupAggregates; uniform: GroupUniformValues; } From 488462ecc2019dd1d958efbc3f84cde8c9cb2c12 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 09:57:13 -0700 Subject: [PATCH 16/28] fix: remove duplicate runIds in portal types --- apps/portal/src/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index 151b33d78..8fb3353e5 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -836,7 +836,6 @@ export interface RunGroup { key: string; label: string; runIds: string[]; - runIds: string[]; aggregates: GroupAggregates; uniform: GroupUniformValues; } From 4b331d7bc0ff87c33632e5086f6598b745d1c372 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 10:04:53 -0700 Subject: [PATCH 17/28] fix: sort groups by key instead of label for deterministic ordering key (taskPromptId or submissionId) is unique per group, whereas label (display name) can have ties, leading to non-deterministic ordering. --- apps/api/src/grouping.test.ts | 4 ++-- apps/api/src/grouping.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index 986a58376..f09229222 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -19,9 +19,9 @@ describe("buildGroupingPipeline", () => { expect(pipeline).toHaveLength(4); }); - it("$sort stage orders by label ascending", () => { + it("$sort stage orders by key ascending", () => { const pipeline = buildGroupingPipeline("task"); - expect(pipeline[3]).toEqual({ $sort: { label: 1 } }); + expect(pipeline[3]).toEqual({ $sort: { key: 1 } }); }); it("groups by taskPromptId for task grouping", () => { diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index c13f758a4..5da52d5bf 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -164,8 +164,8 @@ export function buildGroupingPipeline( }, }, }, - // 4. Sort by label for deterministic ordering - { $sort: { label: 1 } }, + // 4. Sort by grouping key for deterministic ordering + { $sort: { key: 1 } }, ]; } From 54163014c25f36f361fb0f10503706bab3ef825e Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 10:42:06 -0700 Subject: [PATCH 18/28] feat: move status/outcome/task filtering server-side - Wire status and outcome query params into API handler filter object (both were already in ListRequestsQuerySchema but not read) - Add status and outcome params to portal API client (listRuns, listRunGroups) - Pass status, outcome, and taskPromptId from filter dropdowns to API queries - Change task dropdown to store taskPromptId values (display task names) - Merge URL taskPromptId with dropdown selection (dropdown takes precedence) - Remove client-side .filter() for status/outcome/task - Update query keys to include all filter params for proper cache invalidation - Add 5 endpoint tests for status/outcome filtering (flat and grouped) --- apps/api/src/endpoints.test.ts | 57 ++++++++++++++++++++++++++++++ apps/api/src/index.ts | 8 +++++ apps/portal/src/lib/api.ts | 10 ++++-- apps/portal/src/pages/RunsList.tsx | 53 ++++++++++++++++++--------- 4 files changed, 108 insertions(+), 20 deletions(-) diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 67af793c3..97fce3541 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -447,6 +447,63 @@ describe("API Endpoints", () => { await request(app).get("/api/v1/requests"); expect(mocks.collection.aggregate).not.toHaveBeenCalled(); }); + + it("passes status filter to find query", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests?status=done"); + expect(mocks.collection.find).toHaveBeenCalledWith( + expect.objectContaining({ status: "done" }), + ); + }); + + it("passes outcome filter to find query", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests?outcome=succeeded"); + expect(mocks.collection.find).toHaveBeenCalledWith( + expect.objectContaining({ outcome: "succeeded" }), + ); + }); + + it("passes status and outcome filters to aggregate pipeline $match", async () => { + (mocks.collection.aggregate as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }); + + await request(app).get("/api/v1/requests?groupBy=task&status=done&outcome=failed"); + const pipeline = (mocks.collection.aggregate as any).mock.calls[0][0]; + expect(pipeline[0]).toEqual( + expect.objectContaining({ + $match: expect.objectContaining({ status: "done", outcome: "failed" }), + }), + ); + }); + + it("combines status, outcome, and worker filters", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + await request(app).get("/api/v1/requests?status=processing&outcome=succeeded&worker=coder-acp-copilot"); + expect(mocks.collection.find).toHaveBeenCalledWith( + expect.objectContaining({ + status: "processing", + outcome: "succeeded", + workerType: "coder-acp-copilot", + }), + ); + }); }); describe("GET /api/v1/requests/:id", () => { diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 1aa9074fa..57cd4cddd 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1051,6 +1051,8 @@ apiRoute(app, registry, { const taskPromptIdFilter = req.query.taskPromptId as string; const criteriaFilter = req.query.criteria as string; const submissionIdFilter = req.query.submissionId as string; + const statusFilter = req.query.status as string; + const outcomeFilter = req.query.outcome as string; const includeDeleted = req.query.includeDeleted === "true"; const groupByParam = req.query.groupBy as "task" | "submissionId" | undefined; @@ -1061,6 +1063,12 @@ apiRoute(app, registry, { if (taskPromptIdFilter) { filter.taskPromptId = taskPromptIdFilter; } + if (statusFilter) { + filter.status = statusFilter; + } + if (outcomeFilter) { + filter.outcome = outcomeFilter; + } if (submissionIdFilter) { // Prefix-based matching: allow filtering by partial submission ID filter.submissionId = { $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}` }; diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index fdbe71e8f..1ec05f253 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -19,11 +19,13 @@ async function request(path: string, init?: RequestInit): Promise { } export const api = { - /** List all runs, optionally filtered by worker, task prompt, and/or MDP criteria state */ - listRuns: (opts?: { worker?: string; taskPromptId?: string; criteria?: string; submissionId?: string }): Promise => { + /** List all runs, optionally filtered by worker, task prompt, status, outcome, and/or MDP criteria state */ + listRuns: (opts?: { worker?: string; taskPromptId?: string; status?: string; outcome?: string; criteria?: string; submissionId?: string }): Promise => { const params = new URLSearchParams(); if (opts?.worker) params.set("worker", opts.worker); if (opts?.taskPromptId) params.set("taskPromptId", opts.taskPromptId); + if (opts?.status) params.set("status", opts.status); + if (opts?.outcome) params.set("outcome", opts.outcome); if (opts?.criteria) params.set("criteria", opts.criteria); if (opts?.submissionId) params.set("submissionId", opts.submissionId); const qs = params.toString(); @@ -31,11 +33,13 @@ export const api = { }, /** List runs grouped by task or submissionId, with server-computed aggregates */ - listRunGroups: (opts: { groupBy: "task" | "submissionId"; worker?: string; taskPromptId?: string; criteria?: string; submissionId?: string }): Promise => { + listRunGroups: (opts: { groupBy: "task" | "submissionId"; worker?: string; taskPromptId?: string; status?: string; outcome?: string; criteria?: string; submissionId?: string }): Promise => { const params = new URLSearchParams(); params.set("groupBy", opts.groupBy); if (opts.worker) params.set("worker", opts.worker); if (opts.taskPromptId) params.set("taskPromptId", opts.taskPromptId); + if (opts.status) params.set("status", opts.status); + if (opts.outcome) params.set("outcome", opts.outcome); if (opts.criteria) params.set("criteria", opts.criteria); if (opts.submissionId) params.set("submissionId", opts.submissionId); return request(`/requests?${params.toString()}`); diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 33ead5557..d6d183dcb 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -43,11 +43,18 @@ export function RunsList() { const [resubmitOverrides, setResubmitOverrides] = useState({}); const queryClient = useQueryClient(); + // Merge URL taskPromptId with dropdown taskFilter (dropdown takes precedence) + const effectiveTaskPromptId = taskFilter !== "all" ? taskFilter : taskPromptId; + const effectiveStatus = statusFilter !== "all" ? statusFilter : undefined; + const effectiveOutcome = outcomeFilter !== "all" ? outcomeFilter : undefined; + const { data: runs = [], isLoading, isRefetching } = useQuery({ - queryKey: ["runs", workerFilter, taskPromptId, criteriaState, submissionId], + queryKey: ["runs", workerFilter, effectiveTaskPromptId, statusFilter, outcomeFilter, criteriaState, submissionId], queryFn: () => api.listRuns({ worker: workerFilter === "all" ? undefined : workerFilter, - taskPromptId, + taskPromptId: effectiveTaskPromptId, + status: effectiveStatus, + outcome: effectiveOutcome, criteria: criteriaState, submissionId, }), @@ -57,11 +64,13 @@ export function RunsList() { // Fetch server-side groups when groupBy is active const { data: serverGroups = [], isLoading: isGroupsLoading, isRefetching: isGroupsRefetching } = useQuery({ - queryKey: ["run-groups", groupBy, workerFilter, taskPromptId, criteriaState, submissionId], + queryKey: ["run-groups", groupBy, workerFilter, effectiveTaskPromptId, statusFilter, outcomeFilter, criteriaState, submissionId], queryFn: () => api.listRunGroups({ groupBy: groupBy as "task" | "submissionId", worker: workerFilter === "all" ? undefined : workerFilter, - taskPromptId, + taskPromptId: effectiveTaskPromptId, + status: effectiveStatus, + outcome: effectiveOutcome, criteria: criteriaState, submissionId, }), @@ -91,10 +100,24 @@ export function RunsList() { refetchInterval: 10_000, }); - const uniqueTasks = useMemo( - () => [...new Set(runs.map((r) => r.scenario?.task).filter(Boolean) as string[])].sort(), - [runs], - ); + // Derive unique task options (name + taskPromptId) from runs or server groups + const taskOptions = useMemo(() => { + const seen = new Map(); // taskPromptId → task name + if (groupBy === "none") { + for (const r of runs) { + if (r.taskPromptId && r.scenario?.task && !seen.has(r.taskPromptId)) { + seen.set(r.taskPromptId, r.scenario.task); + } + } + } else if (groupBy === "task") { + for (const g of serverGroups) { + if (g.key && g.label && !seen.has(g.key)) { + seen.set(g.key, g.label); + } + } + } + return [...seen.entries()].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)); + }, [runs, serverGroups, groupBy]); // Compute summary of selected runs' values for the resubmit dialog const selectedRunsSummary = useMemo(() => { @@ -189,12 +212,8 @@ export function RunsList() { }, }); - const filteredRuns = runs.filter((r) => { - if (statusFilter !== "all" && r.status !== statusFilter) return false; - if (outcomeFilter !== "all" && r.outcome !== outcomeFilter) return false; - if (taskFilter !== "all" && r.scenario?.task !== taskFilter) return false; - return true; - }); + // Filtering is now server-side via status, outcome, and taskPromptId query params + const filteredRuns = runs; const runGroups = serverGroups; @@ -297,9 +316,9 @@ export function RunsList() { All tasks - {uniqueTasks.map((t) => ( - - {truncate(t, 50)} + {taskOptions.map((t) => ( + + {truncate(t.name, 50)} ))} From 5cf0ee09fc0df65a830948637606dcf066c883de Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 10:50:26 -0700 Subject: [PATCH 19/28] fix: pass status/outcome filters to expanded group runs query When expanding a group, the individual runs query was not passing status and outcome filters, causing unfiltered runs to appear inside filtered groups. --- apps/portal/src/pages/RunsList.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index d6d183dcb..e2a542acb 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -919,6 +919,8 @@ export function RunsList() { deleteMutation={deleteMutation} groupBy={groupBy} workerFilter={workerFilter === "all" ? undefined : workerFilter} + statusFilter={effectiveStatus} + outcomeFilter={effectiveOutcome} criteriaState={criteriaState} /> ); @@ -1125,6 +1127,8 @@ function GroupRows({ deleteMutation, groupBy, workerFilter, + statusFilter, + outcomeFilter, criteriaState, }: { group: RunGroup; @@ -1136,6 +1140,8 @@ function GroupRows({ deleteMutation: { mutate: (id: string) => void; isPending: boolean }; groupBy: GroupByKey; workerFilter?: string; + statusFilter?: string; + outcomeFilter?: string; criteriaState?: string; }) { const { aggregates, uniform } = group; @@ -1144,7 +1150,7 @@ function GroupRows({ // Fetch runs for this group on expand const expandFilter = useMemo(() => { - const opts: Record = { worker: workerFilter, criteria: criteriaState }; + const opts: Record = { worker: workerFilter, status: statusFilter, outcome: outcomeFilter, criteria: criteriaState }; if (groupBy === "task") { // group.key is taskPromptId (or scenario.task fallback) opts.taskPromptId = group.key; @@ -1152,10 +1158,10 @@ function GroupRows({ opts.submissionId = group.key === "no-submission" ? undefined : group.key; } return opts; - }, [group.key, groupBy, workerFilter, criteriaState]); + }, [group.key, groupBy, workerFilter, statusFilter, outcomeFilter, criteriaState]); const { data: expandedRuns = [], isLoading: isExpandLoading } = useQuery({ - queryKey: ["group-runs", group.key, groupBy, workerFilter, criteriaState], + queryKey: ["group-runs", group.key, groupBy, workerFilter, statusFilter, outcomeFilter, criteriaState], queryFn: () => api.listRuns(expandFilter), enabled: isExpanded, refetchInterval: 10_000, From 2e82f100d060093664adc9e977f4440f8853e60f Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 12:59:53 -0700 Subject: [PATCH 20/28] refactor: remove dead client-side grouping code from portal groupRuns(), computeStats(), and related helpers are no longer used since grouping moved server-side. Keep only formatStatRange() which is still used by RunsList for displaying aggregate stats. --- apps/portal/src/lib/grouping.test.ts | 271 +-------------------------- apps/portal/src/lib/grouping.ts | 179 +----------------- 2 files changed, 2 insertions(+), 448 deletions(-) diff --git a/apps/portal/src/lib/grouping.test.ts b/apps/portal/src/lib/grouping.test.ts index 2e77bc966..edf044aab 100644 --- a/apps/portal/src/lib/grouping.test.ts +++ b/apps/portal/src/lib/grouping.test.ts @@ -2,164 +2,7 @@ // Licensed under the MIT License. import { describe, it, expect } from "vitest"; -import { groupRuns, formatStatRange } from "./grouping"; -import type { Run } from "../types"; - -// Re-export internals for testing via a test-only helper -// Since computeStats is private, we test it indirectly through groupRuns - -function makeRun(overrides: Partial = {}): Run { - return { - _id: Math.random().toString(36).slice(2), - id: "", - workerType: "coder-acp-copilot", - status: "done", - createdAt: new Date().toISOString(), - scenario: { task: "Default task", criteria: [] }, - ...overrides, - }; -} - -describe("groupRuns", () => { - it("returns empty array when groupBy is 'none'", () => { - const runs = [makeRun(), makeRun()]; - expect(groupRuns(runs, "none")).toEqual([]); - }); - - it("groups by task using taskPromptId", () => { - const runs = [ - makeRun({ taskPromptId: "t1", scenario: { task: "Task A", criteria: [] } }), - makeRun({ taskPromptId: "t1", scenario: { task: "Task A", criteria: [] } }), - makeRun({ taskPromptId: "t2", scenario: { task: "Task B", criteria: [] } }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups).toHaveLength(2); - expect(groups[0].key).toBe("t1"); - expect(groups[0].label).toBe("Task A"); - expect(groups[0].runs).toHaveLength(2); - expect(groups[1].key).toBe("t2"); - expect(groups[1].runs).toHaveLength(1); - }); - - it("falls back to scenario.task when taskPromptId is missing", () => { - const runs = [ - makeRun({ scenario: { task: "Same task", criteria: [] } }), - makeRun({ scenario: { task: "Same task", criteria: [] } }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups).toHaveLength(1); - expect(groups[0].key).toBe("Same task"); - expect(groups[0].runs).toHaveLength(2); - }); - - it("groups by submissionId", () => { - const runs = [ - makeRun({ submissionId: "sub-1" }), - makeRun({ submissionId: "sub-1" }), - makeRun({ submissionId: "sub-2" }), - makeRun({}), // no submissionId - ]; - const groups = groupRuns(runs, "submissionId"); - expect(groups).toHaveLength(3); - - const sub1 = groups.find((g) => g.key === "sub-1"); - expect(sub1?.runs).toHaveLength(2); - expect(sub1?.label).toBe("sub-1"); - - const noSub = groups.find((g) => g.key === "no-submission"); - expect(noSub?.runs).toHaveLength(1); - expect(noSub?.label).toBe("No submission ID"); - }); - - it("computes turn aggregates correctly", () => { - const runs = [ - makeRun({ turns: [{ iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }] }), - makeRun({ turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - { iteration: 2, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - { iteration: 3, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - ] }), - makeRun({ turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - { iteration: 2, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "" }, - ] }), - ]; - // All same taskPromptId so they group together - runs.forEach((r) => (r.taskPromptId = "t1")); - const groups = groupRuns(runs, "task"); - expect(groups).toHaveLength(1); - const agg = groups[0].aggregates; - expect(agg.count).toBe(3); - expect(agg.turns).not.toBeNull(); - expect(agg.turns!.min).toBe(1); - expect(agg.turns!.max).toBe(3); - expect(agg.turns!.mean).toBe(2); // (1+3+2)/3 - }); - - it("computes duration aggregates from turn durationMs", () => { - const runs = [ - makeRun({ - taskPromptId: "t1", - turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "", durationMs: 10000 }, - { iteration: 2, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "", durationMs: 20000 }, - ], - }), - makeRun({ - taskPromptId: "t1", - turns: [ - { iteration: 1, codingAgentResponse: "", judgeFeedback: "", snapshotUrl: "", passed: true, timestamp: "", durationMs: 5000 }, - ], - }), - ]; - const groups = groupRuns(runs, "task"); - const agg = groups[0].aggregates; - expect(agg.duration).not.toBeNull(); - expect(agg.duration!.min).toBe(5000); // 5s - expect(agg.duration!.max).toBe(30000); // 10+20=30s - expect(agg.duration!.mean).toBe(17500); // (30000+5000)/2 - }); - - it("computes token aggregates", () => { - const runs = [ - makeRun({ - taskPromptId: "t1", - tokenUsage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }, - }), - makeRun({ - taskPromptId: "t1", - tokenUsage: { promptTokens: 200, completionTokens: 100, totalTokens: 300 }, - }), - ]; - const groups = groupRuns(runs, "task"); - const agg = groups[0].aggregates; - expect(agg.promptTokens).not.toBeNull(); - expect(agg.promptTokens!.min).toBe(100); - expect(agg.promptTokens!.max).toBe(200); - expect(agg.promptTokens!.mean).toBe(150); - expect(agg.completionTokens!.min).toBe(50); - expect(agg.completionTokens!.max).toBe(100); - }); - - it("handles runs with no turns or tokens gracefully", () => { - const runs = [ - makeRun({ taskPromptId: "t1" }), // no turns, no tokens - makeRun({ taskPromptId: "t1" }), - ]; - const groups = groupRuns(runs, "task"); - const agg = groups[0].aggregates; - expect(agg.count).toBe(2); - expect(agg.turns).toBeNull(); - expect(agg.duration).toBeNull(); - expect(agg.promptTokens).toBeNull(); - expect(agg.completionTokens).toBeNull(); - }); - - it("handles empty runs array", () => { - expect(groupRuns([], "task")).toEqual([]); - expect(groupRuns([], "submissionId")).toEqual([]); - }); -}); +import { formatStatRange } from "./grouping"; describe("formatStatRange", () => { it("returns dash for null stats", () => { @@ -185,115 +28,3 @@ describe("formatStatRange", () => { expect(result).toContain("200ms"); }); }); - -describe("uniform values", () => { - it("detects uniform workerType", () => { - const runs = [ - makeRun({ taskPromptId: "t1", workerType: "coder-acp-copilot" }), - makeRun({ taskPromptId: "t1", workerType: "coder-acp-copilot" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.workerType).toBe("coder-acp-copilot"); - }); - - it("returns undefined for mixed workerType", () => { - const runs = [ - makeRun({ taskPromptId: "t1", workerType: "coder-acp-copilot" }), - makeRun({ taskPromptId: "t1", workerType: "coder-vscode-web" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.workerType).toBeUndefined(); - }); - - it("detects uniform model", () => { - const runs = [ - makeRun({ taskPromptId: "t1", model: "gpt-4o" }), - makeRun({ taskPromptId: "t1", model: "gpt-4o" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.model).toBe("gpt-4o"); - }); - - it("omits model when all undefined", () => { - const runs = [ - makeRun({ taskPromptId: "t1" }), - makeRun({ taskPromptId: "t1" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.model).toBeUndefined(); - }); - - it("detects uniform status", () => { - const runs = [ - makeRun({ taskPromptId: "t1", status: "done" }), - makeRun({ taskPromptId: "t1", status: "done" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.status).toBe("done"); - }); - - it("returns undefined for mixed status", () => { - const runs = [ - makeRun({ taskPromptId: "t1", status: "done" }), - makeRun({ taskPromptId: "t1", status: "pending" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.status).toBeUndefined(); - }); - - it("detects uniform mcpServers (order-independent)", () => { - const runs = [ - makeRun({ taskPromptId: "t1", mcpServers: ["b", "a"] }), - makeRun({ taskPromptId: "t1", mcpServers: ["a", "b"] }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.mcpServers).toEqual(["b", "a"]); - }); - - it("returns undefined for mixed mcpServers", () => { - const runs = [ - makeRun({ taskPromptId: "t1", mcpServers: ["a"] }), - makeRun({ taskPromptId: "t1", mcpServers: ["b"] }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.mcpServers).toBeUndefined(); - }); - - it("detects uniform skillRevisions", () => { - const runs = [ - makeRun({ taskPromptId: "t1", skillRevisions: ["org/repo/skill@abc"] }), - makeRun({ taskPromptId: "t1", skillRevisions: ["org/repo/skill@abc"] }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.skillRevisions).toEqual(["org/repo/skill@abc"]); - }); - - it("detects uniform agentVersion", () => { - const runs = [ - makeRun({ taskPromptId: "t1", agentVersion: "copilot-0.0.415" }), - makeRun({ taskPromptId: "t1", agentVersion: "copilot-0.0.415" }), - ]; - const groups = groupRuns(runs, "task"); - expect(groups[0].uniform.agentVersion).toBe("copilot-0.0.415"); - }); - - it("single-run group always shows uniform values", () => { - const runs = [ - makeRun({ - taskPromptId: "t1", - workerType: "coder-vscode-web", - model: "gpt-4o", - status: "done", - mcpServers: ["fs"], - agentVersion: "v1", - }), - ]; - const groups = groupRuns(runs, "task"); - const u = groups[0].uniform; - expect(u.workerType).toBe("coder-vscode-web"); - expect(u.model).toBe("gpt-4o"); - expect(u.status).toBe("done"); - expect(u.mcpServers).toEqual(["fs"]); - expect(u.agentVersion).toBe("v1"); - }); -}); diff --git a/apps/portal/src/lib/grouping.ts b/apps/portal/src/lib/grouping.ts index 052acd01d..864507818 100644 --- a/apps/portal/src/lib/grouping.ts +++ b/apps/portal/src/lib/grouping.ts @@ -1,184 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, RunStatus } from "../types"; - -export type GroupByKey = "none" | "task" | "submissionId"; - -export interface AggregateStats { - min: number; - max: number; - mean: number; - stdDev: number; -} - -export interface GroupUniformValues { - workerType?: string; - model?: string; - agentVersion?: string; - platform?: string; - mcpServers?: string[]; - skillRevisions?: string[]; - status?: RunStatus; - submissionId?: string; - task?: string; -} - -export interface GroupAggregates { - count: number; - turns: AggregateStats | null; - duration: AggregateStats | null; - promptTokens: AggregateStats | null; - completionTokens: AggregateStats | null; -} - -export interface RunGroup { - key: string; - label: string; - runs: Run[]; - aggregates: GroupAggregates; - uniform: GroupUniformValues; -} - -function computeStats(values: number[]): AggregateStats | null { - if (values.length === 0) return null; - const min = Math.min(...values); - const max = Math.max(...values); - const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); - return { min, max, mean, stdDev }; -} - -function getRunTurns(run: Run): number | null { - return run.turns?.length ?? null; -} - -function getRunDuration(run: Run): number | null { - if (!run.turns || run.turns.length === 0) return null; - const total = run.turns.reduce((sum, t) => sum + (t.durationMs ?? 0), 0); - return total > 0 ? total : null; -} - -function getRunTokens(run: Run): { prompt: number; completion: number } | null { - const usage = - run.tokenUsage ?? - (run.turns?.some((t) => t.tokenUsage) - ? run.turns!.reduce( - (acc, t) => { - if (!t.tokenUsage) return acc; - return { - promptTokens: acc.promptTokens + t.tokenUsage.promptTokens, - completionTokens: acc.completionTokens + t.tokenUsage.completionTokens, - totalTokens: acc.totalTokens + t.tokenUsage.totalTokens, - }; - }, - { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, - ) - : undefined); - if (!usage) return null; - return { prompt: usage.promptTokens, completion: usage.completionTokens }; -} - -function computeAggregates(runs: Run[]): GroupAggregates { - const turnValues = runs.map(getRunTurns).filter((v): v is number => v !== null); - const durationValues = runs.map(getRunDuration).filter((v): v is number => v !== null); - const tokenData = runs.map(getRunTokens).filter((v): v is { prompt: number; completion: number } => v !== null); - - return { - count: runs.length, - turns: computeStats(turnValues), - duration: computeStats(durationValues), - promptTokens: computeStats(tokenData.map((t) => t.prompt)), - completionTokens: computeStats(tokenData.map((t) => t.completion)), - }; -} - -function arrKey(arr: string[] | undefined): string { - return (arr ?? []).slice().sort().join(","); -} - -function uniform(runs: Run[], getter: (r: Run) => T): T | undefined { - if (runs.length === 0) return undefined; - const first = getter(runs[0]); - return runs.every((r) => getter(r) === first) ? first : undefined; -} - -function computeUniformValues(runs: Run[]): GroupUniformValues { - if (runs.length === 0) return {}; - - const result: GroupUniformValues = {}; - - const wt = uniform(runs, (r) => r.workerType); - if (wt !== undefined) result.workerType = wt; - - const model = uniform(runs, (r) => r.model ?? ""); - if (model !== undefined && model !== "") result.model = model; - - const ver = uniform(runs, (r) => r.agentVersion ?? ""); - if (ver !== undefined && ver !== "") result.agentVersion = ver; - - const plat = uniform(runs, (r) => r.os?.platform ?? ""); - if (plat !== undefined && plat !== "") result.platform = plat; - - const status = uniform(runs, (r) => r.status); - if (status !== undefined) result.status = status; - - const sub = uniform(runs, (r) => r.submissionId ?? ""); - if (sub !== undefined && sub !== "") result.submissionId = sub; - - const task = uniform(runs, (r) => r.scenario?.task ?? ""); - if (task !== undefined && task !== "") result.task = task; - - // Array fields: compare by sorted join - const mcpKey = uniform(runs, (r) => arrKey(r.mcpServers)); - if (mcpKey !== undefined && mcpKey !== "") result.mcpServers = runs[0].mcpServers; - - const skillKey = uniform(runs, (r) => arrKey(r.skillRevisions)); - if (skillKey !== undefined && skillKey !== "") result.skillRevisions = runs[0].skillRevisions; - - return result; -} - -export function groupRuns(runs: Run[], groupBy: GroupByKey): RunGroup[] { - if (groupBy === "none") return []; - - const groups = new Map(); - - for (const run of runs) { - let key: string; - if (groupBy === "task") { - key = run.taskPromptId ?? run.scenario?.task ?? "unknown"; - } else { - key = run.submissionId ?? "no-submission"; - } - const list = groups.get(key); - if (list) { - list.push(run); - } else { - groups.set(key, [run]); - } - } - - const result: RunGroup[] = []; - for (const [key, groupRuns] of groups) { - let label: string; - if (groupBy === "task") { - label = groupRuns[0]?.scenario?.task ?? key; - } else { - label = key === "no-submission" ? "No submission ID" : key; - } - result.push({ - key, - label, - runs: groupRuns, - aggregates: computeAggregates(groupRuns), - uniform: computeUniformValues(groupRuns), - }); - } - - return result; -} +import type { AggregateStats } from "shared"; export function formatStatRange( stat: AggregateStats | null, From 65c2e85fb6c4e0f88893781411ac7217c79c76e0 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 13:20:10 -0700 Subject: [PATCH 21/28] refactor: remove unused computeStats from server-side grouping The MongoDB aggregation pipeline computes stats directly via $min/$max/$avg operators. computeStats was only used in its own tests and never called from application code. --- apps/api/src/grouping.test.ts | 27 +-------------------------- apps/api/src/grouping.ts | 16 ---------------- 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index f09229222..088070a4c 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { describe, it, expect } from "vitest"; -import { buildGroupingPipeline, computeStats } from "./grouping.js"; +import { buildGroupingPipeline } from "./grouping.js"; describe("buildGroupingPipeline", () => { it("returns 4 pipeline stages for task grouping", () => { @@ -103,28 +103,3 @@ describe("buildGroupingPipeline", () => { expect(aggregates.outcomeCounts).toEqual({ succeeded: "$_outcomeSucceeded", failed: "$_outcomeFailed", finished: "$_outcomeFinished" }); }); }); - -describe("computeStats", () => { - it("returns null for empty array", () => { - expect(computeStats([])).toBeNull(); - }); - - it("computes stats for a single value", () => { - const result = computeStats([5]); - expect(result).toEqual({ min: 5, max: 5, mean: 5, stdDev: 0 }); - }); - - it("computes min/max/mean/stdDev for multiple values", () => { - const result = computeStats([2, 4, 4, 4, 5, 5, 7, 9]); - expect(result).not.toBeNull(); - expect(result!.min).toBe(2); - expect(result!.max).toBe(9); - expect(result!.mean).toBe(5); - expect(result!.stdDev).toBeCloseTo(2, 0); - }); - - it("handles identical values", () => { - const result = computeStats([3, 3, 3]); - expect(result).toEqual({ min: 3, max: 3, mean: 3, stdDev: 0 }); - }); -}); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index 5da52d5bf..1a001aaf4 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -168,19 +168,3 @@ export function buildGroupingPipeline( { $sort: { key: 1 } }, ]; } - -/** - * Compute aggregate stats from an array of numbers. - * Used for application-level grouping as a fallback when the MongoDB - * aggregation pipeline cannot be used. - */ -export function computeStats(values: number[]): AggregateStats | null { - if (values.length === 0) return null; - const min = Math.min(...values); - const max = Math.max(...values); - const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = - values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); - return { min, max, mean, stdDev }; -} From 27a267893e135dfc201632449ebb7b374977ccd6 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 16:49:39 -0700 Subject: [PATCH 22/28] chore: remove plan.md from version control --- plan.md | 96 --------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 plan.md diff --git a/plan.md b/plan.md deleted file mode 100644 index bade4480c..000000000 --- a/plan.md +++ /dev/null @@ -1,96 +0,0 @@ -# Plan: Move Runs Grouping Server Side - -**Issue:** https://github.com/growth-ecosystems/scope-core/issues/568 - -## Problem - -Runs grouping is currently implemented client-side in `apps/portal/src/lib/grouping.ts`. The portal fetches **all** runs from `GET /api/v1/requests`, then groups and aggregates them in the browser. This blocks server-side pagination because: - -1. Pagination returns a page of runs, but grouping needs visibility across ALL runs to compute correct groups and aggregates. -2. A paginated response can split a group across pages, producing wrong counts, stats, and uniform-value detection. - -## Solution - -Add a `groupBy` query parameter to the existing `GET /api/v1/requests` endpoint. When absent, the endpoint behaves exactly as today (flat array of runs). When set to `task` or `submissionId`, it runs a MongoDB aggregation pipeline server-side and returns pre-computed group summaries — including all aggregation (turns, duration, tokens stats) and uniform-value detection — so the portal renders them directly with zero client-side computation. - -## Steps - -### Step 1 — Move shared types to `packages/shared` - -Move `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, and `RunGroup` interfaces from `apps/portal/src/lib/grouping.ts` to `packages/shared/src/types/` so both API and portal can import them. - -### Step 2 — Add `groupBy` param to `GET /api/v1/requests` - -Add optional `groupBy` query param (`"task"` | `"submissionId"`) to the existing list endpoint. - -When `groupBy` is provided, run a MongoDB aggregation pipeline: -1. `$match` — same filters as today (worker, taskPromptId, criteria, submissionId, deletedAt) -2. `$addFields` — compute per-run derived values: turn count, total duration (sum of `turn.durationMs`), prompt/completion tokens -3. `$group` — group by `taskPromptId` (or `submissionId`), accumulate min/max/avg/count for turns, duration, tokens; collect first values and distinct-count for uniform-value detection; grab first `scenario.task` for label -4. `$project` — reshape into `RunGroup` response format with `AggregateStats` and `GroupUniformValues` - -Response shape when grouped: -```ts -{ - key: string; - label: string; - count: number; - aggregates: GroupAggregates; // turns, duration, promptTokens, completionTokens stats - uniform: GroupUniformValues; // fields identical across all runs in group -}[] -``` - -When `groupBy` is absent, behavior is unchanged (flat `RequestDocument[]`). - -### Step 3 — Server-side aggregation logic - -Implement the aggregation computation (stats + uniform values) in a dedicated module `apps/api/src/grouping.ts` that builds the MongoDB pipeline stages. This keeps `index.ts` clean and the logic unit-testable. - -The aggregation computes **all** the same metrics the client currently does: -- **Counts**: number of runs per group -- **Stats** (min/max/mean/stdDev): turn count, total duration, prompt tokens, completion tokens -- **Uniform values**: workerType, model, agentVersion, platform, mcpServers, skillRevisions, status, submissionId, task — included only when all runs in the group share the same value - -### Step 4 — Portal API client - -Update `api.listRuns()` in `apps/portal/src/lib/api.ts` to accept an optional `groupBy` param. Add a separate `api.listRunGroups(opts)` method (or overload) that returns `RunGroup[]` when grouping is requested. - -### Step 5 — Update `RunsList.tsx` - -When `groupBy !== "none"`: -1. Call `api.listRunGroups(...)` → get `RunGroup[]` from server with pre-computed aggregates. -2. Render group rows directly from server response — no client-side `groupRuns()` call. -3. On group expand, fetch that group's runs via `api.listRuns({ taskPromptId })` or `api.listRuns({ submissionId })`. - -When `groupBy === "none"`: -1. Fetch runs via `api.listRuns()` as today. -2. Render flat rows, no grouping. - -### Step 6 — Deprecate client-side grouping - -Keep `grouping.ts` and its tests intact (no breaking change), but the portal no longer calls `groupRuns()` in the main list view. Can be removed in a follow-up. - -### Step 7 — Tests - -- **API unit tests** (`apps/api/src/grouping.test.ts`): test the aggregation pipeline builder with different `groupBy` values and filters. -- **API endpoint tests** (`endpoints.test.ts`): test `GET /api/v1/requests?groupBy=task` returns grouped response. -- **Portal tests**: update any `RunsList` tests to use server-side groups. - -## File Changes Summary - -| File | Change | -|------|--------| -| `packages/shared/src/types/types.ts` | Add `GroupByKey`, `AggregateStats`, `GroupAggregates`, `GroupUniformValues`, `RunGroup` types | -| `apps/api/src/grouping.ts` | New — builds MongoDB aggregation pipeline for grouping + stats + uniform values | -| `apps/api/src/grouping.test.ts` | New — tests for aggregation pipeline builder | -| `apps/api/src/index.ts` | Add `groupBy` query param to list endpoint; call pipeline when set | -| `apps/api/src/endpoints.test.ts` | Test grouped response from list endpoint | -| `apps/portal/src/lib/api.ts` | Add `listRunGroups()` method | -| `apps/portal/src/pages/RunsList.tsx` | Use server groups when grouped; stop calling client-side `groupRuns()` | -| `apps/portal/src/lib/grouping.ts` | Keep as-is (deprecated, not deleted) | - -## Risks & Notes - -- **CosmosDB aggregation support**: CosmosDB for MongoDB API supports `$group`, `$project`, `$match`, `$sort`, `$addFields`, `$unwind`, `$min`, `$max`, `$avg`, `$push`, `$sum`. `$stdDevPop` is supported in MongoDB 3.2+ — verify CosmosDB compatibility or compute stdDev from pushed arrays in `$project`. -- **Index coverage**: `$group` runs after `$match`, so existing indexes on `workerType`, `taskPromptId`, `submissionId`, `deletedAt`, `createdAt` remain sufficient. -- **Breaking change**: None — `groupBy` is optional; when absent the endpoint returns the same flat array as before. From 9780643cf91ea0e0fc450a8fa984490f4c061f47 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 17:44:01 -0700 Subject: [PATCH 23/28] feat: add single-field indexes for requests filter/group fields Add migration 009 with indexes on taskPromptId, status, outcome, workerType, and deletedAt. Cosmos DB for MongoDB (RU) uses index intersection for multi-field filters, so single-field indexes are the recommended approach over compound indexes. --- .../src/check-migrations.test.ts | 6 ++- .../009-add-requests-filter-indexes.ts | 44 +++++++++++++++++++ .../db-migrations/src/required-migrations.ts | 1 + 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts diff --git a/packages/db-migrations/src/check-migrations.test.ts b/packages/db-migrations/src/check-migrations.test.ts index d9179c0a0..a83b91b8a 100644 --- a/packages/db-migrations/src/check-migrations.test.ts +++ b/packages/db-migrations/src/check-migrations.test.ts @@ -39,6 +39,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]); const result = await checkMigrations(db); expect(result.ready).toBe(true); @@ -52,6 +53,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]); }); @@ -59,7 +61,7 @@ describe("checkMigrations", () => { const db = makeMockDb(["001-backfill-task-prompts.ts"]); const result = await checkMigrations(db); expect(result.ready).toBe(false); - expect(result.pending).toEqual(["002-create-indexes.ts", "003-create-skill-indexes.ts", "004-add-submission-id-index.ts", "005-backfill-iteration-durations.ts", "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts"]); + expect(result.pending).toEqual(["002-create-indexes.ts", "003-create-skill-indexes.ts", "004-add-submission-id-index.ts", "005-backfill-iteration-durations.ts", "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", "009-add-requests-filter-indexes.ts"]); expect(result.applied).toEqual(["001-backfill-task-prompts.ts"]); }); @@ -76,6 +78,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]); expect(result.applied).toEqual([]); }); @@ -90,6 +93,7 @@ describe("checkMigrations", () => { "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", "999-future-migration.ts", ]); const result = await checkMigrations(db); diff --git a/packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts b/packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts new file mode 100644 index 000000000..dd6c215d9 --- /dev/null +++ b/packages/db-migrations/src/migrations/009-add-requests-filter-indexes.ts @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Migration: Add single-field indexes for commonly filtered fields on the + * requests collection. + * + * The server-side grouping and filtering pipeline uses $match on these fields. + * Cosmos DB for MongoDB (RU) recommends single-field indexes over compound + * indexes for multi-property filters — it uses index intersection internally. + */ + +import type { Db } from "mongodb"; +import type { MigrationInterface } from "mongo-migrate-ts"; + +const INDEXES: Array<{ key: Record; name: string }> = [ + { key: { taskPromptId: 1 }, name: "taskPromptId_1" }, + { key: { status: 1 }, name: "status_1" }, + { key: { outcome: 1 }, name: "outcome_1" }, + { key: { workerType: 1 }, name: "workerType_1" }, + { key: { deletedAt: 1 }, name: "deletedAt_1" }, +]; + +export class AddRequestsFilterIndexes implements MigrationInterface { + async up(db: Db): Promise { + const col = db.collection("requests"); + for (const { key, name } of INDEXES) { + try { + await col.createIndex(key); + console.log(` Created index ${name} on requests`); + } catch (err: any) { + console.log( + ` Index ${name} already exists or couldn't be created: ${err.message ?? err}`, + ); + } + } + } + + async down(db: Db): Promise { + console.log( + " Skipping index drop — indexes should be dropped manually if needed", + ); + } +} diff --git a/packages/db-migrations/src/required-migrations.ts b/packages/db-migrations/src/required-migrations.ts index 4935a8ac4..ad31c2830 100644 --- a/packages/db-migrations/src/required-migrations.ts +++ b/packages/db-migrations/src/required-migrations.ts @@ -21,4 +21,5 @@ export const REQUIRED_MIGRATIONS: readonly string[] = [ "006-split-status-outcome.ts", "007-rename-exhausted-to-finished.ts", "008-backfill-ai-call-count.ts", + "009-add-requests-filter-indexes.ts", ]; From 6f5bb6969accec7882b6e488c030cee36e1e4e00 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 17:48:00 -0700 Subject: [PATCH 24/28] docs: add database schema & indexes doc, update migrations table New docs/architecture/database.md documents all collections, their indexes, and the indexing strategy for Cosmos DB for MongoDB (RU). Updated the migrations table in db-migrations.md which was missing entries 002-009. --- docs/architecture/database.md | 156 +++++++++++++++++++++++++++++ docs/architecture/db-migrations.md | 8 ++ 2 files changed, 164 insertions(+) create mode 100644 docs/architecture/database.md diff --git a/docs/architecture/database.md b/docs/architecture/database.md new file mode 100644 index 000000000..efff555cd --- /dev/null +++ b/docs/architecture/database.md @@ -0,0 +1,156 @@ +# Database Schema & Indexes + +MongoDB database (Cosmos DB for MongoDB RU in production, MongoDB 7.0 locally) storing all platform data. + +- **Database**: `requests-db` (configurable via `MONGO_DATABASE`) +- **Connection**: `MONGO_CONNECTION_STRING` or `MONGO_URI` + +## Collections + +### `requests` + +Benchmark runs — the core entity. Each document represents a single coding agent run against a task. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | +| `submissionId` | `{ submissionId: 1 }` | sparse | 004 | +| `taskPromptId` | `{ taskPromptId: 1 }` | | 009 | +| `status` | `{ status: 1 }` | | 009 | +| `outcome` | `{ outcome: 1 }` | | 009 | +| `workerType` | `{ workerType: 1 }` | | 009 | +| `deletedAt` | `{ deletedAt: 1 }` | | 009 | + +Key fields: `status`, `outcome`, `workerType`, `taskPromptId`, `submissionId`, `scenario.task`, `turns[]`, `tokenUsage`, `model`, `agentVersion`, `os.platform`, `mcpServers[]`, `skillRevisions[]`, `deletedAt`, `createdAt`. + +### `task-prompts` + +Canonical task definitions with prompt text. Runs link to a task prompt via `taskPromptId`. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `reports` + +Evaluation reports generated by the judge for individual runs. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | +| `requestId` | `{ requestId: 1 }` | | 002 | + +### `criteria` + +Evaluation criteria definitions used by the judge. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `id` | `{ id: 1 }` | unique | 002 | + +### `agents` + +Registered coding agent configurations. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `models` + +LLM model records associated with agents. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `agentId` | `{ agentId: 1 }` | | 002 | +| `provider` | `{ provider: 1 }` | | 002 | +| `agentId_provider` | `{ agentId: 1, provider: 1 }` | | 002 | +| `modelId` | `{ modelId: 1 }` | | 002 | + +### `skills` + +Agent skill definitions (source + name). + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 003 | +| `source_skillName` | `{ source: 1, skillName: 1 }` | | 003 | + +### `skill-revisions` + +Immutable revisions of agent skills, referenced by runs. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `ref` | `{ ref: 1 }` | unique | 003 | +| `source_skillName` | `{ source: 1, skillName: 1 }` | | 003 | +| `resolvedAt` | `{ resolvedAt: -1 }` | | 003 | + +### `prompt-features` + +Feature definitions extracted from task prompts. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `id` | `{ id: 1 }` | unique | 002 | + +### `prompt-feature-extractions` + +Cached results of prompt feature extraction (keyed by task text hash). + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `taskTextHash` | `{ taskTextHash: 1 }` | unique | 002 | + +### `insights` + +Generated analysis insights. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `mcp-servers` + +MCP (Model Context Protocol) server configurations. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `createdAt` | `{ createdAt: -1 }` | | 002 | + +### `feature-flags` + +Runtime feature flags. + +| Index | Key | Options | Migration | +|-------|-----|---------|-----------| +| `_id` | `{ _id: 1 }` | default | — | +| `key` | `{ key: 1 }` | unique | 002 | + +### Other collections + +- **`extensions`** — VS Code extension metadata +- **`report-templates`** — Report generation templates +- **`accounts`** — User/service accounts +- **`_migrations`** — Migration tracking (managed by `mongo-migrate-ts`) + +## Indexing Strategy + +Production uses **Cosmos DB for MongoDB (RU-based)**. Key considerations: + +- **No wildcard index** — unlike the default Cosmos DB policy, the `requests` collection uses explicit single-field indexes only. +- **Single-field indexes over compound** — Cosmos DB for MongoDB RU uses index intersection for multi-property `$match` filters. Compound indexes are only needed for sort optimization. +- **Sparse indexes** — used for optional fields (`submissionId`) to avoid indexing documents that lack the field. +- **All index changes go through migrations** — see [db-migrations.md](db-migrations.md) for the migration framework. diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index a34e40c50..e7ad02dc9 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -94,6 +94,14 @@ When you run a command: | Migration | Description | |-----------|-------------| | `001-backfill-task-prompts` | Creates `task-prompts` collection from existing `requests.scenario.task` values, links runs via `taskPromptId`, migrates prompt feature extraction data | +| `002-create-indexes` | Creates single-field indexes on all collections (see [database.md](database.md)) | +| `003-create-skill-indexes` | Adds indexes for `skills` and `skill-revisions` collections | +| `004-add-submission-id-index` | Sparse index on `requests.submissionId` for efficient filtering | +| `005-backfill-iteration-durations` | Estimates `startedAt`/`durationMs` on existing turns from timestamps | +| `006-split-status-outcome` | Splits run status into `status` (pending/processing/done) + `outcome` (succeeded/failed/exhausted) | +| `007-rename-exhausted-to-finished` | Renames outcome `exhausted` → `finished` | +| `008-backfill-ai-call-count` | Downloads HARs from blob storage to count AI completion calls per turn | +| `009-add-requests-filter-indexes` | Adds indexes on `taskPromptId`, `status`, `outcome`, `workerType`, `deletedAt` for server-side filtering/grouping | ## CI/CD From 98cc909d9dd9b37f2681d60c7fd7c610378846db Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 18:51:46 -0700 Subject: [PATCH 25/28] chore: rename database.md to db.md for consistency --- docs/architecture/db-migrations.md | 2 +- docs/architecture/{database.md => db.md} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/architecture/{database.md => db.md} (100%) diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index e7ad02dc9..a0b2a5b06 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -94,7 +94,7 @@ When you run a command: | Migration | Description | |-----------|-------------| | `001-backfill-task-prompts` | Creates `task-prompts` collection from existing `requests.scenario.task` values, links runs via `taskPromptId`, migrates prompt feature extraction data | -| `002-create-indexes` | Creates single-field indexes on all collections (see [database.md](database.md)) | +| `002-create-indexes` | Creates single-field indexes on all collections (see [db.md](db.md)) | | `003-create-skill-indexes` | Adds indexes for `skills` and `skill-revisions` collections | | `004-add-submission-id-index` | Sparse index on `requests.submissionId` for efficient filtering | | `005-backfill-iteration-durations` | Estimates `startedAt`/`durationMs` on existing turns from timestamps | diff --git a/docs/architecture/database.md b/docs/architecture/db.md similarity index 100% rename from docs/architecture/database.md rename to docs/architecture/db.md From c41c12b32d58ad4ef904cb6cefca7f305121ba34 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 18:52:57 -0700 Subject: [PATCH 26/28] =?UTF-8?q?docs:=20fix=20CI/CD=20section=20=E2=80=94?= =?UTF-8?q?=20migrations=20run=20automatically=20via=20K8s=20Job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/architecture/db-migrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index a0b2a5b06..4220b450a 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -107,4 +107,4 @@ When you run a command: The `db-migrations` package is compiled as part of the CI `build` job (`pnpm -r build` → `tsc`). This catches TypeScript errors in migrations before deployment. -Migrations themselves are **not** run automatically in CI — they require a live database connection and are run manually as part of the deployment process. +In Kubernetes, migrations are run automatically by a Job before the API pods start. The API readiness probe checks the `_migrations` collection against the `REQUIRED_MIGRATIONS` list — pods report "not ready" until all required migrations have been applied. From f84bedae4da9efafbc7344fa976af77e43f1ed92 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 18:53:43 -0700 Subject: [PATCH 27/28] docs: mention Docker Compose also runs migrations automatically --- docs/architecture/db-migrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index 4220b450a..9cfd9540e 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -107,4 +107,4 @@ When you run a command: The `db-migrations` package is compiled as part of the CI `build` job (`pnpm -r build` → `tsc`). This catches TypeScript errors in migrations before deployment. -In Kubernetes, migrations are run automatically by a Job before the API pods start. The API readiness probe checks the `_migrations` collection against the `REQUIRED_MIGRATIONS` list — pods report "not ready" until all required migrations have been applied. +In Kubernetes, migrations are run automatically by a Job before the API pods start. Locally, Docker Compose runs migrations automatically as well. The API readiness probe checks the `_migrations` collection against the `REQUIRED_MIGRATIONS` list — pods report "not ready" until all required migrations have been applied. From 8f72dd645f9a4057a93b43490aedcf4f2fde687c Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Sat, 11 Apr 2026 19:01:05 -0700 Subject: [PATCH 28/28] fix: add extensions to GroupUniformValues and fix portal shared import - Add extensions uniform detection to aggregation pipeline - Add extensions to GroupUniformValues type and Zod schema - Add extensions to portal mirrored types - Fix portal grouping.ts import: use ../types instead of shared --- apps/api/src/grouping.ts | 10 ++++++++++ apps/portal/src/lib/grouping.ts | 2 +- apps/portal/src/types.ts | 1 + packages/shared/src/schemas/request.ts | 1 + packages/shared/src/types/types.ts | 1 + 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index 1a001aaf4..bb3d3add8 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -91,6 +91,13 @@ export function buildGroupingPipeline( in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, }, }, + _extensionKey: { + $reduce: { + input: { $sortArray: { input: { $ifNull: ["$extensions", []] }, sortBy: 1 } }, + initialValue: "", + in: { $concat: ["$$value", { $cond: { if: { $eq: ["$$value", ""] }, then: "", else: "," } }, "$$this"] }, + }, + }, }, }, // 2. Group and accumulate per-run values @@ -117,6 +124,7 @@ export function buildGroupingPipeline( _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, _mcpKeys: { $addToSet: "$_mcpKey" }, _skillKeys: { $addToSet: "$_skillKey" }, + _extensionKeys: { $addToSet: "$_extensionKey" }, // Status/outcome distribution counts _statusPending: { $sum: { $cond: [{ $eq: ["$status", "pending"] }, 1, 0] } }, _statusProcessing: { $sum: { $cond: [{ $eq: ["$status", "processing"] }, 1, 0] } }, @@ -127,6 +135,7 @@ export function buildGroupingPipeline( // Keep first run's array values for uniform fields _firstMcpServers: { $first: "$mcpServers" }, _firstSkillRevisions: { $first: "$skillRevisions" }, + _firstExtensions: { $first: "$extensions" }, }, }, // 3. Project into final shape @@ -161,6 +170,7 @@ export function buildGroupingPipeline( task: { $cond: { if: { $and: [{ $eq: [{ $size: "$_tasks" }, 1] }, { $ne: [{ $arrayElemAt: ["$_tasks", 0] }, ""] }] }, then: { $arrayElemAt: ["$_tasks", 0] }, else: "$$REMOVE" } }, mcpServers: { $cond: { if: { $and: [{ $eq: [{ $size: "$_mcpKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_mcpKeys", 0] }, ""] }] }, then: "$_firstMcpServers", else: "$$REMOVE" } }, skillRevisions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_skillKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_skillKeys", 0] }, ""] }] }, then: "$_firstSkillRevisions", else: "$$REMOVE" } }, + extensions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_extensionKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_extensionKeys", 0] }, ""] }] }, then: "$_firstExtensions", else: "$$REMOVE" } }, }, }, }, diff --git a/apps/portal/src/lib/grouping.ts b/apps/portal/src/lib/grouping.ts index 864507818..9b07af8e8 100644 --- a/apps/portal/src/lib/grouping.ts +++ b/apps/portal/src/lib/grouping.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { AggregateStats } from "shared"; +import type { AggregateStats } from "../types"; export function formatStatRange( stat: AggregateStats | null, diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index 8fb3353e5..0cfc44354 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -817,6 +817,7 @@ export interface GroupUniformValues { platform?: string; mcpServers?: string[]; skillRevisions?: string[]; + extensions?: string[]; status?: RunStatus; submissionId?: string; task?: string; diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 6cd0bf083..e4e9b36d1 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -159,6 +159,7 @@ export const GroupUniformValuesSchema = z platform: z.string().optional(), mcpServers: z.array(z.string()).optional(), skillRevisions: z.array(z.string()).optional(), + extensions: z.array(z.string()).optional(), status: RequestStatusSchema.optional(), submissionId: z.string().optional(), task: z.string().optional(), diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 344bc7e39..511f9d9b1 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -259,6 +259,7 @@ export interface GroupUniformValues { platform?: string; mcpServers?: string[]; skillRevisions?: string[]; + extensions?: string[]; status?: "pending" | "processing" | "done"; submissionId?: string; task?: string;