Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions apps/api/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
} from "./prompt-feature-llm.js";
import { computeAnalysis, AnalysisResponse, AnalyzableRun } from "./analysis.js";
import { computeMdp, parseStateKey, type MdpAnalyzableRun } from "./criteria-mdp.js";
import { parseCommaSeparatedIds, buildScenarioCriteriaFilter, buildPromptFeatureTaskPromptFilter } from "./request-filters.js";
import { TaskPromptStore, computeTaskPromptId, type TaskPromptDocument, SkillRevisionStore, SkillResolver, type SkillDocument, type SkillRevisionDocument, type SkillSearchResult } from "shared";
import { evaluateTrigger } from "shared";
import { checkMigrations } from "db-migrations/check-migrations";
Expand Down Expand Up @@ -950,6 +951,8 @@ app.get("/api/v1/requests", async (req: Request, res: Response, next: NextFuncti
const workerFilter = req.query.worker as string;
const taskPromptIdFilter = req.query.taskPromptId as string;
const criteriaFilter = req.query.criteria as string;
const scenarioCriteriaFilter = req.query.scenarioCriteria as string;
const promptFeaturesFilter = req.query.promptFeatures as string;
const includeDeleted = req.query.includeDeleted === "true";

const filter: Record<string, unknown> = {};
Expand Down Expand Up @@ -984,6 +987,29 @@ app.get("/api/v1/requests", async (req: Request, res: Response, next: NextFuncti
}
}

// Filter by scenario criteria IDs (e.g. "has_azure,has_cloud")
// Matches runs whose scenario.criteria array contains ALL specified IDs.
const scenarioCriteriaIds = parseCommaSeparatedIds(scenarioCriteriaFilter);
const scenarioCriteriaClause = buildScenarioCriteriaFilter(scenarioCriteriaIds);
if (scenarioCriteriaClause) {
Object.assign(filter, scenarioCriteriaClause);
}

// Filter by prompt features (e.g. "asks_for_api,asks_for_azure")
// First resolves task prompt IDs that have ALL specified features detected,
// then filters runs by those task prompt IDs.
const featureIds = parseCommaSeparatedIds(promptFeaturesFilter);
const featureTaskPromptFilter = buildPromptFeatureTaskPromptFilter(featureIds);
if (featureTaskPromptFilter) {
const matchingTaskPrompts = await taskPromptCollection
.find(featureTaskPromptFilter, { projection: { _id: 1 } })
.toArray();
const matchingIds = matchingTaskPrompts.map(tp => tp._id);
filter.taskPromptId = taskPromptIdFilter
? (matchingIds.includes(taskPromptIdFilter) ? taskPromptIdFilter : "__no_match__")
: { $in: matchingIds };
}

const resources = await collection
.find(filter)
.sort({ createdAt: -1 })
Expand Down
116 changes: 116 additions & 0 deletions apps/api/src/request-filters.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { describe, it, expect } from "vitest";
import {
parseCommaSeparatedIds,
buildScenarioCriteriaFilter,
buildPromptFeatureTaskPromptFilter,
} from "./request-filters.js";

// ─── parseCommaSeparatedIds ──────────────────────────────────────────────────

describe("parseCommaSeparatedIds", () => {
it("parses a single ID", () => {
expect(parseCommaSeparatedIds("has_azure")).toEqual(["has_azure"]);
});

it("parses multiple comma-separated IDs", () => {
expect(parseCommaSeparatedIds("has_azure,has_cloud,has_node")).toEqual([
"has_azure",
"has_cloud",
"has_node",
]);
});

it("trims whitespace from IDs", () => {
expect(parseCommaSeparatedIds(" has_azure , has_cloud ")).toEqual([
"has_azure",
"has_cloud",
]);
});

it("filters out empty segments", () => {
expect(parseCommaSeparatedIds(",has_azure,,has_cloud,")).toEqual([
"has_azure",
"has_cloud",
]);
});

it("returns empty array for undefined", () => {
expect(parseCommaSeparatedIds(undefined)).toEqual([]);
});

it("returns empty array for null", () => {
expect(parseCommaSeparatedIds(null)).toEqual([]);
});

it("returns empty array for empty string", () => {
expect(parseCommaSeparatedIds("")).toEqual([]);
});

it("returns empty array for whitespace-only string", () => {
expect(parseCommaSeparatedIds(" , , ")).toEqual([]);
});
});

// ─── buildScenarioCriteriaFilter ─────────────────────────────────────────────

describe("buildScenarioCriteriaFilter", () => {
it("returns undefined for empty IDs", () => {
expect(buildScenarioCriteriaFilter([])).toBeUndefined();
});

it("builds $all filter for a single criterion", () => {
expect(buildScenarioCriteriaFilter(["has_azure"])).toEqual({
"scenario.criteria": { $all: ["has_azure"] },
});
});

it("builds $all filter for multiple criteria", () => {
expect(buildScenarioCriteriaFilter(["has_azure", "has_cloud"])).toEqual({
"scenario.criteria": { $all: ["has_azure", "has_cloud"] },
});
});
});

// ─── buildPromptFeatureTaskPromptFilter ──────────────────────────────────────

describe("buildPromptFeatureTaskPromptFilter", () => {
it("returns undefined for empty feature IDs", () => {
expect(buildPromptFeatureTaskPromptFilter([])).toBeUndefined();
});

it("builds $and + $elemMatch filter for a single feature", () => {
const result = buildPromptFeatureTaskPromptFilter(["asks_for_api"]);
expect(result).toEqual({
$and: [
{
features: {
$elemMatch: { featureId: "asks_for_api", detected: true },
},
},
],
deletedAt: { $exists: false },
});
});

it("builds $and + $elemMatch filter for multiple features", () => {
const result = buildPromptFeatureTaskPromptFilter(["asks_for_api", "asks_for_azure"]);
expect(result).toEqual({
$and: [
{
features: {
$elemMatch: { featureId: "asks_for_api", detected: true },
},
},
{
features: {
$elemMatch: { featureId: "asks_for_azure", detected: true },
},
},
],
deletedAt: { $exists: false },
});
});
});
42 changes: 42 additions & 0 deletions apps/api/src/request-filters.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

/**
* Utilities for building MongoDB filter objects for the /api/v1/requests endpoint.
* Extracted for testability.
*/

/**
* Parse a comma-separated string of IDs into a trimmed, non-empty array.
* Returns an empty array if the input is falsy or contains only whitespace.
*/
export function parseCommaSeparatedIds(input: string | undefined | null): string[] {
if (!input) return [];
return input.split(",").map(s => s.trim()).filter(Boolean);
}

/**
* Build a MongoDB filter clause for scenario.criteria containing ALL given IDs.
* Returns undefined if no IDs are provided.
*/
export function buildScenarioCriteriaFilter(ids: string[]): Record<string, unknown> | undefined {
if (ids.length === 0) return undefined;
return { "scenario.criteria": { $all: ids } };
}

/**
* Build a MongoDB $and clause to match task prompts that have ALL given
* prompt features detected (featureId + detected: true).
* Returns undefined if no feature IDs are provided.
*/
export function buildPromptFeatureTaskPromptFilter(featureIds: string[]): Record<string, unknown> | undefined {
if (featureIds.length === 0) return undefined;
return {
$and: featureIds.map(fid => ({
features: {
$elemMatch: { featureId: fid, detected: true },
},
})),
deletedAt: { $exists: false },
};
}
84 changes: 84 additions & 0 deletions apps/portal/src/components/PromptFeatureFilterBar.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { Filter, X } from "lucide-react";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Badge } from "@/components/ui/badge";

interface PromptFeatureFilterBarProps {
availableFeatures: string[];
selectedFeatures: string[];
onToggle: (id: string) => void;
onClear: () => void;
/** Override the card title (default: "Prompt Feature Filter") */
title?: string;
/** Override the description when nothing is selected */
emptyDescription?: string;
/** Override the description template when features are selected (receives count) */
selectedDescription?: (count: number) => string;
}

export function PromptFeatureFilterBar({
availableFeatures,
selectedFeatures,
onToggle,
onClear,
title = "Prompt Feature Filter",
emptyDescription = "Click features to filter runs by task prompt characteristics.",
selectedDescription = (count: number) =>
`Showing runs whose task prompt has ${count} selected feature${count !== 1 ? "s" : ""} detected.`,
}: PromptFeatureFilterBarProps) {
if (availableFeatures.length === 0) {
return null;
}

const selectedSet = new Set(selectedFeatures);

return (
<Card>
<CardHeader className="pb-3">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<Filter className="h-4 w-4 text-muted-foreground" />
<CardTitle className="text-base">{title}</CardTitle>
</div>
{selectedFeatures.length > 0 && (
<button
onClick={onClear}
className="text-xs text-muted-foreground hover:text-foreground underline"
>
Clear all
</button>
)}
</div>
<CardDescription>
{selectedFeatures.length === 0
? emptyDescription
: selectedDescription(selectedFeatures.length)}
</CardDescription>
</CardHeader>
<CardContent>
<div className="flex flex-wrap gap-1.5">
{availableFeatures.map((id) => {
const isSelected = selectedSet.has(id);
return (
<Badge
key={id}
variant={isSelected ? "default" : "outline"}
className={`cursor-pointer transition-colors ${
isSelected
? "bg-primary hover:bg-primary/80"
: "hover:bg-muted"
}`}
onClick={() => onToggle(id)}
>
{id}
{isSelected && <X className="ml-1 h-3 w-3" />}
</Badge>
);
})}
</div>
</CardContent>
</Card>
);
}
6 changes: 4 additions & 2 deletions apps/portal/src/lib/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
}

export const api = {
/** List all runs, optionally filtered by worker, task prompt, and/or MDP criteria state */
listRuns: (opts?: { worker?: string; taskPromptId?: string; criteria?: string }): Promise<Run[]> => {
/** List all runs, optionally filtered by worker, task prompt, criteria state, scenario criteria, and/or prompt features */
listRuns: (opts?: { worker?: string; taskPromptId?: string; criteria?: string; scenarioCriteria?: string[]; promptFeatures?: string[] }): Promise<Run[]> => {
const params = new URLSearchParams();
if (opts?.worker) params.set("worker", opts.worker);
if (opts?.taskPromptId) params.set("taskPromptId", opts.taskPromptId);
if (opts?.criteria) params.set("criteria", opts.criteria);
if (opts?.scenarioCriteria?.length) params.set("scenarioCriteria", opts.scenarioCriteria.join(","));
if (opts?.promptFeatures?.length) params.set("promptFeatures", opts.promptFeatures.join(","));
const qs = params.toString();
return request(`/requests${qs ? `?${qs}` : ""}`);
},
Expand Down
Loading