Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions workers/api/src/agent-do-tools.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { describe, expect, it } from "vitest";
import {
buildAgentToolDefinitions,
CREATOR_SELECTABLE_TOOLS,
storageToolNameSet,
TOOL_CATALOG,
toolNamesFor,
} from "./agent-do-tools.js";
import type { AgentCapabilities } from "./lib/agent-capabilities.js";
Expand Down Expand Up @@ -101,6 +103,52 @@ describe("agent tool definition helpers", () => {
expect(coderTools).toContain("read_terminal");
});

it("honours a declared tool allowlist over the per-surface default", () => {
// A declared allowlist is authoritative: exactly these catalog tools + BASE.
const names = toolNamesFor({ ...caps([]), tools: ["search_knowledge", "upload_file"] });
expect(names.has("search_knowledge")).toBe(true);
expect(names.has("upload_file")).toBe(true);
// BASE facilities are always included.
expect(names.has("read_memory")).toBe(true);
expect(names.has("fetch_url")).toBe(true);
// Not declared → absent, even though the generic default would include them.
expect(names.has("create_collection")).toBe(false);
expect(names.has("read_terminal")).toBe(false);
});

it("declared allowlist wins even against a surface (e.g. a coding agent opting into KB)", () => {
const names = toolNamesFor({ ...caps(["coding"]), tools: ["search_knowledge"] });
expect(names.has("search_knowledge")).toBe(true); // override
expect(names.has("read_terminal")).toBe(false); // not declared → not granted
expect(names.has("read_memory")).toBe(true); // BASE
});

it("ignores ungrantable names in a declared allowlist (permission-gated / legacy / unknown)", () => {
const names = toolNamesFor({
...caps([]),
tools: ["find_confirmation_link", "submit_job_application", "not_a_real_tool", "list_knowledge"],
});
expect(names.has("find_confirmation_link")).toBe(false); // permission-gated, never declarable
expect(names.has("submit_job_application")).toBe(false); // legacy, not in catalog
expect(names.has("not_a_real_tool")).toBe(false);
expect(names.has("list_knowledge")).toBe(true); // real catalog tool
});

it("an empty declared allowlist falls back to the surface default", () => {
expect(toolNamesFor({ ...caps([]), tools: [] }).has("create_collection")).toBe(true);
});

it("the tool catalog is data: base group + selectable groups, no gated/legacy tools", () => {
expect(TOOL_CATALOG.find((g) => g.tier === "base")?.tools).toContain("fetch_url");
expect(TOOL_CATALOG.some((g) => g.id === "kb_read")).toBe(true);
// Creator-selectable excludes BASE, permission-gated, and legacy tools.
expect(CREATOR_SELECTABLE_TOOLS.has("search_knowledge")).toBe(true);
expect(CREATOR_SELECTABLE_TOOLS.has("read_terminal")).toBe(true);
expect(CREATOR_SELECTABLE_TOOLS.has("read_memory")).toBe(false); // BASE, always granted
expect(CREATOR_SELECTABLE_TOOLS.has("find_confirmation_link")).toBe(false); // gated
expect(CREATOR_SELECTABLE_TOOLS.has("submit_job_application")).toBe(false); // legacy
});

it("returns the complete storage tool name set", () => {
const names = storageToolNameSet();

Expand Down
44 changes: 44 additions & 0 deletions workers/api/src/agent-do-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,39 @@ const FULL: readonly string[] = [
...CODING,
];

// ── The tool catalog (data) ──────────────────────────────────────────────────
// The open, data-driven vocabulary a creator picks from when declaring an agent's
// `capabilities.tools`. Enumerable so the authoring UI (#55) and the pre-review
// safety scanner (#54) can list/reason about what an agent may do — instead of the
// tool set being implied by a hardcoded per-surface `switch`. Deliberately EXCLUDES
// the permission-gated `find_confirmation_link` (granted only via user permission,
// never by declaration) and the legacy `submit_job_application` (superseded by the
// apply workflow).

/** One selectable group in the catalog. `base` is always granted; the rest are opt-in. */
export interface ToolCatalogGroup {
id: string;
label: string;
tools: readonly string[];
/** base = always granted · standard = creator-selectable · runtime = needs a local runner. */
tier: "base" | "standard" | "runtime";
}

export const TOOL_CATALOG: readonly ToolCatalogGroup[] = [
{ id: "base", label: "Memory, tasks, web fetch & context", tools: BASE, tier: "base" },
{ id: "kb_read", label: "Knowledge base — read (RAG)", tools: KB_READ, tier: "standard" },
{ id: "kb_write", label: "Knowledge base — write", tools: KB_WRITE, tier: "standard" },
{ id: "files", label: "File storage", tools: FILES, tier: "standard" },
{ id: "collections", label: "Structured collections", tools: COLLECTIONS, tier: "standard" },
{ id: "coding", label: "Live coding session", tools: CODING, tier: "runtime" },
];

/** Tool names a creator may grant via `capabilities.tools` (everything non-`base` in
* the catalog). BASE is always added on top, so it's intentionally excluded here. */
export const CREATOR_SELECTABLE_TOOLS: ReadonlySet<string> = new Set(
TOOL_CATALOG.filter((g) => g.tier !== "base").flatMap((g) => g.tools),
);

/**
* The tool names an agent may use, resolved from its capabilities:
*
Expand All @@ -58,8 +91,19 @@ const FULL: readonly string[] = [
* coding tools ONLY. Withholding `search_knowledge` is what stops the empty-index
* hallucination at the source, not just in the prompt.
* - **everything else** (apply, insurance, generic, unknown): the FULL set, unchanged.
*
* A declared `capabilities.tools` allowlist takes precedence over all of the above:
* the agent gets exactly those catalog tools plus the universal BASE facilities. This
* is the data-driven path that lets a third-party creator scope tools without a code
* change; the per-surface cases remain the default for agents that don't declare one.
*/
export function toolNamesFor(capabilities?: AgentCapabilities): Set<string> {
const declared = capabilities?.tools;
if (declared && declared.length) {
const set = new Set<string>(BASE);
for (const name of declared) if (CREATOR_SELECTABLE_TOOLS.has(name)) set.add(name);
return set;
}
const surfaces = capabilities?.surfaces ?? [];
if (surfaces.includes("repo")) return new Set<string>([...BASE, ...KB_READ]);
if (surfaces.includes("coding")) return new Set<string>([...BASE, ...CODING]);
Expand Down
28 changes: 27 additions & 1 deletion workers/api/src/lib/agent-capabilities.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { agentCapabilities, hasSurface, sanitizeSettingsSchema } from "./agent-capabilities.js";
import { agentCapabilities, hasSurface, sanitizeSettingsSchema, sanitizeToolList } from "./agent-capabilities.js";

describe("agentCapabilities", () => {
it("uses declared config.capabilities when present", () => {
Expand Down Expand Up @@ -129,4 +129,30 @@ describe("agentCapabilities", () => {
expect(sanitizeSettingsSchema([])).toBeUndefined();
});
});

describe("declared tool allowlist", () => {
it("resolves config.capabilities.tools onto the capabilities", () => {
const cfg = JSON.stringify({ capabilities: { surfaces: ["repo"], tools: ["search_knowledge", "read_knowledge"] } });
expect(agentCapabilities({ config: cfg }).tools).toEqual(["search_knowledge", "read_knowledge"]);
});

it("resolves tools even when no surfaces are declared (fallback path)", () => {
const cfg = JSON.stringify({ capabilities: { tools: ["upload_file"] } });
expect(agentCapabilities({ slug: "generic", config: cfg }).tools).toEqual(["upload_file"]);
});

it("is undefined when no tools are declared", () => {
expect(agentCapabilities({ slug: "coder" }).tools).toBeUndefined();
expect(agentCapabilities({ config: JSON.stringify({ capabilities: { surfaces: ["coding"] } }) }).tools).toBeUndefined();
});

it("sanitizeToolList dedupes, caps at 40, and drops junk", () => {
expect(sanitizeToolList(["read_memory", "read_memory", "fetch_url"])).toEqual(["read_memory", "fetch_url"]);
expect(sanitizeToolList(["search_knowledge", 3, null, "Bad Name", "-x"])).toEqual(["search_knowledge"]);
expect(sanitizeToolList(Array.from({ length: 50 }, (_, i) => `t${i}`))).toHaveLength(40);
expect(sanitizeToolList("nope")).toBeUndefined();
expect(sanitizeToolList([])).toBeUndefined();
expect(sanitizeToolList([1, 2, 3])).toBeUndefined();
});
});
});
35 changes: 34 additions & 1 deletion workers/api/src/lib/agent-capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ export interface AgentCapabilities {
runtime: AgentRuntimeKind;
/** Brain workflow binding name, when the agent has an autonomous loop. */
workflow: "JOB_APPLY" | "CODING_SESSION" | "INSURANCE_QUOTES" | null;
/** Declared tool allowlist — names from the platform tool catalog (see
* agent-do-tools `TOOL_CATALOG`). When present it is AUTHORITATIVE: the agent gets
* exactly these catalog tools plus the universal BASE facilities, replacing the
* per-surface default. Absent → the surface-derived default applies. This is the
* open, data-driven vocabulary a third-party creator scopes without a code change. */
tools?: string[];
/** Phase 3: agent-published UIs the console loads dynamically from bundles. */
customSurfaces?: CustomSurface[];
/** The agent's single work board columns — declared, else a per-surface default. */
Expand Down Expand Up @@ -210,6 +216,29 @@ export function sanitizeSettingsSchema(value: unknown): SettingsField[] | undefi
return out.length ? out : undefined;
}

const MAX_DECLARED_TOOLS = 40;
const TOOL_NAME_RE = /^[a-z][a-z0-9_]{1,48}$/;

/** Validate a declared tool allowlist: an array of tool-name strings, deduped and
* capped. Names are intersected with the real catalog at runtime (agent-do-tools
* `toolNamesFor`), so an unknown/ungrantable name here is simply ignored — this only
* trims obvious junk before it reaches config. Exported so create/update-agent routes
* can sanitize with the same rules. */
export function sanitizeToolList(value: unknown): string[] | undefined {
if (!Array.isArray(value)) return undefined;
const out: string[] = [];
const seen = new Set<string>();
for (const v of value) {
if (out.length >= MAX_DECLARED_TOOLS) break;
if (typeof v !== "string") continue;
const name = v.trim();
if (!TOOL_NAME_RE.test(name) || seen.has(name)) continue;
seen.add(name);
out.push(name);
}
return out.length ? out : undefined;
}

const KNOWN_SURFACES = new Set<AgentSurface>(["apply", "coding", "insurance", "repo"]);

/** Minimal shape we need off an `agents` row to resolve capabilities. */
Expand Down Expand Up @@ -245,13 +274,17 @@ export function agentCapabilities(agent: AgentLike): AgentCapabilities {
// so the seed migrations' json_set('$.settingsSchema', …) is unconditionally
// idempotent. Honored in every path, like customSurfaces.
const settingsSchema = sanitizeSettingsSchema(cfg.settingsSchema);
// Declared tool allowlist (sibling of surfaces under config.capabilities). Honored in
// every path; runtime intersects it with the real catalog (agent-do-tools).
const tools = sanitizeToolList((declared as Record<string, unknown> | undefined)?.tools);

if (declared && Array.isArray(declared.surfaces)) {
const surfaces = declared.surfaces.filter((s): s is AgentSurface => KNOWN_SURFACES.has(s as AgentSurface));
return {
surfaces,
runtime: declared.runtime ?? null,
workflow: declared.workflow ?? null,
tools,
customSurfaces,
boardColumns: declaredColumns ?? defaultBoardColumns(surfaces),
settingsSchema,
Expand All @@ -269,7 +302,7 @@ export function agentCapabilities(agent: AgentLike): AgentCapabilities {
} else {
base = { surfaces: [], runtime: null, workflow: null };
}
return { ...base, customSurfaces, boardColumns: declaredColumns ?? defaultBoardColumns(base.surfaces), settingsSchema };
return { ...base, tools, customSurfaces, boardColumns: declaredColumns ?? defaultBoardColumns(base.surfaces), settingsSchema };
}

/** True if the agent opts into a given console surface. */
Expand Down
Loading