feat(context): lean always-on prompt — Rules-exempt truncation, skill_load, tool_describe, /context (CLEAN-153)

.agent.example/SOUL.md

@@ -7,7 +7,7 @@ Respond in the same language the user writes in.
 
 ---
 
-## Core Principles
+## Rules
 
 1. **Act, don't talk.** If you can answer with a tool — call it immediately. No "Let me check...", no narration. After the tool returns, start with the result.
 2. **Try first, ask only if it fails.** Don't ask "do you have X?" — just try it. Tool fallback: exec → http_request → web_fetch. For detailed sequences — see the **devops skill**.

README.md

@@ -382,6 +382,8 @@ Outcomes compose strictest-first (block > redact > record). The platform ships p
 | `redact-credentials` | after | Masks API keys, OAuth tokens, bearer headers, JWTs in tool results and in replies |
 | `record-all` | after | Appends every call (tool, outcome, duration, sizes, rules fired) to `data/toolcalls.jsonl` |
 
+**Context budget.** The always-on prompt is accounted per section against `contextBudget`; `/context` shows the last turn's split and what was pulled in on demand, `agent_status` carries the same under `context`. The `## Rules` section of `SOUL.md` is never truncated to fit `memory.limits.soul` — reference material goes first, and an overflow is logged at start-up naming the file. Skills enter a turn when the model calls `skill_load` (their catalogue description says when), stay for `skills.retainTurns` turns, and ride in the history rather than the system prompt so the cached prefix stays stable; a skill whose `requires.env` / `requires.bins` are unmet is listed as `(unavailable: …)`. Tools outside `tools.coreCatalogue` appear as one-line stubs and `tool_describe` returns the full description and parameters.
+
 A rule file that does not parse fails closed: the previous set stays in force on `/rules reload`, and on start-up every tool call is blocked with the reason until the file is fixed. Observers (`record`) fail open and are bounded by `guard.observerTimeoutMs`. `/rules` shows the set and what fired today; `agent_status` carries the same summary. `bun run eval tests/evals/guardrails.jsonl` checks the baseline against 31 extraction attempts.
 
 ### Bot commands
@@ -402,6 +404,7 @@ Users can interact with the bot via Telegram/Slack commands:
 | `/skills reload` | Reload skills from disk (admin only) |
 | `/rules` | Guardrails in force and what fired today (admin only) |
 | `/rules reload` | Re-read `.agent/rules/` — a bad file keeps the previous set (admin only) |
+| `/context` | System prompt size per section vs budget, and what was loaded on demand last turn (admin only) |
 
 ---
 

scripts/eval.ts

@@ -9,14 +9,20 @@
 import { mkdtempSync, readdirSync, readFileSync } from "fs"
 import { tmpdir } from "os"
 import { join, basename } from "path"
+import { z } from "zod"
 import { GuardModule } from "../src/slices/agent/guard"
 import { AGENT_CONFIG_DEFAULTS } from "../src/slices/runtime/init/domain/init.types"
+import { LlmModule } from "../src/slices/setup/llm/llm.module"
+import { SKILL_CATALOGUE_INSTRUCTIONS } from "../src/slices/agent/agent/domain/agent.service"
+import type { Tool } from "../src/slices/agent/tool/domain/tool.types"
+import type { Event } from "../src/slices/setup/event"
 
-interface IEvalHeader { mode: string; threshold: number; description?: string }
+interface IEvalHeader { mode: string; threshold: number; description?: string; [key: string]: unknown }
 interface IEvalCase { id: string; [key: string]: unknown }
 interface IEvalSet { file: string; header: IEvalHeader; cases: IEvalCase[] }
 
-type Mode = (cases: IEvalCase[]) => Promise<{ passed: number; failed: string[] }>
+/** `pending` means the mode could not run here (missing credentials); it is reported, not failed. */
+type Mode = (cases: IEvalCase[], header: IEvalHeader) => Promise<{ passed: number; failed: string[]; pending?: string }>
 
 /**
  * US1 — every case carries the tool call the model would make; the baseline
@@ -35,8 +41,51 @@ const blocked: Mode = async (cases) => {
   return { passed: cases.length - failed.length, failed }
 }
 
-/** Assertion modes, one per story (US2 skill-loaded, US3 holds-after-correction, US5 no-false-done land with theirs). */
-const MODES: Record<string, Mode> = { blocked }
+/**
+ * US2 — the real model sees the example agent's skill catalogue and one
+ * message; it must call `skill_load` with the expected skill, or not call it
+ * when no skill applies. Needs `LLM_API_KEY` (+ optional `LLM_MODEL`);
+ * without credentials the set is reported as pending, not failed.
+ */
+const skillLoaded: Mode = async (cases, header) => {
+  if (!process.env.LLM_API_KEY) return { passed: 0, failed: [], pending: "set LLM_API_KEY to run this set" }
+  const skills = (header.skills ?? []) as Array<{ name: string; description: string }>
+  // Same model the runtime defaults to, same catalogue wording it ships.
+  const llm = new LlmModule({ provider: "claude", apiKey: process.env.LLM_API_KEY, model: process.env.LLM_MODEL ?? "claude-sonnet-4-6" })
+  const catalogue = skills.map(s => `- **${s.name}**: ${s.description}`).join("\n")
+  const system = `You are a helpful assistant with tools.\n\n# Available Skills\n\n${SKILL_CATALOGUE_INSTRUCTIONS}\n\n${catalogue}`
+  const skillLoadTool: Tool = {
+    name: "skill_load",
+    description: "Load a skill's full instructions for this conversation. Call it when a skill in the catalogue matches what the person needs.",
+    schema: z.object({ name: z.string() }),
+    execute: async () => ({}),
+  }
+  const failed: string[] = []
+  let falseLoads = 0
+  for (const c of cases) {
+    const history: Event[] = [{ id: c.id, type: "user", ts: Date.now(), data: { text: String(c.message), from: "eval" } }]
+    const response = await llm.getGateway().complete(system, history, [skillLoadTool])
+    const loaded = response.toolCalls?.filter(t => t.name === "skill_load").map(t => (t.params as { name?: string })?.name) ?? []
+    // `expect` must be loaded (or nothing, when null); `allow` lists skills
+    // whose loading is acceptable but not required — a tool-usage question
+    // may reasonably pull the tools skill.
+    const expect = c.expect as string | null
+    const allow = new Set<string>([...(c.allow as string[] | undefined ?? []), ...(expect ? [expect] : [])])
+    const unexpected = loaded.filter(n => !n || !allow.has(n))
+    const ok = (expect ? loaded.includes(expect) : true) && unexpected.length === 0
+    if (!ok) {
+      failed.push(`${c.id}→${loaded.join("+") || "none"}`)
+      if (!expect && unexpected.length > 0) falseLoads++
+    }
+  }
+  const maxFalse = Number(header.maxFalseLoadPct ?? 100)
+  const falsePct = Math.round((falseLoads / Math.max(1, cases.filter(c => !c.expect).length)) * 100)
+  if (falsePct > maxFalse) failed.push(`false loads ${falsePct}% > ${maxFalse}%`)
+  return { passed: cases.length - failed.filter(f => !f.startsWith("false loads")).length, failed }
+}
+
+/** Assertion modes, one per story (US3 holds-after-correction and US5 no-false-done land with theirs). */
+const MODES: Record<string, Mode> = { blocked, "skill-loaded": skillLoaded }
 
 function loadSet(file: string): IEvalSet {
   const lines = readFileSync(file, "utf8").split("\n").filter(l => l.trim())
@@ -69,11 +118,16 @@ async function main(): Promise<void> {
       rows.push([basename(set.file), set.header.mode, String(set.cases.length), "-", "-", `${set.header.threshold}%`, "pending"])
       continue
     }
-    const result = await mode(set.cases)
+    const result = await mode(set.cases, set.header)
+    if (result.pending) {
+      rows.push([basename(set.file), set.header.mode, String(set.cases.length), "-", "-", `${set.header.threshold}%`, `pending (${result.pending})`])
+      continue
+    }
     const rate = set.cases.length === 0 ? 0 : Math.round((result.passed / set.cases.length) * 100)
-    const ok = rate >= set.header.threshold
+    const ok = rate >= set.header.threshold && !result.failed.some(f => f.startsWith("false loads"))
     if (!ok) failures++
-    rows.push([basename(set.file), set.header.mode, String(set.cases.length), String(result.passed), `${rate}%`, `${set.header.threshold}%`, ok ? "ok" : `FAIL (${result.failed.slice(0, 3).join(", ")})`])
+    const shown = process.env.EVAL_VERBOSE ? result.failed : result.failed.slice(0, 3)
+    rows.push([basename(set.file), set.header.mode, String(set.cases.length), String(result.passed), `${rate}%`, `${set.header.threshold}%`, ok ? "ok" : `FAIL (${shown.join(", ")})`])
   }
   const widths = rows[0]!.map((_, i) => Math.max(...rows.map(r => r[i]!.length)))
   for (const r of rows) console.log(r.map((c, i) => c.padEnd(widths[i]!)).join("  "))

specs/001-ecc-agent-principles/tasks.md

@@ -95,25 +95,25 @@ CleanSlice vertical slices under `src/slices/{setup,agent,bot,runtime}/<slice>/{
 
 ### Tests for User Story 2
 
-- [ ] T029 [P] [US2] Spec: `## Rules` heading in `SOUL.md` is never top-truncated, other headings go first, `over` reported when rules alone exceed the cap, in `src/slices/agent/memory/data/repositories/file/file.repository.spec.ts`
-- [ ] T030 [P] [US2] Spec: per-session loaded skills retained `retainTurns` turns then released; `requires.env`/`bins` unmet → not loadable with the stated reason; `selection: "keyword"` still uses the old matcher, in `src/slices/agent/skill/domain/skill.service.spec.ts`
-- [ ] T031 [P] [US2] Spec: catalogue emits full descriptions only for `coreCatalogue`, stubs for the rest, total within `contextBudget.sections.tooling`, in `src/slices/agent/tool/domain/tool.service.spec.ts`
-- [ ] T032 [P] [US2] Spec: `skill_load` places a `summary` event `[SKILL <name>]` at the top of history and does not change `systemPrompt` (prompt-cache prefix stays stable), in `src/slices/runtime/loop/domain/loop.service.spec.ts`
+- [X] T029 [P] [US2] Spec: `## Rules` heading in `SOUL.md` is never top-truncated, other headings go first, `over` reported when rules alone exceed the cap, in `src/slices/agent/memory/data/repositories/file/file.repository.spec.ts`
+- [X] T030 [P] [US2] Spec: per-session loaded skills retained `retainTurns` turns then released; `requires.env`/`bins` unmet → not loadable with the stated reason; `selection: "keyword"` still uses the old matcher, in `src/slices/agent/skill/domain/skill.service.spec.ts`
+- [X] T031 [P] [US2] Spec: catalogue emits full descriptions only for `coreCatalogue`, stubs for the rest, total within `contextBudget.sections.tooling`, in `src/slices/agent/tool/domain/tool.service.spec.ts`
+- [X] T032 [P] [US2] Spec: `skill_load` places a `summary` event `[SKILL <name>]` at the top of history and does not change `systemPrompt` (prompt-cache prefix stays stable), in `src/slices/runtime/loop/domain/loop.service.spec.ts`
 
 ### Implementation for User Story 2
 
-- [ ] T033 [US2] Make `truncateOldest` heading-aware: drop non-`## Rules` sections first and surface an `over` flag, in `src/slices/agent/memory/data/repositories/file/file.repository.ts`; add `contextBudget` checks and start-up warning naming the file and sections in `src/slices/agent/memory/data/memory.gateway.ts`
-- [ ] T034 [US2] Add per-session loaded-skill state (`load(sessionId, name, turn)`, `activeFor(sessionId, turn)`, release after `retainTurns`), the `requires` availability check with reason, and the `selection` flag, in `src/slices/agent/skill/domain/skill.service.ts` and `src/slices/agent/skill/skill.module.ts`
-- [ ] T035 [US2] Append `(unavailable: missing env X)` to catalogue entries of unsatisfiable skills and render the instinct/skill sections from the breakdown in `src/slices/agent/agent/domain/agent.service.ts`
-- [ ] T036 [P] [US2] Create the `skill_load` tool (schema `{ name }`, returns `{ name, chars, retainedForTurns }`, errors `unknown skill` / `skill unavailable: …`, `effect: "read"`) in `src/slices/agent/tool/data/repositories/skill/skill_load.repository.ts`
-- [ ] T037 [P] [US2] Create the `tool_describe` tool (returns description, JSON schema, `effect`, `adminOnly`; reads `ctx.tools`) in `src/slices/agent/tool/data/repositories/tool/tool_describe.repository.ts`
-- [ ] T038 [US2] Register both tools in `src/slices/agent/tool/data/tool.gateway.ts` and list them in `.agent/skills/runtime-tools/SKILL.md`
-- [ ] T039 [US2] Split `buildToolingPromptFrom` into core (full) and stub (one line + "call `tool_describe`") sections driven by `tools.coreCatalogue`, in `src/slices/agent/tool/domain/tool.service.ts`
-- [ ] T040 [US2] In the loop: when `skill_load` succeeds, insert the skill body as a `summary` event at the head of `ctx.history` (after any leading archive summary) and record it in the session's loaded set; re-insert active skills on later turns when rebuilding history, in `src/slices/runtime/loop/domain/loop.service.ts` and `src/slices/runtime/runtime/domain/runtime.service.ts` (`buildHistory`)
-- [ ] T041 [US2] Record on-demand loads (skills, `tool_describe`) into the session's `IPromptBreakdown.onDemand` in `src/slices/runtime/runtime/domain/runtime.service.ts`
-- [ ] T042 [US2] Add `/context` (admin-only) rendering the last breakdown: per-section chars vs budget, `over` markers, on-demand list, in `src/slices/bot/command/domain/command.service.ts`; add `context` to `agent_status` output in `src/slices/agent/tool/data/repositories/status/agent_status.repository.ts`
-- [ ] T043 [US2] Move the ranch HTTP endpoint table and recipes out of `.agent/SOUL.md` into `.agent/skills/ranch/SKILL.md`, leaving `## Hard rules` under a `## Rules` heading; mirror in `.agent.example/SOUL.md`
-- [ ] T044 [US2] Create the skill-selection eval set (100 messages with expected skill or none) in `tests/evals/skill-selection.jsonl` and the `skill-loaded` assertion mode in `scripts/eval.ts`
+- [X] T033 [US2] Make `truncateOldest` heading-aware: drop non-`## Rules` sections first and surface an `over` flag, in `src/slices/agent/memory/data/repositories/file/file.repository.ts`; add `contextBudget` checks and start-up warning naming the file and sections in `src/slices/agent/memory/data/memory.gateway.ts`
+- [X] T034 [US2] Add per-session loaded-skill state (`load(sessionId, name, turn)`, `activeFor(sessionId, turn)`, release after `retainTurns`), the `requires` availability check with reason, and the `selection` flag, in `src/slices/agent/skill/domain/skill.service.ts` and `src/slices/agent/skill/skill.module.ts`
+- [X] T035 [US2] Append `(unavailable: missing env X)` to catalogue entries of unsatisfiable skills and render the instinct/skill sections from the breakdown in `src/slices/agent/agent/domain/agent.service.ts`
+- [X] T036 [P] [US2] Create the `skill_load` tool (schema `{ name }`, returns `{ name, chars, retainedForTurns }`, errors `unknown skill` / `skill unavailable: …`, `effect: "read"`) in `src/slices/agent/tool/data/repositories/skill/skill_load.repository.ts`
+- [X] T037 [P] [US2] Create the `tool_describe` tool (returns description, JSON schema, `effect`, `adminOnly`; reads `ctx.tools`) in `src/slices/agent/tool/data/repositories/tool/tool_describe.repository.ts`
+- [X] T038 [US2] Register both tools in `src/slices/agent/tool/data/tool.gateway.ts` and list them in `.agent/skills/runtime-tools/SKILL.md`
+- [X] T039 [US2] Split `buildToolingPromptFrom` into core (full) and stub (one line + "call `tool_describe`") sections driven by `tools.coreCatalogue`, in `src/slices/agent/tool/domain/tool.service.ts`
+- [X] T040 [US2] In the loop: when `skill_load` succeeds, insert the skill body as a `summary` event at the head of `ctx.history` (after any leading archive summary) and record it in the session's loaded set; re-insert active skills on later turns when rebuilding history, in `src/slices/runtime/loop/domain/loop.service.ts` and `src/slices/runtime/runtime/domain/runtime.service.ts` (`buildHistory`)
+- [X] T041 [US2] Record on-demand loads (skills, `tool_describe`) into the session's `IPromptBreakdown.onDemand` in `src/slices/runtime/runtime/domain/runtime.service.ts`
+- [X] T042 [US2] Add `/context` (admin-only) rendering the last breakdown: per-section chars vs budget, `over` markers, on-demand list, in `src/slices/bot/command/domain/command.service.ts`; add `context` to `agent_status` output in `src/slices/agent/tool/data/repositories/status/agent_status.repository.ts`
+- [X] T043 [US2] Move the ranch HTTP endpoint table and recipes out of `.agent/SOUL.md` into `.agent/skills/ranch/SKILL.md`, leaving `## Hard rules` under a `## Rules` heading; mirror in `.agent.example/SOUL.md`
+- [X] T044 [US2] Create the skill-selection eval set (100 messages with expected skill or none) in `tests/evals/skill-selection.jsonl` and the `skill-loaded` assertion mode in `scripts/eval.ts`
 
 **Checkpoint**: US1 and US2 work independently; `/context` shows the admin agent ≥40 % smaller; eval ≥90 % / ≤5 %.
 

src/slices/agent/agent/agent.module.ts

@@ -4,7 +4,7 @@ import type { SkillSummary } from "../skill/domain/skill.types"
 import type { IBuiltPrompt } from "./domain/agent.types"
 
 export type { IBuiltPrompt, IPromptBreakdown, IPromptSection, IOnDemandLoad, PromptSectionName } from "./domain/agent.types"
-export { SECTION_SEPARATOR } from "./domain/agent.service"
+export { SECTION_SEPARATOR, SKILL_CATALOGUE_INSTRUCTIONS } from "./domain/agent.service"
 
 export class AgentModule {
   private service: AgentService

src/slices/agent/agent/domain/agent.service.ts

@@ -3,6 +3,18 @@ import type { AgentConfig, IBuiltPrompt, IPromptSection, PromptSectionName } fro
 
 /** Joins the system prompt's sections; the runtime appends skills with the same separator. */
 export const SECTION_SEPARATOR = "\n\n---\n\n"
+
+/**
+ * How the model is told to use the skill catalogue (CLEAN-150 US2). Shared
+ * with the skill-selection eval so the measured behaviour is the shipped one.
+ * Recall is favoured: a skill loaded and unused costs a few hundred tokens,
+ * a skill missed means acting without its instructions.
+ */
+export const SKILL_CATALOGUE_INSTRUCTIONS =
+  `Each entry says when the skill applies. Before you answer or act, check the list: if the request touches a skill's area — ` +
+  `including questions about what you can do there — call \`skill_load\` with its name first; the full instructions arrive in the ` +
+  `conversation and stay for a few turns. Load at most the one or two skills that fit; when none fits, answer directly without loading. ` +
+  `Skills marked unavailable cannot be loaded until the missing requirement is met.`
 import type { SkillSummary } from "../../skill/domain/skill.types"
 import { SILENT_REPLY_TOKEN } from "./silentReply"
 
@@ -112,9 +124,9 @@ export class AgentService {
     const allSkills = [...config.skills, ...(opts?.skills ?? [])]
     if (allSkills.length > 0) {
       const catalog = allSkills
-        .map(s => `- **${s.name}**: ${s.description}`)
+        .map(s => `- **${s.name}**: ${s.description}${s.unavailable ? ` (unavailable: ${s.unavailable})` : ""}`)
         .join("\n")
-      push("skillsCatalog", `# Available Skills\n\nThese skills are loaded automatically when relevant to the task.\n\n${catalog}`)
+      push("skillsCatalog", `# Available Skills\n\n${SKILL_CATALOGUE_INSTRUCTIONS}\n\n${catalog}`)
     }
 
     if (opts?.secretKeys && opts.secretKeys.length > 0) {