From a8e909862276ed3658c749da9fb402e107d0ff03 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 10 Jun 2026 16:45:05 +0300 Subject: [PATCH 1/8] feat(churn): git churn ingest and churn-complexity-hotspots recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add file_churn substrate with full/incremental/idle refresh on every index pass, codemap ingest-churn for golden seeds, and a refactor-priority recipe ranked by churn × complexity with perf-baseline churn_ms gating. --- .changeset/churn-complexity-hotspots.md | 5 + docs/architecture.md | 17 + docs/glossary.md | 6 +- docs/golden-queries.md | 2 +- docs/plans/churn-complexity-hotspots.md | 148 ++---- docs/roadmap.md | 2 +- fixtures/CAPABILITIES.json | 14 + fixtures/benchmark/perf-baseline.json | 17 +- .../churn-complexity-hotspots-by-symbol.json | 132 ++++++ .../minimal/churn-complexity-hotspots.json | 41 ++ fixtures/golden/minimal/files-count.json | 2 +- fixtures/golden/minimal/files-hashes.json | 6 + fixtures/golden/minimal/files-largest.json | 12 +- fixtures/golden/minimal/index-summary.json | 2 +- .../golden/minimal/index-table-stats.json | 5 +- .../golden/minimal/source-fts-row-count.json | 2 +- fixtures/golden/scenarios.json | 19 +- fixtures/minimal/file-churn-seed.json | 32 ++ scripts/agent-eval/scenarios.json | 9 + scripts/check-perf-baseline.ts | 2 + scripts/query-golden-coverage-matrix.test.mjs | 1 + scripts/query-golden/run-setup.ts | 24 +- scripts/query-golden/schema.ts | 5 + src/application/churn-ingest.test.ts | 222 +++++++++ src/application/churn-ingest.ts | 420 ++++++++++++++++++ src/application/context-engine.test.ts | 12 +- src/application/context-engine.ts | 19 +- src/application/index-engine.ts | 4 +- src/application/ingest-churn-run.test.ts | 72 +++ src/application/ingest-churn-run.ts | 135 ++++++ src/application/run-index.ts | 211 +++++---- src/application/types.ts | 3 + src/cli/bootstrap-codemap.ts | 8 +- src/cli/bootstrap.ts | 19 +- src/cli/cmd-index.ts | 1 + src/cli/cmd-ingest-churn.ts | 109 +++++ src/cli/main.ts | 33 +- src/config.test.ts | 17 + src/config.ts | 58 +++ src/db.ts | 75 +++- src/file-churn.test.ts | 126 ++++++ src/runtime.ts | 12 + src/worker-pool.dist.test.ts | 28 +- templates/agent-content/rule/00-full.md | 1 + .../recipes/churn-complexity-hotspots.md | 33 ++ .../recipes/churn-complexity-hotspots.sql | 66 +++ 46 files changed, 1948 insertions(+), 241 deletions(-) create mode 100644 .changeset/churn-complexity-hotspots.md create mode 100644 fixtures/golden/minimal/churn-complexity-hotspots-by-symbol.json create mode 100644 fixtures/golden/minimal/churn-complexity-hotspots.json create mode 100644 fixtures/minimal/file-churn-seed.json create mode 100644 src/application/churn-ingest.test.ts create mode 100644 src/application/churn-ingest.ts create mode 100644 src/application/ingest-churn-run.test.ts create mode 100644 src/application/ingest-churn-run.ts create mode 100644 src/cli/cmd-ingest-churn.ts create mode 100644 src/file-churn.test.ts create mode 100644 templates/recipes/churn-complexity-hotspots.md create mode 100644 templates/recipes/churn-complexity-hotspots.sql diff --git a/.changeset/churn-complexity-hotspots.md b/.changeset/churn-complexity-hotspots.md new file mode 100644 index 00000000..4609d31c --- /dev/null +++ b/.changeset/churn-complexity-hotspots.md @@ -0,0 +1,5 @@ +--- +"@stainless-code/codemap": minor +--- + +Add churn × complexity hotspot ranking: `file_churn` from git on every index (incremental scoped refresh, idle HEAD cache), `codemap ingest-churn` + `churn.file` for non-git, bundled `churn-complexity-hotspots` recipe with file/symbol grain (`by_symbol`), raw + 0–100 normalized scores, and `churn_trend`. Outcome alias `hotspots` still maps to fan-in. diff --git a/docs/architecture.md b/docs/architecture.md index 611131b3..01581155 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -496,6 +496,23 @@ One row per leaf parameter binding, ordered by `position`. Pattern params (`func | column_start | INTEGER | 0-based column of the binding token | | column_end | INTEGER | One-past-last column | +### `file_churn` — Git churn metrics per indexed file (`STRICT`) + +One row per indexed file with git history in scope. Populated on **every index pass** by `refreshFileChurn` → `ingestFileChurnFromGit` (`git log --numstat` scoped to the project root pathspec). Tunable via config `churn.halfLifeDays` (default 90) and optional `churn.since` / CLI `--churn-since `. Non-git repos skip ingest (empty table; recipe returns no rows). `churn_trend` is `accelerating` \| `stable` \| `cooling` when enough history exists, else NULL. + +| Column | Type | Description | +| ---------------- | ------- | -------------------------------------------------------------------------- | +| file_path | TEXT PK | FK → `files(path)` CASCADE | +| commit_count | INTEGER | Distinct commits touching the file in scope | +| weighted_commits | REAL | Recency-weighted commit count (default 90-day half-life exponential decay) | +| lines_added | INTEGER | Sum of added lines from numstat | +| lines_removed | INTEGER | Sum of removed lines from numstat | +| last_commit_at | TEXT | ISO timestamp of most recent commit touching the file | +| churn_trend | TEXT | `"accelerating"` \| `"stable"` \| `"cooling"` — nullable in v1 | +| computed_at | TEXT | ISO timestamp when ingest last ran | + +Powers **`churn-complexity-hotspots`** recipe (`hotspot_score`, `hotspot_score_normalized`; file or symbol grain via `by_symbol`). Non-git: **`codemap ingest-churn`** or config **`churn.file`**. Distinct from outcome alias **`hotspots`** → `fan-in`. + ### `file_metrics` — Per-file aggregate metrics (`STRICT`) One row per indexed TS/JS file. Line classification is regex-light (blank if `/^\s*$/`; comment if line starts with `//`, `/*`, `*`, `*/`). diff --git a/docs/glossary.md b/docs/glossary.md index c44bee8b..3111ea6e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -260,6 +260,10 @@ Number of edges _into_ a file in the `dependencies` table — `COUNT(*) FROM dep Number of edges _out of_ a file — `COUNT(*) FROM dependencies WHERE from_path = ?`. Surfaces as the `fan-out` recipe. +### `file_churn` (table) + +Per-file git churn metrics refreshed on **every index pass** (incremental scoped recompute; idle HEAD cache). Git via `ingestFileChurnFromGit`; non-git via **`codemap ingest-churn`** or config **`churn.file`**. Config: `churn.halfLifeDays`, `churn.since` / `--churn-since`. Powers **`churn-complexity-hotspots`** (file or symbol grain, normalized score) — distinct from outcome alias **`hotspots`** → `fan-in`. + ### `files` (table) Header row for every indexed file. `path` is the primary key; all other tables FK to it with `ON DELETE CASCADE`. Flags: `is_barrel` (100% re-exports, no local value symbols) and `has_side_effects` (module-level call/assignment seen at parse time). See `FileRow`. @@ -410,7 +414,7 @@ Key-value metadata table. Holds `schema_version`, `last_indexed_commit`, `indexe ### outcome aliases (`dead-code` / `deprecated` / `boundaries` / `hotspots` / `coverage-gaps`) -Top-level CLI verbs that thin-wrap `query --recipe `: `dead-code` → `untested-and-dead`, `deprecated` → `deprecated-symbols`, `boundaries` → `boundary-violations`, `hotspots` → `fan-in`, `coverage-gaps` → `worst-covered-exports`. Every `query` flag passes through (`--json`, `--format`, `--ci`, `--summary`, `--changed-since`, `--group-by`, `--params`, `--save-baseline`, `--baseline`). Mapping lives in `src/cli/aliases.ts` (`OUTCOME_ALIASES`). Capped at 5 to avoid alias-sprawl — promote a sixth only when the recipe becomes a headline outcome. Moat-A clean: the alias is a one-line rewrite, not a new primitive; the recipe IS the SQL. **Write alias (distinct):** `codemap rename` thin-wraps `apply rename-preview` (not `query --recipe`) — mapping in `src/cli/rename-alias.ts`; same Moat-A rule (no new write semantics). +Top-level CLI verbs that thin-wrap `query --recipe `: `dead-code` → `untested-and-dead`, `deprecated` → `deprecated-symbols`, `boundaries` → `boundary-violations`, `hotspots` → `fan-in`, `coverage-gaps` → `worst-covered-exports`. For **change-frequency × complexity** refactor targets use recipe **`churn-complexity-hotspots`** (not the `hotspots` alias). Every `query` flag passes through (`--json`, `--format`, `--ci`, `--summary`, `--changed-since`, `--group-by`, `--params`, `--save-baseline`, `--baseline`). Mapping lives in `src/cli/aliases.ts` (`OUTCOME_ALIASES`). Capped at 5 to avoid alias-sprawl — promote a sixth only when the recipe becomes a headline outcome. Moat-A clean: the alias is a one-line rewrite, not a new primitive; the recipe IS the SQL. **Write alias (distinct):** `codemap rename` thin-wraps `apply rename-preview` (not `query --recipe`) — mapping in `src/cli/rename-alias.ts`; same Moat-A rule (no new write semantics). ### oxc-parser diff --git a/docs/golden-queries.md b/docs/golden-queries.md index 5ced88d6..af9b376c 100644 --- a/docs/golden-queries.md +++ b/docs/golden-queries.md @@ -62,7 +62,7 @@ We **do not** commit another product’s source tree, paths, business strings, o ## Scenario shape (implemented) -Scenarios live in **`fixtures/golden/scenarios.json`** (Tier A) or optional **`scenarios.external.json`** / **example** (Tier B). The file may be a **bare array** of scenarios (legacy) or an object `{ "setup": [...], "scenarios": [...] }`. Optional top-level **`setup`** runs once after index, before scenarios — today **`ingest-coverage`** and **`clear-coverage`** (see [run-setup.ts](../scripts/query-golden/run-setup.ts)); missing coverage files are skipped with a warning. Per-scenario **`preSetup`** runs after global setup (global setup restores after the scenario when `preSetup` mutates the index). Each scenario has **`id`**, **`sql` or `recipe`**, optional **`match`** (`exact`, `minRows`, `everyRowContains`, `everyRowFieldEquals`), optional **`budgetMs`**. Goldens: **`fixtures/golden/minimal/*.json`** etc. Refresh: **`bun scripts/query-golden.ts --update`**. +Scenarios live in **`fixtures/golden/scenarios.json`** (Tier A) or optional **`scenarios.external.json`** / **example** (Tier B). The file may be a **bare array** of scenarios (legacy) or an object `{ "setup": [...], "scenarios": [...] }`. Optional top-level **`setup`** runs once after index, before scenarios — today **`ingest-coverage`**, **`clear-coverage`**, and **`seed-file-churn`** (see [run-setup.ts](../scripts/query-golden/run-setup.ts)); missing coverage files are skipped with a warning. Per-scenario **`preSetup`** runs after global setup (global setup restores after the scenario when `preSetup` mutates the index). Each scenario has **`id`**, **`sql` or `recipe`**, optional **`match`** (`exact`, `minRows`, `everyRowContains`, `everyRowFieldEquals`), optional **`budgetMs`**. Goldens: **`fixtures/golden/minimal/*.json`** etc. Refresh: **`bun scripts/query-golden.ts --update`**. **Prompts** in JSON are **intent labels**, not pasted chat logs — pair with queries whose literals come from **fixture-owned** data (see [fixtures/qa/prompts.external.template.md](../fixtures/qa/prompts.external.template.md) for optional chat QA). diff --git a/docs/plans/churn-complexity-hotspots.md b/docs/plans/churn-complexity-hotspots.md index b3e21d00..abc6f09b 100644 --- a/docs/plans/churn-complexity-hotspots.md +++ b/docs/plans/churn-complexity-hotspots.md @@ -1,140 +1,50 @@ # Churn × complexity hotspots — plan -> **Status:** open · **Priority:** P2 · **Effort:** L–M (~2–3 weeks) -> -> **Motivator:** Agents and maintainers prioritize refactors by **change frequency × structural complexity** — files that churn often _and_ carry heavy symbols are higher-risk touch points than either signal alone. Today `symbols.complexity` exists but git churn is not indexed; `codemap hotspots` alias maps to import **fan-in**, not change×complexity. +> **Status:** shipped (complete) · **Priority:** P2 > > **Roadmap:** [§ Core substrate & platform](../roadmap.md#core-substrate--platform) --- -## Agent start here - -Ship **schema + mocked churn rows + recipe SQL** before wiring real `git log` ingest — proves the JOIN path without git subprocess flakiness in CI. Read `symbols.complexity` and outcome aliases in [`src/cli/aliases.ts`](../../src/cli/aliases.ts); do **not** repoint `hotspots` alias (stays `fan-in`). - -### Key touchpoints - -| File | What to read | -| -------------------------------------------------------------------------- | ------------------------------------------------------------- | -| [`src/db.ts`](../../src/db.ts) | `SCHEMA_VERSION`, table DDL, migrations | -| [`src/application/index-engine.ts`](../../src/application/index-engine.ts) | Full / incremental index hooks (where churn refresh attaches) | -| [`src/extractors/complexity.ts`](../../src/extractors/complexity.ts) | Existing `symbols.complexity` population | -| [`templates/recipes/`](../../templates/recipes/) | Recipe `.sql` + `.md` pair pattern (e.g. `fan-in`) | -| [`src/cli/aliases.ts`](../../src/cli/aliases.ts) | Outcome alias `hotspots` → `fan-in` — leave unchanged | -| [`src/cli/cmd-query.ts`](../../src/cli/cmd-query.ts) | Recipe catalog registration path | - -### Architecture - -```text -index (full or incremental) - → churn-ingest: git log --numstat scoped to indexed files.path - → file_churn rows (weighted_commits, trend, …) -recipe churn-complexity-hotspots - → SQL JOIN file_churn × symbols.complexity → hotspot_score - → query / MCP / HTTP (Moat A — no new verb) -``` - -### Tracer bullet (slice 1) - -1. `file_churn` table + migration. 2. Insert fixture churn rows in test. 3. `churn-complexity-hotspots.sql` returns ranked paths. 4. Wire real git ingest in slice 2. - -### Out of scope (v1) - -Repurposing `codemap hotspots` CLI alias; symbol-level hotspot rows (file-level only unless Q1 resolves otherwise); non-git VCS import (`--churn-file`). - ---- - -## Pre-locked decisions - -| # | Decision | Source | -| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | -| H.1 | **Moat-B substrate** — new `file_churn` table populated at index time (or lazy first-query with cache in `meta`). | [Moat B](../roadmap.md#moats-load-bearing) | -| H.2 | **Moat-A exposure** — bundled recipe `churn-complexity-hotspots` (distinct from outcome alias `hotspots` → `fan-in`). | [Moat A](../roadmap.md#moats-load-bearing); `aliases.ts` | -| H.3 | **Git-native churn v1** — `git log --numstat` (or `git log --format` + diff stat) scoped to indexed `files.path` set; optional `--churn-since ` CLI flag on ingest. | No network; matches incremental git invalidation story | -| H.4 | **Recency weighting** — store `weighted_commits` with exponential half-life (default 90 days) so recent edits rank above ancient history. | Tunable via config `churn.halfLifeDays` | -| H.5 | **Score is a recipe column, not a verdict** — `hotspot_score` computed in SQL JOIN (`file_churn` × aggregated `symbols.complexity` / `file_metrics`); consumer applies `LIMIT`. | Moat A | -| H.6 | **Optional trend column** — `churn_trend: "accelerating" \| "stable" \| "cooling"` from recent-vs-older window ratio; nullable when insufficient history. | v1 nice-to-have; ship score first if schedule tight | - ---- +## Shipped -## Schema sketch - -```sql -CREATE TABLE file_churn ( - file_path TEXT PRIMARY KEY, - commit_count INTEGER NOT NULL, - weighted_commits REAL NOT NULL, - lines_added INTEGER NOT NULL, - lines_removed INTEGER NOT NULL, - last_commit_at TEXT, - churn_trend TEXT, - computed_at TEXT NOT NULL -) STRICT; -``` - -Recipe joins `file_churn` to per-file max/avg `symbols.complexity` (and optionally `file_metrics.line_count`): - -```sql --- sketch only; final SQL lives in templates/recipes/churn-complexity-hotspots.sql -SELECT f.path, - fc.weighted_commits, - MAX(s.complexity) AS max_complexity, - (fc.weighted_commits * MAX(s.complexity)) AS hotspot_score -FROM files f -JOIN file_churn fc ON fc.file_path = f.path -JOIN symbols s ON s.file_path = f.path -WHERE s.complexity IS NOT NULL -GROUP BY f.path -ORDER BY hotspot_score DESC; -``` - -Normalize score to 0–100 in recipe if cross-repo comparability matters (divide by corpus max). - ---- - -## Implementation steps - -1. **Churn ingest module** — `src/application/churn-ingest.ts`: spawn bounded git subprocess; map paths to indexed files only; respect `.gitignore` / codemap excludes. -2. **Index hook** — run churn refresh on full rebuild; incremental path refreshes churn for changed files + ancestors if needed (v1: full churn recompute acceptable on incremental if <2s on medium repos — measure in plan PR). -3. **`file_churn` table** + `SCHEMA_VERSION` bump + migration in `db.ts`. -4. **Recipe** — `churn-complexity-hotspots` with params `limit`, optional `min_complexity`. -5. **CLI** — no new outcome alias (fan-in keeps `hotspots`); document recipe in catalog + `context` intent keywords ("refactor priority", "hotspot", "churn"). -6. **Golden fixture** — synthetic git history in test repo or mocked churn rows. -7. Docs — `architecture.md` schema row; `glossary.md` disambiguate `hotspots` alias vs `churn-complexity-hotspots` recipe. - ---- +| Layer | Delivered | +| ------------ | ---------------------------------------------------------------------------------------------- | +| **Moat B** | `file_churn` table; git ingest every index pass; incremental scoped recompute; idle HEAD cache | +| **Non-git** | `codemap ingest-churn ` + config `churn.file` fallback | +| **Moat A** | Recipe `churn-complexity-hotspots` — file (default) or symbol (`by_symbol=true`) grain | +| **Scores** | `hotspot_score` + `hotspot_score_normalized` (0–100 vs result-set max) | +| **Config** | `churn.halfLifeDays`, `churn.since`, `churn.file`; CLI `--churn-since` | +| **Trend** | `churn_trend`: accelerating \| stable \| cooling | +| **Agent AX** | Rule trigger; `refactor-priority` + `refactor` intent cards; golden + agent-eval | +| **Perf** | `churn_ms` in `--performance` JSON + perf baseline gate | +| **Alias** | `hotspots` → `fan-in` unchanged (Moat-A cap) | ### Verification ```bash -bun test src/application/churn-ingest.test.ts # after module lands -bun src/index.ts query --recipe churn-complexity-hotspots --json -bun src/index.ts query --recipe fan-in --json # alias hotspots unchanged -bun run typecheck # db.ts schema + SymbolRow if touched +bun test src/application/churn-ingest.test.ts src/application/ingest-churn-run.test.ts src/file-churn.test.ts +bun run test:golden +bun src/index.ts ingest-churn fixtures/minimal/file-churn-seed.json +bun src/index.ts query --recipe churn-complexity-hotspots --params by_symbol=true --json ``` --- -## Acceptance - -- [ ] Recipe returns files ranked by churn×complexity on codemap self-index -- [ ] Outcome alias `codemap hotspots` still resolves to `fan-in` -- [ ] Churn ingest skips non-git repos gracefully (empty `file_churn`, recipe returns empty set + stderr hint) -- [ ] No new pass/fail CLI verb - ---- - -## Open decisions (resolve in plan PR) +## Intentionally not shipped -| # | Question | -| --- | -------------------------------------------------------------------------------------------- | -| Q1 | File-level vs symbol-level hotspot rows — v1 file-level only? | -| Q2 | Recompute churn on every incremental index vs explicit `codemap index --refresh-churn` flag? | -| Q3 | Import `--churn-file` JSON for non-git VCS — defer unless consumer asks? | +| Item | Why | +| ------------------------------------- | -------------------------------------------------------- | +| **Repurpose `hotspots` alias** | Moat-A alias cap; recipe is the churn×complexity surface | +| **Cross-repo absolute normalization** | `hotspot_score_normalized` is corpus-relative per query | --- -## Dependencies +## Key touchpoints -- Existing: `symbols.complexity`, `files`, `file_metrics`, git helpers in incremental index -- Independent of [C.9 plugin layer](./c9-plugin-layer.md) +| File | Role | +| ---------------------------------------------------------------------------------------------------------- | --------------------------------------------- | +| [`src/application/churn-ingest.ts`](../../src/application/churn-ingest.ts) | Git ingest, trend, incremental scope, refresh | +| [`src/application/ingest-churn-run.ts`](../../src/application/ingest-churn-run.ts) | JSON import | +| [`src/cli/cmd-ingest-churn.ts`](../../src/cli/cmd-ingest-churn.ts) | `ingest-churn` verb | +| [`templates/recipes/churn-complexity-hotspots.sql`](../../templates/recipes/churn-complexity-hotspots.sql) | Recipe | diff --git a/docs/roadmap.md b/docs/roadmap.md index 37d3733b..e1d7362e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -111,7 +111,7 @@ Predicate-as-API only — enrich row shape and audit deltas; no standalone pass/ - [ ] **`history` table** (deferred — revisit-triggered) — temporal queries: "when did symbol X get `@deprecated`?", "coverage trend over last 50 commits", "files that became dead this week". `audit --base ` covers the most-common temporal question (PR-scoped diff) without schema growth, so the table earns its place only when bigger questions emerge. Two shapes (per-commit snapshots ~N × DB size; append-only event log heavier CTE walks); both pay an N-reindexes backfill cost (~30s per reindex). **Revisit triggers:** two consumers ship `jq`-based "audit-runs-over-time" workflows, OR `query_baselines` evolution becomes a recurring agent need. - [ ] **`codemap audit` verdict + thresholds** (v1.x) — `verdict: "pass" | "warn" | "fail"` driven by an `audit.deltas[].{added_max, action}` field on the config object (`.codemap/config.{ts,js,json}`). Triggers: two consumers ship `jq`-based threshold scripts with similar shapes, OR one consumer asks with a concrete config sketch. Until then, raw deltas + consumer-side `jq` is the CI exit-code idiom. **Likely accelerant:** the Marketplace Action (next item) shipping is the most plausible path to firing the trigger — once `- uses: stainless-code/codemap@v1` is the dominant CI path, real `jq` threshold scripts will surface. - [ ] **GitHub Marketplace Action — publish + listing finish** — core Action implementation is in-tree: root `action.yml`, `query --ci`, `audit --format sarif` / `--ci`, package-manager detection, dogfood smoke, and opt-in `pr-comment` summary renderer have shipped. Remaining work is the release/listing slice: `MARKETPLACE.md`, `v1.0.0` / floating `v1` tags, Marketplace setup, sacrificial-repo smoke, and making `action-smoke` blocking once the Action tag exists. Action version stream is independent of CLI version (`package.json` currently drives CLI/npm version; Action publishes at its own `v1.0.0`). Plan: [`plans/github-marketplace-action.md`](./plans/github-marketplace-action.md). Effort: S. -- [ ] **Churn × complexity hotspots** — `file_churn` table (git `log --numstat` over indexed paths, recency-weighted commits, optional trend) + bundled recipe **`churn-complexity-hotspots`** JOINing `symbols.complexity` for ranked refactor targets. Distinct from outcome alias `hotspots` → `fan-in`. Score is a recipe column, not a verdict ([Moat A](./roadmap.md#moats-load-bearing)). Plan: [`plans/churn-complexity-hotspots.md`](./plans/churn-complexity-hotspots.md). Effort: L–M. +- [x] **Churn × complexity hotspots** — `file_churn` (git ingest every index + `ingest-churn` / `churn.file` fallback), recipe **`churn-complexity-hotspots`** (file/symbol grain, normalized score, trend). Alias `hotspots` → `fan-in`. Plan: [`plans/churn-complexity-hotspots.md`](./plans/churn-complexity-hotspots.md). - [x] **AST-hash duplication** — `symbols.body_hash` (canonical body AST, identifiers → `$id`, literals → kind, absent returns → `Literal:nullish`; function-shaped symbols; skip `body_line_count < 2`) + partial index + bundled `duplicates` recipe (per-symbol rows, CTE `GROUP BY`). **Different shape from token-level suffix-array dupes.** Contract: [architecture § `symbols` table](./architecture.md#symbols--functions-constants-classes-interfaces-types-enums-strict), [glossary § body_hash](./glossary.md#symbolsbody_hash--structural-duplicate-bodies). Effort: M. - [ ] **Falsifiable benchmark CI on named external fixtures** — structural-cost A/B (indexed queries vs `find` + `grep` + `Read`-loop discovery) on zod, fastify, vue-core, next.js. Numbers land in [`docs/benchmark.md`](./benchmark.md); headline figures surface in `MARKETPLACE.md` only after external runs land. Harness: [benchmark § Agent eval harness](./benchmark.md#agent-eval-harness) + external fixture extension; pair with **Agent eval: quality × tokens × wall** for scored completion metrics. **Partial:** manual [`.github/workflows/agent-eval-external.yml`](../.github/workflows/agent-eval-external.yml) for in-repo fixture paths (not zod/fastify/nightly). Effort: M. **Self-index regression guardrail shipped** (#96): `bun run check:perf-baseline` + weekly scheduled workflow (demoted from PR hard gate — GHA runner variance). - [ ] **In-repo test bench scale (optional)** — if `fixtures/minimal` outgrows one corpus: add committed `fixtures/bench/` or rename `minimal`→`bench`. Harness map: [`testing-coverage.md`](./testing-coverage.md), [`fixtures/README.md`](../fixtures/README.md). diff --git a/fixtures/CAPABILITIES.json b/fixtures/CAPABILITIES.json index cbebddd7..d512df8d 100644 --- a/fixtures/CAPABILITIES.json +++ b/fixtures/CAPABILITIES.json @@ -184,6 +184,20 @@ ], "goldenScenarios": ["duplicates"] }, + { + "id": "churn.file-churn", + "description": "file_churn git metrics and churn-complexity-hotspots recipe", + "fixtureFiles": [ + "src/lib/complexity-fixture.ts", + "src/lib/cache.ts", + "src/api/client.ts", + "file-churn-seed.json" + ], + "goldenScenarios": [ + "churn-complexity-hotspots", + "churn-complexity-hotspots-by-symbol" + ] + }, { "id": "boundaries.suppressions", "description": "boundary_rules, suppressions, config-driven violations", diff --git a/fixtures/benchmark/perf-baseline.json b/fixtures/benchmark/perf-baseline.json index f420cdac..02ca01bf 100644 --- a/fixtures/benchmark/perf-baseline.json +++ b/fixtures/benchmark/perf-baseline.json @@ -1,13 +1,14 @@ { - "captured_at": "2026-05-25T16:09:06.000Z", - "commit": "3911abf", + "captured_at": "2026-06-10T13:38:17.300Z", + "commit": "36106ff66b691bfbe682da5e5cb29a1779e24a1e", "phases": { - "collect_ms": 18, - "parse_ms": 559, - "insert_ms": 231, - "index_create_ms": 137, - "bindings_ms": 153, - "total_ms": 1117 + "collect_ms": 14, + "parse_ms": 299, + "insert_ms": 282, + "index_create_ms": 149, + "bindings_ms": 174, + "churn_ms": 182, + "total_ms": 1194 }, "regression_pct": 25, "noise_floor_ms": 10 diff --git a/fixtures/golden/minimal/churn-complexity-hotspots-by-symbol.json b/fixtures/golden/minimal/churn-complexity-hotspots-by-symbol.json new file mode 100644 index 00000000..069d5dbb --- /dev/null +++ b/fixtures/golden/minimal/churn-complexity-hotspots-by-symbol.json @@ -0,0 +1,132 @@ +[ + { + "file_path": "src/lib/complexity-fixture.ts", + "symbol_name": "labyrinth", + "symbol_kind": "function", + "line_start": 22, + "weighted_commits": 22.5, + "commit_count": 28, + "churn_trend": "accelerating", + "max_complexity": 19, + "avg_complexity": 19, + "hotspot_score": 427.5, + "hotspot_score_normalized": 100 + }, + { + "file_path": "src/lib/complexity-fixture.ts", + "symbol_name": "deeplyNested", + "symbol_kind": "function", + "line_start": 6, + "weighted_commits": 22.5, + "commit_count": 28, + "churn_trend": "accelerating", + "max_complexity": 5, + "avg_complexity": 5, + "hotspot_score": 112.5, + "hotspot_score_normalized": 26.3 + }, + { + "file_path": "src/lib/complexity-fixture.ts", + "symbol_name": "catchDirectRethrow", + "symbol_kind": "function", + "line_start": 98, + "weighted_commits": 22.5, + "commit_count": 28, + "churn_trend": "accelerating", + "max_complexity": 2, + "avg_complexity": 2, + "hotspot_score": 45, + "hotspot_score_normalized": 10.5 + }, + { + "file_path": "src/lib/complexity-fixture.ts", + "symbol_name": "catchInnerArrowRethrow", + "symbol_kind": "function", + "line_start": 86, + "weighted_commits": 22.5, + "commit_count": 28, + "churn_trend": "accelerating", + "max_complexity": 2, + "avg_complexity": 2, + "hotspot_score": 45, + "hotspot_score_normalized": 10.5 + }, + { + "file_path": "src/lib/complexity-fixture.ts", + "symbol_name": "relay", + "symbol_kind": "function", + "line_start": 90, + "weighted_commits": 22.5, + "commit_count": 28, + "churn_trend": "accelerating", + "max_complexity": 1, + "avg_complexity": 1, + "hotspot_score": 22.5, + "hotspot_score_normalized": 5.3 + }, + { + "file_path": "src/api/client.ts", + "symbol_name": "createClient", + "symbol_kind": "function", + "line_start": 20, + "weighted_commits": 8, + "commit_count": 14, + "churn_trend": "cooling", + "max_complexity": 2, + "avg_complexity": 2, + "hotspot_score": 16, + "hotspot_score_normalized": 3.7 + }, + { + "file_path": "src/lib/cache.ts", + "symbol_name": "get", + "symbol_kind": "function", + "line_start": 8, + "weighted_commits": 4.2, + "commit_count": 9, + "churn_trend": "stable", + "max_complexity": 3, + "avg_complexity": 3, + "hotspot_score": 12.6, + "hotspot_score_normalized": 2.9 + }, + { + "file_path": "src/lib/cache.ts", + "symbol_name": "invalidate", + "symbol_kind": "function", + "line_start": 16, + "weighted_commits": 4.2, + "commit_count": 9, + "churn_trend": "stable", + "max_complexity": 3, + "avg_complexity": 3, + "hotspot_score": 12.6, + "hotspot_score_normalized": 2.9 + }, + { + "file_path": "src/api/client.ts", + "symbol_name": "handshake", + "symbol_kind": "function", + "line_start": 37, + "weighted_commits": 8, + "commit_count": 14, + "churn_trend": "cooling", + "max_complexity": 1, + "avg_complexity": 1, + "hotspot_score": 8, + "hotspot_score_normalized": 1.9 + }, + { + "file_path": "src/api/client.ts", + "symbol_name": "legacyClient", + "symbol_kind": "function", + "line_start": 46, + "weighted_commits": 8, + "commit_count": 14, + "churn_trend": "cooling", + "max_complexity": 1, + "avg_complexity": 1, + "hotspot_score": 8, + "hotspot_score_normalized": 1.9 + } +] diff --git a/fixtures/golden/minimal/churn-complexity-hotspots.json b/fixtures/golden/minimal/churn-complexity-hotspots.json new file mode 100644 index 00000000..d359d4c2 --- /dev/null +++ b/fixtures/golden/minimal/churn-complexity-hotspots.json @@ -0,0 +1,41 @@ +[ + { + "file_path": "src/lib/complexity-fixture.ts", + "symbol_name": null, + "symbol_kind": null, + "line_start": null, + "weighted_commits": 22.5, + "commit_count": 28, + "churn_trend": "accelerating", + "max_complexity": 19, + "avg_complexity": 5.8, + "hotspot_score": 427.5, + "hotspot_score_normalized": 100 + }, + { + "file_path": "src/api/client.ts", + "symbol_name": null, + "symbol_kind": null, + "line_start": null, + "weighted_commits": 8, + "commit_count": 14, + "churn_trend": "cooling", + "max_complexity": 2, + "avg_complexity": 1.2, + "hotspot_score": 16, + "hotspot_score_normalized": 3.7 + }, + { + "file_path": "src/lib/cache.ts", + "symbol_name": null, + "symbol_kind": null, + "line_start": null, + "weighted_commits": 4.2, + "commit_count": 9, + "churn_trend": "stable", + "max_complexity": 3, + "avg_complexity": 3, + "hotspot_score": 12.6, + "hotspot_score_normalized": 2.9 + } +] diff --git a/fixtures/golden/minimal/files-count.json b/fixtures/golden/minimal/files-count.json index c68f12eb..6ae0eac1 100644 --- a/fixtures/golden/minimal/files-count.json +++ b/fixtures/golden/minimal/files-count.json @@ -1,5 +1,5 @@ [ { - "n": 45 + "n": 46 } ] diff --git a/fixtures/golden/minimal/files-hashes.json b/fixtures/golden/minimal/files-hashes.json index f181d16f..8c5d6bb0 100644 --- a/fixtures/golden/minimal/files-hashes.json +++ b/fixtures/golden/minimal/files-hashes.json @@ -17,6 +17,12 @@ "language": "md", "line_count": 60 }, + { + "path": "file-churn-seed.json", + "content_hash": "9f5241c6e2dc081b0398228178dd10dd6e37f9efc0ef2157e7d417a81b7ffd0b", + "language": "json", + "line_count": 33 + }, { "path": "package.json", "content_hash": "a8e79efc943697cee4d4435360de9150bd7261376abe9e2c34c0c7238177c7de", diff --git a/fixtures/golden/minimal/files-largest.json b/fixtures/golden/minimal/files-largest.json index 2797e814..c44a7998 100644 --- a/fixtures/golden/minimal/files-largest.json +++ b/fixtures/golden/minimal/files-largest.json @@ -23,6 +23,12 @@ "size": 1068, "language": "ts" }, + { + "path": "file-churn-seed.json", + "line_count": 33, + "size": 819, + "language": "json" + }, { "path": "src/utils/date.ts", "line_count": 29, @@ -112,11 +118,5 @@ "line_count": 11, "size": 418, "language": "md" - }, - { - "path": "src/bench/jsx-synthesis/PageShell.tsx", - "line_count": 10, - "size": 153, - "language": "tsx" } ] diff --git a/fixtures/golden/minimal/index-summary.json b/fixtures/golden/minimal/index-summary.json index fee69116..cbb12431 100644 --- a/fixtures/golden/minimal/index-summary.json +++ b/fixtures/golden/minimal/index-summary.json @@ -1,6 +1,6 @@ [ { - "files": 45, + "files": 46, "symbols": 110, "imports": 26, "components": 5, diff --git a/fixtures/golden/minimal/index-table-stats.json b/fixtures/golden/minimal/index-table-stats.json index b618b394..a9858745 100644 --- a/fixtures/golden/minimal/index-table-stats.json +++ b/fixtures/golden/minimal/index-table-stats.json @@ -1,6 +1,6 @@ [ { - "files": 45, + "files": 46, "symbols": 110, "imports": 26, "exports": 64, @@ -32,6 +32,7 @@ "jsdoc_tags": 10, "jsx_attributes": 10, "boundary_rules": 1, - "suppressions": 1 + "suppressions": 1, + "file_churn": 3 } ] diff --git a/fixtures/golden/minimal/source-fts-row-count.json b/fixtures/golden/minimal/source-fts-row-count.json index c68f12eb..6ae0eac1 100644 --- a/fixtures/golden/minimal/source-fts-row-count.json +++ b/fixtures/golden/minimal/source-fts-row-count.json @@ -1,5 +1,5 @@ [ { - "n": 45 + "n": 46 } ] diff --git a/fixtures/golden/scenarios.json b/fixtures/golden/scenarios.json index bbd6d341..60168146 100644 --- a/fixtures/golden/scenarios.json +++ b/fixtures/golden/scenarios.json @@ -462,6 +462,23 @@ "prompt": "Function-shaped symbols with identical structural body_hash across files.", "recipe": "duplicates" }, + { + "id": "churn-complexity-hotspots", + "prompt": "Files ranked by git churn × cyclomatic complexity (hotspot_score).", + "recipe": "churn-complexity-hotspots", + "preSetup": [ + { "kind": "seed-file-churn", "path": "file-churn-seed.json" } + ] + }, + { + "id": "churn-complexity-hotspots-by-symbol", + "prompt": "Symbols ranked by file churn × per-symbol complexity.", + "recipe": "churn-complexity-hotspots", + "params": { "by_symbol": true, "row_limit": 10 }, + "preSetup": [ + { "kind": "seed-file-churn", "path": "file-churn-seed.json" } + ] + }, { "id": "circular-imports", "prompt": "Files in import cycles (SCCs of size >= 2) via Tarjan.", @@ -653,7 +670,7 @@ { "id": "index-table-stats", "prompt": "Row counts for all indexed substrate tables (mirrors fetchTableStats + post-pass tables)", - "sql": "SELECT (SELECT COUNT(*) FROM files) AS files, (SELECT COUNT(*) FROM symbols) AS symbols, (SELECT COUNT(*) FROM imports) AS imports, (SELECT COUNT(*) FROM exports) AS exports, (SELECT COUNT(*) FROM components) AS components, (SELECT COUNT(*) FROM dependencies) AS dependencies, (SELECT COUNT(*) FROM markers) AS markers, (SELECT COUNT(*) FROM type_members) AS type_members, (SELECT COUNT(*) FROM type_heritage) AS type_heritage, (SELECT COUNT(*) FROM calls) AS calls, (SELECT COUNT(*) FROM css_variables) AS css_vars, (SELECT COUNT(*) FROM css_classes) AS css_classes, (SELECT COUNT(*) FROM css_keyframes) AS css_keyframes, (SELECT COUNT(*) FROM scopes) AS scopes, (SELECT COUNT(*) FROM \"references\") AS ref_count, (SELECT COUNT(*) FROM bindings) AS bindings, (SELECT COUNT(*) FROM import_specifiers) AS import_specifiers, (SELECT COUNT(*) FROM function_params) AS function_params, (SELECT COUNT(*) FROM runtime_markers) AS runtime_markers, (SELECT COUNT(*) FROM test_suites) AS test_suites, (SELECT COUNT(*) FROM re_export_chains) AS re_export_chains, (SELECT COUNT(*) FROM module_cycles) AS module_cycles, (SELECT COUNT(*) FROM dynamic_imports) AS dynamic_imports, (SELECT COUNT(*) FROM file_metrics) AS file_metrics, (SELECT COUNT(*) FROM unresolved_calls) AS unresolved_calls, (SELECT COUNT(*) FROM jsx_elements) AS jsx_elements, (SELECT COUNT(*) FROM async_calls) AS async_calls, (SELECT COUNT(*) FROM decorators) AS decorators, (SELECT COUNT(*) FROM try_catch) AS try_catch, (SELECT COUNT(*) FROM jsdoc_tags) AS jsdoc_tags, (SELECT COUNT(*) FROM jsx_attributes) AS jsx_attributes, (SELECT COUNT(*) FROM boundary_rules) AS boundary_rules, (SELECT COUNT(*) FROM suppressions) AS suppressions" + "sql": "SELECT (SELECT COUNT(*) FROM files) AS files, (SELECT COUNT(*) FROM symbols) AS symbols, (SELECT COUNT(*) FROM imports) AS imports, (SELECT COUNT(*) FROM exports) AS exports, (SELECT COUNT(*) FROM components) AS components, (SELECT COUNT(*) FROM dependencies) AS dependencies, (SELECT COUNT(*) FROM markers) AS markers, (SELECT COUNT(*) FROM type_members) AS type_members, (SELECT COUNT(*) FROM type_heritage) AS type_heritage, (SELECT COUNT(*) FROM calls) AS calls, (SELECT COUNT(*) FROM css_variables) AS css_vars, (SELECT COUNT(*) FROM css_classes) AS css_classes, (SELECT COUNT(*) FROM css_keyframes) AS css_keyframes, (SELECT COUNT(*) FROM scopes) AS scopes, (SELECT COUNT(*) FROM \"references\") AS ref_count, (SELECT COUNT(*) FROM bindings) AS bindings, (SELECT COUNT(*) FROM import_specifiers) AS import_specifiers, (SELECT COUNT(*) FROM function_params) AS function_params, (SELECT COUNT(*) FROM runtime_markers) AS runtime_markers, (SELECT COUNT(*) FROM test_suites) AS test_suites, (SELECT COUNT(*) FROM re_export_chains) AS re_export_chains, (SELECT COUNT(*) FROM module_cycles) AS module_cycles, (SELECT COUNT(*) FROM dynamic_imports) AS dynamic_imports, (SELECT COUNT(*) FROM file_metrics) AS file_metrics, (SELECT COUNT(*) FROM unresolved_calls) AS unresolved_calls, (SELECT COUNT(*) FROM jsx_elements) AS jsx_elements, (SELECT COUNT(*) FROM async_calls) AS async_calls, (SELECT COUNT(*) FROM decorators) AS decorators, (SELECT COUNT(*) FROM try_catch) AS try_catch, (SELECT COUNT(*) FROM jsdoc_tags) AS jsdoc_tags, (SELECT COUNT(*) FROM jsx_attributes) AS jsx_attributes, (SELECT COUNT(*) FROM boundary_rules) AS boundary_rules, (SELECT COUNT(*) FROM suppressions) AS suppressions, (SELECT COUNT(*) FROM file_churn) AS file_churn" }, { "id": "meta-fts5-enabled", diff --git a/fixtures/minimal/file-churn-seed.json b/fixtures/minimal/file-churn-seed.json new file mode 100644 index 00000000..c0cd9eab --- /dev/null +++ b/fixtures/minimal/file-churn-seed.json @@ -0,0 +1,32 @@ +[ + { + "file_path": "src/lib/complexity-fixture.ts", + "commit_count": 28, + "weighted_commits": 22.5, + "lines_added": 520, + "lines_removed": 180, + "last_commit_at": "2026-06-08T12:00:00Z", + "churn_trend": "accelerating", + "computed_at": "2026-06-10T00:00:00Z" + }, + { + "file_path": "src/lib/cache.ts", + "commit_count": 9, + "weighted_commits": 4.2, + "lines_added": 95, + "lines_removed": 40, + "last_commit_at": "2026-05-15T09:00:00Z", + "churn_trend": "stable", + "computed_at": "2026-06-10T00:00:00Z" + }, + { + "file_path": "src/api/client.ts", + "commit_count": 14, + "weighted_commits": 8.0, + "lines_added": 210, + "lines_removed": 90, + "last_commit_at": "2026-06-01T00:00:00Z", + "churn_trend": "cooling", + "computed_at": "2026-06-10T00:00:00Z" + } +] diff --git a/scripts/agent-eval/scenarios.json b/scripts/agent-eval/scenarios.json index c11eff39..8c7e6602 100644 --- a/scripts/agent-eval/scenarios.json +++ b/scripts/agent-eval/scenarios.json @@ -171,6 +171,15 @@ "regex": "export\\s+function\\s+\\w+", "mode": "files" } + }, + { + "id": "churn-complexity-hotspots-recipe", + "goldenId": "churn-complexity-hotspots", + "traditional": { + "globs": ["src/lib/**/*.ts", "src/api/**/*.ts"], + "regex": "export\\s+(function|const)\\s+\\w+", + "mode": "files" + } } ] } diff --git a/scripts/check-perf-baseline.ts b/scripts/check-perf-baseline.ts index 40cf282a..c04e1969 100644 --- a/scripts/check-perf-baseline.ts +++ b/scripts/check-perf-baseline.ts @@ -31,6 +31,7 @@ type Phase = | "bindings_ms" | "module_cycles_ms" | "re_export_chains_ms" + | "churn_ms" | "total_ms"; const GATED_PHASES: Phase[] = [ @@ -39,6 +40,7 @@ const GATED_PHASES: Phase[] = [ "insert_ms", "index_create_ms", "bindings_ms", + "churn_ms", "total_ms", ]; diff --git a/scripts/query-golden-coverage-matrix.test.mjs b/scripts/query-golden-coverage-matrix.test.mjs index 5570da6c..d4ac2128 100644 --- a/scripts/query-golden-coverage-matrix.test.mjs +++ b/scripts/query-golden-coverage-matrix.test.mjs @@ -15,6 +15,7 @@ const GOLDEN_DIR = join(REPO_ROOT, "fixtures/golden/minimal"); /** Tables that must have a dedicated SQL pin-down scenario (not recipe-only). */ const SUBSTRATE_SCENARIO_BY_TABLE = { file_metrics: "file-metrics-complexity-fixture", + file_churn: "churn-complexity-hotspots", scopes: "scopes-product-card", references: "references-product-card-perms", bindings: "bindings-createClient", diff --git a/scripts/query-golden/run-setup.ts b/scripts/query-golden/run-setup.ts index fa1ee497..b4fef0bf 100644 --- a/scripts/query-golden/run-setup.ts +++ b/scripts/query-golden/run-setup.ts @@ -5,7 +5,8 @@ import { ingestIstanbul, ingestLcov, } from "../../src/application/coverage-engine"; -import { closeDb, openDb } from "../../src/db"; +import { closeDb, openDb, replaceFileChurn } from "../../src/db"; +import type { FileChurnRow } from "../../src/db"; import type { GoldenSetupStep } from "./schema"; /** @@ -26,6 +27,27 @@ export function runGoldenSetup( db.run("DELETE FROM coverage"); continue; } + if (step.kind === "seed-file-churn") { + const absPath = resolve(fixtureRoot, step.path); + if ( + absPath !== fixtureAbs && + !absPath.startsWith(`${fixtureAbs}${sep}`) + ) { + throw new Error( + `query-golden setup: path must stay under fixture root (${step.path})`, + ); + } + if (!existsSync(absPath)) { + throw new Error( + `query-golden setup: missing file-churn seed ${absPath}`, + ); + } + const rows = JSON.parse( + readFileSync(absPath, "utf-8"), + ) as FileChurnRow[]; + replaceFileChurn(db, rows); + continue; + } if (step.kind !== "ingest-coverage") continue; const absPath = resolve(fixtureRoot, step.path); if ( diff --git a/scripts/query-golden/schema.ts b/scripts/query-golden/schema.ts index 49eaeb21..d457cca4 100644 --- a/scripts/query-golden/schema.ts +++ b/scripts/query-golden/schema.ts @@ -41,6 +41,11 @@ export const setupStepSchema = z.discriminatedUnion("kind", [ z.object({ kind: z.literal("clear-coverage"), }), + z.object({ + kind: z.literal("seed-file-churn"), + /** Path relative to fixture root (JSON array of FileChurnRow). */ + path: z.string().min(1), + }), ]); export type GoldenSetupStep = z.infer; diff --git a/src/application/churn-ingest.test.ts b/src/application/churn-ingest.test.ts new file mode 100644 index 00000000..0bb4f26e --- /dev/null +++ b/src/application/churn-ingest.test.ts @@ -0,0 +1,222 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { closeDb, createSchema, insertFile } from "../db"; +import { openCodemapDatabase } from "../sqlite-db"; +import { computeChurnTrend, ingestFileChurnFromGit } from "./churn-ingest"; + +let projectRoot: string; + +function fixtureEnv(dates: { + author: string; + committer: string; +}): NodeJS.ProcessEnv { + const e: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(process.env)) { + if (k.startsWith("GIT_") || k.startsWith("HUSKY")) continue; + e[k] = v; + } + e.GIT_AUTHOR_DATE = dates.author; + e.GIT_COMMITTER_DATE = dates.committer; + return e; +} + +function git(args: string[], env?: NodeJS.ProcessEnv): void { + const r = spawnSync("git", args, { + cwd: projectRoot, + env: + env ?? + fixtureEnv({ + author: "2026-06-01T12:00:00Z", + committer: "2026-06-01T12:00:00Z", + }), + }); + if (r.status !== 0) { + throw new Error(`git ${args.join(" ")}: ${r.stderr.toString().trim()}`); + } +} + +function commitAll(message: string, env?: NodeJS.ProcessEnv): void { + git(["add", "."], env); + git(["commit", "-m", message, "--no-gpg-sign"], env); +} + +describe("computeChurnTrend", () => { + it("returns null when commit_count is below threshold", () => { + expect( + computeChurnTrend({ + commit_count: 3, + recent_weighted: 2, + older_weighted: 1, + }), + ).toBeNull(); + }); + + it("classifies accelerating when recent mass dominates", () => { + expect( + computeChurnTrend({ + commit_count: 8, + recent_weighted: 7, + older_weighted: 2, + }), + ).toBe("accelerating"); + }); + + it("classifies cooling when older mass dominates", () => { + expect( + computeChurnTrend({ + commit_count: 10, + recent_weighted: 1, + older_weighted: 9, + }), + ).toBe("cooling"); + }); +}); + +describe("ingestFileChurnFromGit", () => { + beforeEach(() => { + projectRoot = mkdtempSync(join(tmpdir(), "codemap-churn-")); + git(["init", "-q", "-b", "main"]); + git(["config", "user.email", "t@example.com"]); + git(["config", "user.name", "T"]); + git(["config", "commit.gpgsign", "false"]); + }); + + afterEach(() => { + rmSync(projectRoot, { recursive: true, force: true }); + }); + + it("populates file_churn for indexed paths from git history", () => { + mkdirSync(join(projectRoot, "src"), { recursive: true }); + writeFileSync(join(projectRoot, "src/hot.ts"), "export const a = 1;\n"); + commitAll("add hot"); + writeFileSync(join(projectRoot, "src/hot.ts"), "export const a = 2;\n"); + commitAll("edit hot"); + + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/hot.ts", + content_hash: "h", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + + const result = ingestFileChurnFromGit(db, { + projectRoot, + quiet: true, + }); + expect(result.ok).toBe(true); + expect(result.rowCount).toBe(1); + expect(result.elapsedMs).toBeGreaterThanOrEqual(0); + + const row = db + .query<{ + file_path: string; + commit_count: number; + weighted_commits: number; + }>( + "SELECT file_path, commit_count, weighted_commits FROM file_churn WHERE file_path = ?", + ) + .get("src/hot.ts"); + expect(row?.commit_count).toBe(2); + expect(row?.weighted_commits).toBeGreaterThan(0); + } finally { + closeDb(db); + } + }); + + it("respects since ref (only commits after anchor)", () => { + mkdirSync(join(projectRoot, "src"), { recursive: true }); + writeFileSync(join(projectRoot, "src/a.ts"), "v1\n"); + commitAll( + "old", + fixtureEnv({ + author: "2024-01-01T12:00:00Z", + committer: "2024-01-01T12:00:00Z", + }), + ); + const anchor = spawnSync("git", ["rev-parse", "HEAD"], { + cwd: projectRoot, + encoding: "utf-8", + }) + .stdout.toString() + .trim(); + writeFileSync(join(projectRoot, "src/a.ts"), "v2\n"); + commitAll( + "new", + fixtureEnv({ + author: "2026-06-01T12:00:00Z", + committer: "2026-06-01T12:00:00Z", + }), + ); + + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/a.ts", + content_hash: "a", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + const result = ingestFileChurnFromGit(db, { + projectRoot, + since: anchor, + quiet: true, + }); + expect(result.ok).toBe(true); + const row = db + .query<{ commit_count: number }>( + "SELECT commit_count FROM file_churn WHERE file_path = 'src/a.ts'", + ) + .get(); + expect(row?.commit_count).toBe(1); + } finally { + closeDb(db); + } + }); + + it("skips gracefully when not a git repository", () => { + const noGit = mkdtempSync(join(tmpdir(), "codemap-no-git-")); + try { + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/x.ts", + content_hash: "x", + size: 1, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + const result = ingestFileChurnFromGit(db, { + projectRoot: noGit, + quiet: true, + }); + expect(result.ok).toBe(false); + expect(result.reason).toContain("not a git"); + const n = db + .query<{ c: number }>("SELECT COUNT(*) AS c FROM file_churn") + .get()?.c; + expect(n).toBe(0); + } finally { + closeDb(db); + } + } finally { + rmSync(noGit, { recursive: true, force: true }); + } + }); +}); diff --git a/src/application/churn-ingest.ts b/src/application/churn-ingest.ts new file mode 100644 index 00000000..ee4df153 --- /dev/null +++ b/src/application/churn-ingest.ts @@ -0,0 +1,420 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, realpathSync } from "node:fs"; +import { relative, resolve } from "node:path"; + +import { + getMeta, + mergeFileChurnForPaths, + META_CHURN_INDEXED_COMMIT, + pruneFileChurnOrphans, + replaceFileChurn, + setMeta, +} from "../db"; +import type { FileChurnRow } from "../db"; +import { + getChurnFilePath, + getChurnHalfLifeDays, + getChurnSince, + getProjectRoot, +} from "../runtime"; +import type { CodemapDatabase } from "../sqlite-db"; +import { ingestChurnFromConfigPath } from "./ingest-churn-run"; + +export const DEFAULT_CHURN_HALF_LIFE_DAYS = 90; +const MIN_COMMITS_FOR_TREND = 4; +const TREND_ACCELERATING_RATIO = 0.6; +const TREND_COOLING_RATIO = 0.4; + +/** Strip inherited GIT_* so subprocess targets the resolved repo root. */ +function gitSpawnEnv(): NodeJS.ProcessEnv { + const e: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(process.env)) { + if (k.startsWith("GIT_")) continue; + e[k] = v; + } + return e; +} + +function gitTopLevel(projectRoot: string): string | null { + const r = spawnSync("git", ["rev-parse", "--show-toplevel"], { + cwd: projectRoot, + env: gitSpawnEnv(), + }); + if (r.status !== 0) return null; + return r.stdout.toString().trim(); +} + +function toPosix(p: string): string { + return p.split("\\").join("/"); +} + +function resolveRealPath(path: string): string { + try { + return realpathSync(path); + } catch { + return resolve(path); + } +} + +/** Map a git path (repo-relative) to an indexed `files.path` under `projectRoot`. */ +function indexedPathFromGit( + gitPath: string, + projectPrefix: string, +): string | null { + const p = toPosix(gitPath); + if (projectPrefix === "") return p; + const prefix = `${projectPrefix}/`; + if (!p.startsWith(prefix)) return null; + return p.slice(prefix.length); +} + +function indexedToGitPath(filePath: string, projectPrefix: string): string { + if (projectPrefix === "" || projectPrefix === ".") return filePath; + return `${projectPrefix}/${filePath}`; +} + +export type ChurnTrend = "accelerating" | "stable" | "cooling"; + +export type ChurnRefreshMode = "full" | "incremental" | "idle"; + +export interface ChurnIngestResult { + ok: boolean; + rowCount: number; + elapsedMs: number; + /** User/agent-readable skip reason (non-git, git error, idle cache hit). */ + reason?: string; +} + +interface ChurnAcc { + commit_count: number; + weighted_commits: number; + recent_weighted: number; + older_weighted: number; + lines_added: number; + lines_removed: number; + last_commit_at: string | null; + last_commit_ts: number; +} + +/** + * Classify churn trend from recency-split weighted commit mass. + * Recent window = commits within `halfLifeDays / 2`; older = the rest. + */ +export function computeChurnTrend( + acc: Pick, +): ChurnTrend | null { + if (acc.commit_count < MIN_COMMITS_FOR_TREND) return null; + const total = acc.recent_weighted + acc.older_weighted; + if (total <= 0) return null; + const ratio = acc.recent_weighted / total; + if (ratio >= TREND_ACCELERATING_RATIO) return "accelerating"; + if (ratio <= TREND_COOLING_RATIO) return "cooling"; + return "stable"; +} + +function accsToRows( + byFile: Map, + computedAt: string, +): FileChurnRow[] { + return [...byFile.entries()].map(([file_path, a]) => ({ + file_path, + commit_count: a.commit_count, + weighted_commits: Math.round(a.weighted_commits * 1000) / 1000, + lines_added: a.lines_added, + lines_removed: a.lines_removed, + last_commit_at: a.last_commit_at, + churn_trend: computeChurnTrend(a), + computed_at: computedAt, + })); +} + +function parseGitNumstatLog( + stdout: string, + options: { + indexedPaths: Set; + projectPrefix: string; + halfLife: number; + scopeFilter?: Set; + }, +): Map { + const nowSec = Math.floor(Date.now() / 1000); + const recentWindowDays = options.halfLife / 2; + const byFile = new Map(); + let commitTs = 0; + + for (const line of stdout.split("\n")) { + if (line.startsWith("COMMIT ")) { + const parts = line.split(" "); + commitTs = Number(parts[2] ?? 0); + continue; + } + if (!line.trim() || commitTs <= 0) continue; + const tab = line.indexOf("\t"); + if (tab < 0) continue; + const rest = line.slice(tab + 1); + const tab2 = rest.indexOf("\t"); + if (tab2 < 0) continue; + const addedRaw = line.slice(0, tab); + const removedRaw = rest.slice(0, tab2); + const gitPath = rest.slice(tab2 + 1); + if (addedRaw === "-" || removedRaw === "-") continue; + + const filePath = indexedPathFromGit(gitPath, options.projectPrefix); + if (!filePath || !options.indexedPaths.has(filePath)) continue; + if (options.scopeFilter && !options.scopeFilter.has(filePath)) continue; + + const added = Number(addedRaw) || 0; + const removed = Number(removedRaw) || 0; + const ageDays = Math.max(0, (nowSec - commitTs) / 86_400); + const weight = 0.5 ** (ageDays / options.halfLife); + + let acc = byFile.get(filePath); + if (!acc) { + acc = { + commit_count: 0, + weighted_commits: 0, + recent_weighted: 0, + older_weighted: 0, + lines_added: 0, + lines_removed: 0, + last_commit_at: null, + last_commit_ts: 0, + }; + byFile.set(filePath, acc); + } + acc.commit_count += 1; + acc.weighted_commits += weight; + if (ageDays <= recentWindowDays) { + acc.recent_weighted += weight; + } else { + acc.older_weighted += weight; + } + acc.lines_added += added; + acc.lines_removed += removed; + if (commitTs >= acc.last_commit_ts) { + acc.last_commit_ts = commitTs; + acc.last_commit_at = new Date(commitTs * 1000).toISOString(); + } + } + return byFile; +} + +function countFileChurn(db: CodemapDatabase): number { + return ( + db.query<{ n: number }>("SELECT COUNT(*) AS n FROM file_churn").get()?.n ?? + 0 + ); +} + +function resolveGitHead(projectRoot: string): string | null { + const r = spawnSync("git", ["rev-parse", "HEAD"], { + cwd: projectRoot, + env: gitSpawnEnv(), + }); + if (r.status !== 0) return null; + const head = r.stdout.toString().trim(); + return head.length > 0 ? head : null; +} + +function stampChurnCommit(db: CodemapDatabase, projectRoot: string): void { + const head = resolveGitHead(projectRoot); + if (head) setMeta(db, META_CHURN_INDEXED_COMMIT, head); +} + +function tryConfigChurnFallback( + db: CodemapDatabase, + projectRoot: string, + quiet: boolean, + finish: (partial: Omit) => ChurnIngestResult, +): ChurnIngestResult | null { + let churnFile: string | null = null; + try { + churnFile = getChurnFilePath(); + } catch { + return null; + } + if (!churnFile) return null; + const loaded = ingestChurnFromConfigPath(db, { + projectRoot, + churnFile, + }); + if (!loaded || !loaded.ok) return null; + if (!quiet) { + console.error( + `[churn] file_churn loaded from config churn.file: ${loaded.ingested} files`, + ); + } + return finish({ ok: true, rowCount: loaded.ingested }); +} + +/** + * Populate `file_churn` from `git log --numstat` scoped to indexed paths. + */ +export function ingestFileChurnFromGit( + db: CodemapDatabase, + options: { + projectRoot: string; + halfLifeDays?: number; + since?: string | null; + quiet?: boolean; + /** When set, only these indexed paths are recomputed (merge, not full replace). */ + scopePaths?: string[]; + }, +): ChurnIngestResult { + const t0 = performance.now(); + const projectRoot = resolveRealPath(resolve(options.projectRoot)); + const halfLife = options.halfLifeDays ?? DEFAULT_CHURN_HALF_LIFE_DAYS; + const since = options.since?.trim() || null; + const quiet = options.quiet ?? false; + const scopePaths = options.scopePaths; + const merge = scopePaths !== undefined && scopePaths.length > 0; + + const finish = ( + partial: Omit, + ): ChurnIngestResult => ({ + ...partial, + elapsedMs: Math.round(performance.now() - t0), + }); + + if (!existsSync(resolve(projectRoot, ".git"))) { + const top = gitTopLevel(projectRoot); + if (!top) { + if (!merge) replaceFileChurn(db, []); + const reason = "skipped: not a git repository (file_churn empty)"; + if (!quiet) console.error(`[churn] ${reason}`); + const fallback = tryConfigChurnFallback(db, projectRoot, quiet, finish); + return fallback ?? finish({ ok: false, rowCount: 0, reason }); + } + } + + const gitRootRaw = gitTopLevel(projectRoot); + if (!gitRootRaw) { + if (!merge) replaceFileChurn(db, []); + const reason = "skipped: git unavailable (file_churn empty)"; + if (!quiet) console.error(`[churn] ${reason}`); + const fallback = tryConfigChurnFallback(db, projectRoot, quiet, finish); + return fallback ?? finish({ ok: false, rowCount: 0, reason }); + } + + const indexedPaths = new Set( + db + .query<{ path: string }>("SELECT path FROM files") + .all() + .map((r) => r.path), + ); + if (indexedPaths.size === 0) { + replaceFileChurn(db, []); + return finish({ ok: true, rowCount: 0 }); + } + + const gitRoot = resolveRealPath(gitRootRaw); + const projectPrefix = toPosix(relative(gitRoot, projectRoot)); + + let pathspecArgs: string[]; + let scopeFilter: Set | undefined; + if (merge && scopePaths) { + scopeFilter = new Set(scopePaths); + pathspecArgs = scopePaths.map((p) => indexedToGitPath(p, projectPrefix)); + } else { + pathspecArgs = [ + projectPrefix === "" || projectPrefix === "." ? "." : projectPrefix, + ]; + } + + const logArgs = [ + "log", + "--numstat", + "--format=COMMIT %H %ct", + ...(since ? [`${since}..HEAD`] : []), + "--", + ...pathspecArgs, + ]; + const log = spawnSync("git", logArgs, { + cwd: gitRootRaw, + env: gitSpawnEnv(), + encoding: "utf-8", + maxBuffer: 64 * 1024 * 1024, + }); + if (log.status !== 0) { + if (!merge) replaceFileChurn(db, []); + const reason = `skipped: git log failed (${log.stderr?.toString().trim() || "unknown"})`; + if (!quiet) console.error(`[churn] ${reason}`); + const fallback = tryConfigChurnFallback(db, projectRoot, quiet, finish); + return fallback ?? finish({ ok: false, rowCount: 0, reason }); + } + + const computedAt = new Date().toISOString(); + const byFile = parseGitNumstatLog(log.stdout, { + indexedPaths, + projectPrefix, + halfLife, + scopeFilter, + }); + const rows = accsToRows(byFile, computedAt); + + if (merge && scopePaths) { + mergeFileChurnForPaths(db, rows, scopePaths); + pruneFileChurnOrphans(db); + } else { + replaceFileChurn(db, rows); + } + + const rowCount = countFileChurn(db); + if (!quiet && rows.length > 0) { + console.error( + `[churn] file_churn ${merge ? "merged" : "populated"}: ${rows.length} files (${rowCount} total)`, + ); + } + stampChurnCommit(db, projectRoot); + return finish({ ok: true, rowCount }); +} + +/** Index-time churn refresh — git-native with config JSON fallback. */ +export function refreshFileChurn( + db: CodemapDatabase, + options?: { + quiet?: boolean; + projectRoot?: string; + halfLifeDays?: number; + since?: string | null; + mode?: ChurnRefreshMode; + changedPaths?: string[]; + }, +): ChurnIngestResult { + const t0 = performance.now(); + const quiet = options?.quiet ?? false; + const mode = options?.mode ?? "full"; + const projectRoot = options?.projectRoot ?? getProjectRoot(); + const head = resolveGitHead(projectRoot); + const prevHead = getMeta(db, META_CHURN_INDEXED_COMMIT); + + if (mode === "idle" && head && prevHead && head === prevHead) { + const rowCount = countFileChurn(db); + return { + ok: true, + rowCount, + elapsedMs: Math.round(performance.now() - t0), + reason: "skipped: HEAD unchanged", + }; + } + + const base = { + projectRoot, + halfLifeDays: options?.halfLifeDays ?? getChurnHalfLifeDays(), + since: options?.since !== undefined ? options.since : getChurnSince(), + quiet, + }; + + if ( + mode === "incremental" && + options?.changedPaths && + options.changedPaths.length > 0 + ) { + return ingestFileChurnFromGit(db, { + ...base, + scopePaths: options.changedPaths, + }); + } + + return ingestFileChurnFromGit(db, base); +} diff --git a/src/application/context-engine.test.ts b/src/application/context-engine.test.ts index dc6f4a9f..016e92f9 100644 --- a/src/application/context-engine.test.ts +++ b/src/application/context-engine.test.ts @@ -224,6 +224,14 @@ describe("readRecipeSqlLimit", () => { }); }); +describe("classifyIntent", () => { + it("maps hotspot/churn intents to refactor-priority recipe cards", () => { + const c = classifyIntent("which files churn often and are complex"); + expect(c.classified_as).toBe("refactor-priority"); + expect(c.matched_recipes[0]).toBe("churn-complexity-hotspots"); + }); +}); + describe("composeStartHere", () => { it("includes intent-ranked recipe cards and hub leaders with signatures", () => { withSeededDb((db) => { @@ -234,10 +242,10 @@ describe("composeStartHere", () => { ); expect(start.classified_as).toBe("refactor"); expect(start.recipes.map((r) => r.id)).toEqual([ + "churn-complexity-hotspots", "fan-in", "fan-out", "barrel-files", - "deprecated-symbols", ]); expect(start.index_summary.files).toBe(3); expect(start.recipes[0]?.tool).toBe("query_recipe"); @@ -491,10 +499,10 @@ describe("buildContextEnvelope", () => { }); expect(envelope.start_here?.classified_as).toBe("refactor"); expect(envelope.start_here?.recipes.map((r) => r.id)).toEqual([ + "churn-complexity-hotspots", "fan-in", "fan-out", "barrel-files", - "deprecated-symbols", ]); expect(envelope.intent?.classified_as).toBe("refactor"); }); diff --git a/src/application/context-engine.ts b/src/application/context-engine.ts index f4169290..1cc0d43c 100644 --- a/src/application/context-engine.ts +++ b/src/application/context-engine.ts @@ -149,16 +149,33 @@ export function classifyIntent(intent: string): { hint: string; } { const t = intent.toLowerCase(); + if ( + /hotspot|churn|refactor priority|risky to refactor|high.?churn|change.?often/.test( + t, + ) + ) { + return { + classified_as: "refactor-priority", + matched_recipes: [ + "churn-complexity-hotspots", + "refactor-risk-ranking", + "high-complexity-untested", + "fan-in", + ], + hint: "churn-complexity-hotspots ranks files by git churn × complexity (distinct from `hotspots` alias → fan-in); pair with refactor-risk-ranking and snippet before large edits.", + }; + } if (/refactor|rename|restructur|extract|move\b/.test(t)) { return { classified_as: "refactor", matched_recipes: [ + "churn-complexity-hotspots", "fan-in", "fan-out", "barrel-files", "deprecated-symbols", ], - hint: "Inspect fan-in / fan-out before moving symbols; barrel-files surfaces public-API hubs; deprecated-symbols flags risky callers.", + hint: "churn-complexity-hotspots surfaces high-churn × high-complexity files; inspect fan-in / fan-out before moving symbols.", }; } if (/bug|fix|debug|error|crash|broken|regress/.test(t)) { diff --git a/src/application/index-engine.ts b/src/application/index-engine.ts index 7860c6b3..07b23121 100644 --- a/src/application/index-engine.ts +++ b/src/application/index-engine.ts @@ -417,7 +417,8 @@ export function fetchTableStats(db: CodemapDatabase): IndexTableStats { (SELECT COUNT(*) FROM re_export_chains) as re_export_chains, (SELECT COUNT(*) FROM module_cycles) as module_cycles, (SELECT COUNT(*) FROM dynamic_imports) as dynamic_imports, - (SELECT COUNT(*) FROM file_metrics) as file_metrics`, + (SELECT COUNT(*) FROM file_metrics) as file_metrics, + (SELECT COUNT(*) FROM file_churn) as file_churn`, ) .get()!; return row as IndexTableStats; @@ -636,6 +637,7 @@ export async function indexFiles( module_cycles_ms: Math.round(moduleCyclesMs), re_export_chains_ms: Math.round(reExportChainsMs), heritage_ms: Math.round(heritageMs), + churn_ms: 0, total_ms: elapsed, slowest_files: slowest, }; diff --git a/src/application/ingest-churn-run.test.ts b/src/application/ingest-churn-run.test.ts new file mode 100644 index 00000000..8f51acb4 --- /dev/null +++ b/src/application/ingest-churn-run.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { closeDb, createSchema, insertFile } from "../db"; +import { openCodemapDatabase } from "../sqlite-db"; +import { ingestChurnFromJsonFile } from "./ingest-churn-run"; + +describe("ingestChurnFromJsonFile", () => { + it("loads indexed paths and skips unindexed rows", () => { + const root = mkdtempSync(join(tmpdir(), "codemap-ingest-churn-")); + try { + const jsonPath = join(root, "churn.json"); + writeFileSync( + jsonPath, + JSON.stringify([ + { + file_path: "src/a.ts", + commit_count: 5, + weighted_commits: 4, + lines_added: 10, + lines_removed: 2, + last_commit_at: "2026-06-01T00:00:00Z", + churn_trend: "stable", + computed_at: "2026-06-10T00:00:00Z", + }, + { + file_path: "src/missing.ts", + commit_count: 1, + weighted_commits: 1, + lines_added: 1, + lines_removed: 0, + last_commit_at: null, + churn_trend: null, + computed_at: "2026-06-10T00:00:00Z", + }, + ]), + ); + + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/a.ts", + content_hash: "a", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + const result = ingestChurnFromJsonFile(db, { + projectRoot: root, + path: "churn.json", + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.ingested).toBe(1); + expect(result.skipped_unindexed).toBe(1); + const n = db + .query<{ c: number }>("SELECT COUNT(*) AS c FROM file_churn") + .get()?.c; + expect(n).toBe(1); + } finally { + closeDb(db); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/application/ingest-churn-run.ts b/src/application/ingest-churn-run.ts new file mode 100644 index 00000000..bfbee27d --- /dev/null +++ b/src/application/ingest-churn-run.ts @@ -0,0 +1,135 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { isAbsolute, resolve } from "node:path"; + +import { + META_CHURN_INDEXED_COMMIT, + pruneFileChurnOrphans, + replaceFileChurn, + setMeta, +} from "../db"; +import type { FileChurnRow } from "../db"; +import type { CodemapDatabase } from "../sqlite-db"; + +export interface IngestChurnRunOk { + ok: true; + ingested: number; + skipped_unindexed: number; + sourcePath: string; +} + +export interface IngestChurnRunError { + ok: false; + error: string; +} + +export type IngestChurnRunResult = IngestChurnRunOk | IngestChurnRunError; + +function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { + if (!Array.isArray(raw)) { + throw new TypeError("churn JSON must be an array of file_churn rows"); + } + const rows: FileChurnRow[] = []; + for (const item of raw) { + if (item === null || typeof item !== "object") { + throw new TypeError("each churn row must be an object"); + } + const r = item as Record; + const file_path = r.file_path; + if (typeof file_path !== "string" || file_path.length === 0) { + throw new TypeError("file_path must be a non-empty string"); + } + rows.push({ + file_path, + commit_count: Number(r.commit_count) || 0, + weighted_commits: Number(r.weighted_commits) || 0, + lines_added: Number(r.lines_added) || 0, + lines_removed: Number(r.lines_removed) || 0, + last_commit_at: + r.last_commit_at === null || r.last_commit_at === undefined + ? null + : String(r.last_commit_at), + churn_trend: + r.churn_trend === null || r.churn_trend === undefined + ? null + : String(r.churn_trend), + computed_at: + typeof r.computed_at === "string" + ? r.computed_at + : new Date().toISOString(), + }); + } + return rows; +} + +/** + * Load churn rows from JSON and replace `file_churn` (indexed paths only). + * Used by `codemap ingest-churn` and config `churn.file` fallback. + */ +export function ingestChurnFromJsonFile( + db: CodemapDatabase, + options: { projectRoot: string; path: string }, +): IngestChurnRunResult { + const absPath = isAbsolute(options.path) + ? options.path + : resolve(options.projectRoot, options.path); + if (!existsSync(absPath)) { + return { ok: false, error: `churn file not found: ${absPath}` }; + } + let rows: FileChurnRow[]; + try { + rows = parseChurnJsonPayload( + JSON.parse(readFileSync(absPath, "utf-8")) as unknown, + ); + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } + + const indexed = new Set( + db + .query<{ path: string }>("SELECT path FROM files") + .all() + .map((r) => r.path), + ); + const kept: FileChurnRow[] = []; + let skipped = 0; + for (const row of rows) { + if (!indexed.has(row.file_path)) { + skipped += 1; + continue; + } + kept.push(row); + } + + replaceFileChurn(db, kept); + pruneFileChurnOrphans(db); + const headResult = spawnSync("git", ["rev-parse", "HEAD"], { + cwd: options.projectRoot, + }); + if (headResult.status === 0) { + const head = headResult.stdout.toString().trim(); + if (head) setMeta(db, META_CHURN_INDEXED_COMMIT, head); + } + + return { + ok: true, + ingested: kept.length, + skipped_unindexed: skipped, + sourcePath: absPath, + }; +} + +/** Config `churn.file` ingest when git churn is unavailable. */ +export function ingestChurnFromConfigPath( + db: CodemapDatabase, + options: { projectRoot: string; churnFile: string | null }, +): IngestChurnRunResult | null { + if (!options.churnFile) return null; + return ingestChurnFromJsonFile(db, { + projectRoot: options.projectRoot, + path: options.churnFile, + }); +} diff --git a/src/application/run-index.ts b/src/application/run-index.ts index b4ec0fc4..57f6a8fd 100644 --- a/src/application/run-index.ts +++ b/src/application/run-index.ts @@ -1,3 +1,5 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; + import { createSchema, getMeta, @@ -8,6 +10,8 @@ import { import type { CodemapDatabase } from "../db"; import { getBoundaryRules, getFts5Enabled } from "../runtime"; import { getStateDir } from "../runtime"; +import { refreshFileChurn } from "./churn-ingest"; +import type { ChurnRefreshMode } from "./churn-ingest"; import { expandHeritageResolveScope } from "./heritage-resolver"; import { collectFiles, @@ -20,7 +24,11 @@ import { targetedReindex, } from "./index-engine"; import { acquireIndexLock } from "./index-lock"; -import type { IndexResult, IndexTableStats } from "./types"; +import type { + IndexPerformanceReport, + IndexResult, + IndexTableStats, +} from "./types"; /** * Returns `true` when the persisted `meta.fts5_enabled` differs from the @@ -73,9 +81,29 @@ function emptyStats(): IndexTableStats { module_cycles: 0, dynamic_imports: 0, file_metrics: 0, + file_churn: 0, }; } +function patchPerformanceJsonWithChurn(churnMs: number): void { + const perfJsonPath = process.env.CODEMAP_PERFORMANCE_JSON; + if (perfJsonPath === undefined || perfJsonPath === "") return; + if (!existsSync(perfJsonPath)) return; + try { + const perf = JSON.parse( + readFileSync(perfJsonPath, "utf-8"), + ) as IndexPerformanceReport; + writeFileSync( + perfJsonPath, + JSON.stringify({ ...perf, churn_ms: churnMs }, null, 2), + ); + } catch (err) { + console.error( + `[churn] failed to patch performance JSON: ${err instanceof Error ? err.message : err}`, + ); + } +} + /** * - `incremental` — git-based diff vs last indexed commit (default). * - `full` — re-glob and re-index everything. @@ -163,8 +191,12 @@ async function runCodemapIndexBody( // `finally` because full rebuild calls `dropAll` inside `indexFiles` which // wipes `boundary_rules` (config-derived); reconciling AFTER the index // pipeline returns survives that drop on every code path. + let result: IndexResult; + let churnMode: ChurnRefreshMode = "full"; + let churnChangedPaths: string[] | undefined; try { if (mode === "full") { + churnMode = "full"; if (!quiet) console.log(" Full rebuild requested..."); const collectStart = performance.now(); const files = collectFiles(); @@ -175,19 +207,18 @@ async function runCodemapIndexBody( collectMs, commit: options.commit, }); - return { + result = { mode: "full", indexed: run.indexed, skipped: run.skipped, elapsedMs: run.elapsedMs, stats: run.stats, }; - } - - if (mode === "files") { + } else if (mode === "files") { const targetFiles = options.files ?? []; if (targetFiles.length === 0) { - return { + churnMode = "idle"; + result = { mode: "files", indexed: 0, skipped: 0, @@ -195,91 +226,111 @@ async function runCodemapIndexBody( stats: emptyStats(), idle: true, }; - } - const run = await targetedReindex(db, targetFiles, quiet); - return { - mode: "files", - indexed: run.indexed, - skipped: run.skipped, - elapsedMs: run.elapsedMs, - stats: run.stats, - }; - } - - // getChangedFiles reads `meta`; the up-front createSchema above (before the toggle check) covers it. - const diff = getChangedFiles(db); - if (diff) { - if (!quiet) { - console.log( - ` Incremental: ${diff.changed.length} changed, ${diff.deleted.length} deleted`, - ); - } - if (diff.changed.length > 0) { - const indexedPaths = diff.existingPaths; - for (const f of diff.changed) indexedPaths.add(f); - const run = await indexFiles(db, diff.changed, false, indexedPaths, { - quiet, - sourceCache: diff.sourceCache, - existingHashes: diff.existingHashes, - deletedPaths: diff.deleted, - }); - return { - mode: "incremental", + } else { + churnMode = "incremental"; + churnChangedPaths = targetFiles; + const run = await targetedReindex(db, targetFiles, quiet); + result = { + mode: "files", indexed: run.indexed, skipped: run.skipped, elapsedMs: run.elapsedMs, stats: run.stats, }; } - if (diff.deleted.length > 0) { - deleteFilesFromIndex(db, diff.deleted, quiet); - const callScope = expandHeritageResolveScope(db, diff.deleted); - if (callScope.length > 0) { - runCallResolveAndSynthesis(db, callScope); + } else { + // getChangedFiles reads `meta`; the up-front createSchema above covers it. + const diff = getChangedFiles(db); + if (diff) { + if (!quiet) { + console.log( + ` Incremental: ${diff.changed.length} changed, ${diff.deleted.length} deleted`, + ); } - setMeta(db, "last_indexed_commit", getCurrentCommit()); - if (!quiet) console.log(" Index updated (deletions only)"); - return { - mode: "incremental", - indexed: 0, - skipped: 0, - elapsedMs: 0, - stats: fetchTableStats(db), - idle: true, + if (diff.changed.length > 0) { + churnMode = "incremental"; + churnChangedPaths = diff.changed; + const indexedPaths = diff.existingPaths; + for (const f of diff.changed) indexedPaths.add(f); + const run = await indexFiles(db, diff.changed, false, indexedPaths, { + quiet, + sourceCache: diff.sourceCache, + existingHashes: diff.existingHashes, + deletedPaths: diff.deleted, + }); + result = { + mode: "incremental", + indexed: run.indexed, + skipped: run.skipped, + elapsedMs: run.elapsedMs, + stats: run.stats, + }; + } else if (diff.deleted.length > 0) { + churnMode = "incremental"; + churnChangedPaths = diff.deleted; + deleteFilesFromIndex(db, diff.deleted, quiet); + const callScope = expandHeritageResolveScope(db, diff.deleted); + if (callScope.length > 0) { + runCallResolveAndSynthesis(db, callScope); + } + setMeta(db, "last_indexed_commit", getCurrentCommit()); + if (!quiet) console.log(" Index updated (deletions only)"); + result = { + mode: "incremental", + indexed: 0, + skipped: 0, + elapsedMs: 0, + stats: fetchTableStats(db), + idle: true, + }; + } else { + churnMode = "idle"; + if (!quiet) console.log(" Index is up to date"); + result = { + mode: "incremental", + indexed: 0, + skipped: 0, + elapsedMs: 0, + stats: fetchTableStats(db), + idle: true, + }; + } + } else { + churnMode = "full"; + if (!quiet) { + console.log( + " No previous index or incompatible history, doing full rebuild...", + ); + } + const fallbackCollectStart = performance.now(); + const files = collectFiles(); + const fallbackCollectMs = performance.now() - fallbackCollectStart; + const run = await indexFiles(db, files, true, undefined, { + quiet, + performance: wantPerformance, + collectMs: fallbackCollectMs, + }); + result = { + mode: "full", + indexed: run.indexed, + skipped: run.skipped, + elapsedMs: run.elapsedMs, + stats: run.stats, }; } - if (!quiet) console.log(" Index is up to date"); - return { - mode: "incremental", - indexed: 0, - skipped: 0, - elapsedMs: 0, - stats: fetchTableStats(db), - idle: true, - }; } - - if (!quiet) { - console.log( - " No previous index or incompatible history, doing full rebuild...", - ); - } - const fallbackCollectStart = performance.now(); - const files = collectFiles(); - const fallbackCollectMs = performance.now() - fallbackCollectStart; - const run = await indexFiles(db, files, true, undefined, { - quiet, - performance: wantPerformance, - collectMs: fallbackCollectMs, - }); - return { - mode: "full", - indexed: run.indexed, - skipped: run.skipped, - elapsedMs: run.elapsedMs, - stats: run.stats, - }; } finally { reconcileBoundaryRules(db, getBoundaryRules()); } + + const churn = refreshFileChurn(db, { + quiet, + mode: churnMode, + changedPaths: churnChangedPaths, + }); + patchPerformanceJsonWithChurn(churn.elapsedMs); + if (wantPerformance && !quiet && churn.elapsedMs > 0) { + console.error(`[churn] ingest: ${churn.elapsedMs}ms`); + } + return { ...result, stats: fetchTableStats(db) }; } diff --git a/src/application/types.ts b/src/application/types.ts index 797539c1..3e9bb944 100644 --- a/src/application/types.ts +++ b/src/application/types.ts @@ -29,6 +29,7 @@ export interface IndexTableStats extends Record { module_cycles: number; dynamic_imports: number; file_metrics: number; + file_churn: number; } /** @@ -53,6 +54,8 @@ export interface IndexPerformanceReport { re_export_chains_ms: number; /** `resolveTypeHeritage` + persist wall. */ heritage_ms: number; + /** `refreshFileChurn` wall (every index pass). */ + churn_ms: number; /** * `indexFiles` wall-clock — `parse + insert + index_create + DDL + bindings * + module_cycles + re_export_chains + heritage_ms`. Does **not** include diff --git a/src/cli/bootstrap-codemap.ts b/src/cli/bootstrap-codemap.ts index cc9f7f16..183dfd35 100644 --- a/src/cli/bootstrap-codemap.ts +++ b/src/cli/bootstrap-codemap.ts @@ -24,6 +24,8 @@ export interface BootstrapCodemapOpts { stateDir?: string | undefined; /** CLI `--with-fts`; `undefined` defers to `.codemap/config.ts` `fts5`. */ fts5Cli?: boolean | undefined; + /** CLI `--churn-since `; overrides config `churn.since`. */ + churnSinceCli?: string | undefined; } export async function bootstrapCodemap( @@ -39,7 +41,11 @@ export async function bootstrapCodemap( const user = await loadUserConfig(opts.root, opts.configFile, { stateDir }); initCodemap( - resolveCodemapConfig(opts.root, user, { stateDir, fts5Cli: opts.fts5Cli }), + resolveCodemapConfig(opts.root, user, { + stateDir, + fts5Cli: opts.fts5Cli, + churnSinceCli: opts.churnSinceCli, + }), ); configureResolver(getProjectRoot(), getTsconfigPath()); // Sanity: getStateDir() must mirror what we passed into resolveCodemapConfig. diff --git a/src/cli/bootstrap.ts b/src/cli/bootstrap.ts index 8e1d58cc..85b1b275 100644 --- a/src/cli/bootstrap.ts +++ b/src/cli/bootstrap.ts @@ -91,6 +91,9 @@ Options: --state-dir DIR State directory for codemap-managed files (default .codemap/ under root) --performance Print per-phase timing breakdown + top-10 slowest files (full rebuild only) + --churn-since REF + Only count git commits after REF for file_churn ingest + (overrides config churn.since) --help, -h Show this help `); } @@ -171,6 +174,14 @@ export function validateIndexModeArgs(rest: string[]): void { i++; continue; } + if (a === "--churn-since") { + if (i + 1 >= rest.length || rest[i + 1].startsWith("-")) { + console.error("codemap: --churn-since requires a git revision"); + process.exit(1); + } + i += 2; + continue; + } if (a === "--files") { i++; const start = i; @@ -199,6 +210,7 @@ export function parseBootstrapArgs(argv: string[]) { let configFile: string | undefined; let stateDir: string | undefined; let fts5Cli: boolean | undefined; + let churnSinceCli: string | undefined; const rest: string[] = []; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -221,10 +233,15 @@ export function parseBootstrapArgs(argv: string[]) { rest.push(a); continue; } + if (a === "--churn-since" && argv[i + 1]) { + churnSinceCli = argv[++i]; + rest.push("--churn-since", churnSinceCli); + continue; + } rest.push(a); } if (!root) root = process.cwd(); // --state-dir wins over CODEMAP_STATE_DIR (precedence per plan §D7). if (!stateDir) stateDir = process.env.CODEMAP_STATE_DIR; - return { root, configFile, stateDir, fts5Cli, rest }; + return { root, configFile, stateDir, fts5Cli, churnSinceCli, rest }; } diff --git a/src/cli/cmd-index.ts b/src/cli/cmd-index.ts index 0104709d..a3b02739 100644 --- a/src/cli/cmd-index.ts +++ b/src/cli/cmd-index.ts @@ -13,6 +13,7 @@ export async function runIndexCmd(opts: { configFile: string | undefined; stateDir?: string | undefined; fts5Cli?: boolean | undefined; + churnSinceCli?: string | undefined; rest: string[]; }): Promise { await bootstrapCodemap(opts); diff --git a/src/cli/cmd-ingest-churn.ts b/src/cli/cmd-ingest-churn.ts new file mode 100644 index 00000000..0779de6c --- /dev/null +++ b/src/cli/cmd-ingest-churn.ts @@ -0,0 +1,109 @@ +import { ingestChurnFromJsonFile } from "../application/ingest-churn-run"; +import { closeDb, openDb } from "../db"; +import { bootstrapCodemap } from "./bootstrap-codemap"; + +interface IngestChurnOpts { + root: string; + configFile: string | undefined; + stateDir?: string | undefined; + path: string; + json: boolean; +} + +export function printIngestChurnCmdHelp(): void { + console.log(`Usage: codemap ingest-churn [--json] + +Import precomputed git churn metrics into \`file_churn\` for non-git +repositories or CI fixtures. JSON must be an array of rows matching the +\`file_churn\` schema (see docs/architecture.md). Only paths present in +the index are kept; unindexed paths are skipped. + +Args: + Path to JSON file (relative to project root or absolute) + +Flags: + --json Emit result envelope on stdout + --help, -h Show this help + +Output (JSON): + { "ingested": N, "skipped_unindexed": K, "sourcePath": "..." } + +Examples: + codemap ingest-churn metrics/churn.json + codemap ingest-churn metrics/churn.json --json +`); +} + +export function parseIngestChurnRest( + rest: string[], +): + | { kind: "help" } + | { kind: "error"; message: string } + | { kind: "run"; path: string; json: boolean } { + if (rest[0] !== "ingest-churn") { + throw new Error("parseIngestChurnRest: expected ingest-churn"); + } + let path: string | undefined; + let json = false; + for (let i = 1; i < rest.length; i++) { + const a = rest[i]!; + if (a === "--help" || a === "-h") return { kind: "help" }; + if (a === "--json") { + json = true; + continue; + } + if (a.startsWith("-")) { + return { + kind: "error", + message: `codemap ingest-churn: unknown option "${a}"`, + }; + } + if (path !== undefined) { + return { + kind: "error", + message: "codemap ingest-churn: unexpected extra path argument", + }; + } + path = a; + } + if (!path) { + return { + kind: "error", + message: "codemap ingest-churn: missing argument", + }; + } + return { kind: "run", path, json }; +} + +export async function runIngestChurnCmd(opts: IngestChurnOpts): Promise { + await bootstrapCodemap(opts); + const db = openDb(); + try { + const result = ingestChurnFromJsonFile(db, { + projectRoot: opts.root, + path: opts.path, + }); + if (!result.ok) { + console.error(result.error); + process.exit(1); + } + if (opts.json) { + console.log( + JSON.stringify({ + ingested: result.ingested, + skipped_unindexed: result.skipped_unindexed, + sourcePath: result.sourcePath, + }), + ); + } else { + console.log( + ` Ingested ${result.ingested} file_churn rows from ${result.sourcePath}` + + (result.skipped_unindexed > 0 + ? ` (${result.skipped_unindexed} skipped — not in index)` + : ""), + ); + } + } finally { + closeDb(db); + } +} diff --git a/src/cli/main.ts b/src/cli/main.ts index 2b4a8f02..2194e9a3 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -18,7 +18,7 @@ import { emitJsonError } from "./emit-tool-result.js"; */ export async function main(): Promise { const argv = process.argv.slice(2); - const { root, configFile, stateDir, fts5Cli, rest } = + const { root, configFile, stateDir, fts5Cli, churnSinceCli, rest } = parseBootstrapArgs(argv); if (rest[0] === "--help" || rest[0] === "-h") { @@ -430,6 +430,28 @@ Copies bundled agent templates into .agents/ under the project root. return; } + if (rest[0] === "ingest-churn") { + const { parseIngestChurnRest, printIngestChurnCmdHelp, runIngestChurnCmd } = + await import("./cmd-ingest-churn.js"); + const parsed = parseIngestChurnRest(rest); + if (parsed.kind === "help") { + printIngestChurnCmdHelp(); + return; + } + if (parsed.kind === "error") { + console.error(parsed.message); + process.exit(1); + } + await runIngestChurnCmd({ + root, + configFile, + stateDir, + path: parsed.path, + json: parsed.json, + }); + return; + } + if (rest[0] === "ingest-coverage") { const { parseIngestCoverageRest, @@ -710,5 +732,12 @@ Copies bundled agent templates into .agents/ under the project root. } const { runIndexCmd } = await import("./cmd-index.js"); - await runIndexCmd({ root, configFile, stateDir, fts5Cli, rest }); + await runIndexCmd({ + root, + configFile, + stateDir, + fts5Cli, + churnSinceCli, + rest, + }); } diff --git a/src/config.test.ts b/src/config.test.ts index 3f6024f2..0d4f35eb 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -118,6 +118,23 @@ describe("resolveCodemapConfig", () => { expect(r.excludeDirNames).not.toEqual(DEFAULT_EXCLUDE_DIR_NAMES); }); + it("defaults churn halfLifeDays and null since/file", () => { + const r = resolveCodemapConfig(dir, undefined); + expect(r.churn.halfLifeDays).toBe(90); + expect(r.churn.since).toBeNull(); + expect(r.churn.file).toBeNull(); + }); + + it("resolves churn config and CLI since override", () => { + const r = resolveCodemapConfig( + dir, + { churn: { halfLifeDays: 30, since: "v1.0.0" } }, + { churnSinceCli: "abc123" }, + ); + expect(r.churn.halfLifeDays).toBe(30); + expect(r.churn.since).toBe("abc123"); + }); + it("defaults boundaries to []", () => { const r = resolveCodemapConfig(dir, undefined); expect(r.boundaries).toEqual([]); diff --git a/src/config.ts b/src/config.ts index f0b2f63f..fd7de83e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -100,6 +100,35 @@ export const codemapUserConfigSchema = z .describe( "Track per-recipe `last_run_at` + `run_count` in the `recipe_recency` table; surfaces inline on `--recipes-json` for agent-host ranking. Default `true` (opt-out). Set `false` to short-circuit every write — no rows ever land. Local-only — no upload primitive. See `docs/architecture.md` § `recipe_recency`.", ), + churn: z + .object({ + halfLifeDays: z + .number() + .positive() + .optional() + .describe( + "Exponential half-life (days) for `file_churn.weighted_commits`. Default `90`.", + ), + since: z + .string() + .min(1) + .optional() + .describe( + "Git revision (commit/tag/branch) — only commits after this ref contribute to churn. CLI `--churn-since` overrides.", + ), + file: z + .string() + .min(1) + .optional() + .describe( + "JSON array of `file_churn` rows (relative to project root). Used when git churn is unavailable; also loadable via `codemap ingest-churn`.", + ), + }) + .strict() + .optional() + .describe( + "Git churn ingest for `file_churn` (runs after every index pass).", + ), synthesis: z .object({ heuristicCalls: z @@ -225,6 +254,14 @@ export interface ResolvedCodemapConfig { readonly to_glob: string; readonly action: "deny" | "allow"; }>; + /** Git churn ingest tuning for `file_churn`. */ + readonly churn: { + readonly halfLifeDays: number; + /** When set, only commits after this git ref are counted. */ + readonly since: string | null; + /** Optional JSON churn import when git is unavailable. */ + readonly file: string | null; + }; /** When `heuristicCalls` is true, runs callback-synthesis after `resolveCalls`. */ readonly synthesis: { readonly heuristicCalls: boolean; @@ -273,6 +310,8 @@ export interface ResolveCodemapConfigOpts { * parsing in the bootstrap layer. */ fts5Cli?: boolean | undefined; + /** CLI `--churn-since ` — overrides config `churn.since`. */ + churnSinceCli?: string | undefined; } /** @@ -339,6 +378,20 @@ export function resolveCodemapConfig( const heuristicCalls = parsed?.synthesis?.heuristicCalls === true; + const churnHalfLifeDays = parsed?.churn?.halfLifeDays ?? 90; + const churnFile = parsed?.churn?.file + ? resolve(absRoot, parsed.churn.file) + : null; + let churnSince: string | null = parsed?.churn?.since ?? null; + if (opts.churnSinceCli !== undefined && opts.churnSinceCli !== "") { + churnSince = opts.churnSinceCli; + if (parsed?.churn?.since && parsed.churn.since !== opts.churnSinceCli) { + console.error( + `[churn] CLI override: --churn-since ${opts.churnSinceCli} (config churn.since ignored)`, + ); + } + } + const autoApply = parsed?.apply?.autoApplyRecipes; const applyAutoApplyRecipes = autoApply !== undefined && autoApply.length > 0 @@ -355,6 +408,11 @@ export function resolveCodemapConfig( fts5, boundaries, recipeRecency, + churn: { + halfLifeDays: churnHalfLifeDays, + since: churnSince, + file: churnFile, + }, synthesis: { heuristicCalls }, applyAutoApplyRecipes, }; diff --git a/src/db.ts b/src/db.ts index 0e7ebf3e..2ff1638d 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,7 +3,7 @@ import type { CodemapDatabase, BindValues } from "./sqlite-db"; /** Bump only on rebuild-forcing DDL changes (NOT on additive tables/columns). * See `docs/architecture.md` § Schema Versioning. */ -export const SCHEMA_VERSION = 39; +export const SCHEMA_VERSION = 40; /** Moat-A: default call-graph surfaces exclude callback-synthesis edges. */ export const CALLS_AST_ONLY_SQL = "(provenance IS NULL OR provenance = 'ast')"; @@ -93,6 +93,18 @@ export function createTables(db: CodemapDatabase) { export_count INTEGER NOT NULL DEFAULT 0 ) STRICT; + -- Git churn metrics per indexed file (populated each index pass via churn-ingest, golden uses seed-file-churn). + CREATE TABLE IF NOT EXISTS file_churn ( + file_path TEXT PRIMARY KEY REFERENCES files(path) ON DELETE CASCADE, + commit_count INTEGER NOT NULL, + weighted_commits REAL NOT NULL, + lines_added INTEGER NOT NULL, + lines_removed INTEGER NOT NULL, + last_commit_at TEXT, + churn_trend TEXT, + computed_at TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS imports ( id INTEGER PRIMARY KEY AUTOINCREMENT, file_path TEXT NOT NULL REFERENCES files(path) ON DELETE CASCADE, @@ -763,6 +775,7 @@ export function dropAll(db: CodemapDatabase) { DROP TABLE IF EXISTS runtime_markers; DROP TABLE IF EXISTS test_suites; DROP TABLE IF EXISTS file_metrics; + DROP TABLE IF EXISTS file_churn; DROP TABLE IF EXISTS unresolved_calls; DROP TABLE IF EXISTS bindings; DROP TABLE IF EXISTS "references"; @@ -1697,6 +1710,66 @@ export function insertFileMetrics(db: CodemapDatabase, rows: FileMetricsRow[]) { ); } +/** One row per indexed file with git churn metrics (see `file_churn` table). */ +export interface FileChurnRow { + file_path: string; + commit_count: number; + weighted_commits: number; + lines_added: number; + lines_removed: number; + last_commit_at: string | null; + churn_trend: string | null; + computed_at: string; +} + +export function insertFileChurn(db: CodemapDatabase, rows: FileChurnRow[]) { + batchInsert( + db, + rows, + "INSERT INTO file_churn (file_path, commit_count, weighted_commits, lines_added, lines_removed, last_commit_at, churn_trend, computed_at)", + "(?,?,?,?,?,?,?,?)", + (r, v) => + v.push( + r.file_path, + r.commit_count, + r.weighted_commits, + r.lines_added, + r.lines_removed, + r.last_commit_at, + r.churn_trend, + r.computed_at, + ), + ); +} + +/** Replace all churn rows (full-rebuild git ingest or golden seed-file-churn). */ +export function replaceFileChurn(db: CodemapDatabase, rows: FileChurnRow[]) { + db.run("DELETE FROM file_churn"); + insertFileChurn(db, rows); +} + +/** `meta` key: last `HEAD` when `file_churn` was refreshed (idle skip). */ +export const META_CHURN_INDEXED_COMMIT = "churn_indexed_commit"; + +/** Replace churn rows for `scopePaths` only; other paths are left unchanged. */ +export function mergeFileChurnForPaths( + db: CodemapDatabase, + rows: FileChurnRow[], + scopePaths: Iterable, +) { + for (const p of scopePaths) { + db.run("DELETE FROM file_churn WHERE file_path = ?", [p]); + } + insertFileChurn(db, rows); +} + +/** Drop churn rows whose `file_path` is no longer indexed. */ +export function pruneFileChurnOrphans(db: CodemapDatabase) { + db.run( + "DELETE FROM file_churn WHERE file_path NOT IN (SELECT path FROM files)", + ); +} + /** * One row per leaf parameter binding, ordered by position. Owner-kind * lets `(file_path, owner_name, owner_kind)` disambiguate same-name diff --git a/src/file-churn.test.ts b/src/file-churn.test.ts new file mode 100644 index 00000000..a5d40ae7 --- /dev/null +++ b/src/file-churn.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + closeDb, + createSchema, + insertFile, + insertFileChurn, + replaceFileChurn, +} from "./db"; +import type { FileChurnRow } from "./db"; +import { openCodemapDatabase } from "./sqlite-db"; + +const REPO_ROOT = join(import.meta.dir, ".."); + +describe("file_churn + churn-complexity-hotspots recipe", () => { + it("ranks files by weighted_commits × max complexity", () => { + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/hot.ts", + content_hash: "a", + size: 100, + line_count: 10, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + insertFile(db, { + path: "src/cold.ts", + content_hash: "b", + size: 50, + line_count: 5, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + const churn: FileChurnRow[] = [ + { + file_path: "src/hot.ts", + commit_count: 20, + weighted_commits: 15, + lines_added: 200, + lines_removed: 50, + last_commit_at: "2026-06-01T00:00:00Z", + churn_trend: "accelerating", + computed_at: "2026-06-10T00:00:00Z", + }, + { + file_path: "src/cold.ts", + commit_count: 2, + weighted_commits: 1, + lines_added: 10, + lines_removed: 2, + last_commit_at: "2026-01-01T00:00:00Z", + churn_trend: "cooling", + computed_at: "2026-06-10T00:00:00Z", + }, + ]; + replaceFileChurn(db, churn); + db.run( + `INSERT INTO symbols (file_path, name, kind, line_start, line_end, signature, is_exported, is_default_export, members, doc_comment, value, parent_name, visibility, name_column_start, name_column_end, scope_local_id, body_line_count, param_count, complexity) + VALUES ('src/hot.ts', 'hotFn', 'function', 1, 5, 'hotFn()', 1, 0, NULL, NULL, NULL, NULL, NULL, 1, 4, 0, 5, 0, 12), + ('src/cold.ts', 'coldFn', 'function', 1, 3, 'coldFn()', 1, 0, NULL, NULL, NULL, NULL, NULL, 1, 5, 0, 3, 0, 3)`, + ); + + const sql = readFileSync( + join(REPO_ROOT, "templates/recipes/churn-complexity-hotspots.sql"), + "utf-8", + ); + const rows = db.query(sql).all(20, 1, 0) as Array<{ + file_path: string; + hotspot_score: number; + hotspot_score_normalized: number; + symbol_name: string | null; + }>; + expect(rows.length).toBe(2); + expect(rows[0]?.file_path).toBe("src/hot.ts"); + expect(rows[0]?.hotspot_score).toBe(180); + expect(rows[0]?.hotspot_score_normalized).toBe(100); + expect(rows[0]?.symbol_name).toBeNull(); + expect(rows[1]?.file_path).toBe("src/cold.ts"); + expect(rows[1]?.hotspot_score).toBe(3); + expect(rows[1]?.hotspot_score_normalized).toBe(1.7); + } finally { + closeDb(db); + } + }); + + it("replaceFileChurn clears prior rows", () => { + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/x.ts", + content_hash: "x", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + insertFileChurn(db, [ + { + file_path: "src/x.ts", + commit_count: 1, + weighted_commits: 1, + lines_added: 1, + lines_removed: 0, + last_commit_at: null, + churn_trend: null, + computed_at: "2026-06-10T00:00:00Z", + }, + ]); + replaceFileChurn(db, []); + const count = db + .query<{ n: number }>("SELECT COUNT(*) AS n FROM file_churn") + .get(); + expect(count?.n).toBe(0); + } finally { + closeDb(db); + } + }); +}); diff --git a/src/runtime.ts b/src/runtime.ts index fc82a5b8..9d5c8a62 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -63,6 +63,18 @@ export function getApplyAutoApplyRecipes(): readonly string[] | undefined { return getCodemapConfig().applyAutoApplyRecipes; } +export function getChurnHalfLifeDays(): number { + return getCodemapConfig().churn.halfLifeDays; +} + +export function getChurnSince(): string | null { + return getCodemapConfig().churn.since; +} + +export function getChurnFilePath(): string | null { + return getCodemapConfig().churn.file; +} + /** True if any path segment matches an excluded directory name (e.g. `node_modules`). */ export function isPathExcluded(relPath: string): boolean { const set = getExcludeDirNames(); diff --git a/src/worker-pool.dist.test.ts b/src/worker-pool.dist.test.ts index b1c4dc80..aae6c64f 100644 --- a/src/worker-pool.dist.test.ts +++ b/src/worker-pool.dist.test.ts @@ -4,7 +4,8 @@ */ import { describe, expect, test } from "bun:test"; -import { existsSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { globSync } from "./glob-sync"; @@ -42,14 +43,23 @@ describe("node dist --full exit delay", () => { const files = globSync(["**/*.ts", "**/*.tsx", "**/*.css"], minimalRoot); expect(files.length).toBeGreaterThan(WORKER_POOL_INLINE_PARSE_MAX); - await expectSubprocessExits(() => - Bun.spawn(["node", distEntry, "--full"], { - cwd: repoRoot, - env: { ...process.env, CODEMAP_ROOT: minimalRoot }, - stdout: "ignore", - stderr: "pipe", - }), - ); + const stateDir = mkdtempSync(join(tmpdir(), "codemap-dist-full-")); + try { + await expectSubprocessExits(() => + Bun.spawn(["node", distEntry, "--full"], { + cwd: repoRoot, + env: { + ...process.env, + CODEMAP_ROOT: minimalRoot, + CODEMAP_STATE_DIR: stateDir, + }, + stdout: "ignore", + stderr: "pipe", + }), + ); + } finally { + rmSync(stateDir, { recursive: true, force: true }); + } }, 8_000, ); diff --git a/templates/agent-content/rule/00-full.md b/templates/agent-content/rule/00-full.md index 83de3b50..94cca3be 100644 --- a/templates/agent-content/rule/00-full.md +++ b/templates/agent-content/rule/00-full.md @@ -71,6 +71,7 @@ If the question matches any of these, use the index instead of grepping: | "Complex + undertested without coverage ingest?" | `--recipe high-crap-score` (graph-estimated tiers; `coverage_source: estimated`) | | "What's cognitively complex (nesting-heavy)?" | `--recipe high-cognitive-complexity` (default `min_score=15`; `--params min_score=20` to tighten) | | "Structurally duplicate function bodies?" | `--recipe duplicates` (rename-insensitive `body_hash`; triage with `snippet` before refactor) | +| "What files churn often AND are complex?" | `--recipe churn-complexity-hotspots` (or `--params by_symbol=true`; non-git: `ingest-churn` / `churn.file`; alias `hotspots` → `fan-in`) | ## Quick reference queries diff --git a/templates/recipes/churn-complexity-hotspots.md b/templates/recipes/churn-complexity-hotspots.md new file mode 100644 index 00000000..947ff179 --- /dev/null +++ b/templates/recipes/churn-complexity-hotspots.md @@ -0,0 +1,33 @@ +--- +params: + - name: row_limit + type: number + required: false + default: 20 + description: Maximum rows to return (default 20) + - name: min_complexity + type: number + required: false + default: 1 + description: Minimum cyclomatic complexity (default 1) + - name: by_symbol + type: boolean + required: false + default: false + description: When true, one row per symbol; default false ranks files (max complexity in-file) +--- + +# churn-complexity-hotspots + +Files or symbols ranked by **git churn × cyclomatic complexity**. Distinct from the outcome alias `codemap hotspots` (import **fan-in** via `fan-in` recipe). + +Populated on every index pass from `git log --numstat` (config `churn.halfLifeDays`, `churn.since` / `--churn-since`). Non-git repos: `codemap ingest-churn ` or config `churn.file`. + +```bash +codemap query --recipe churn-complexity-hotspots +codemap query --recipe churn-complexity-hotspots --params min_complexity=10,row_limit=10 +codemap query --recipe churn-complexity-hotspots --params by_symbol=true +codemap ingest-churn churn-metrics.json +``` + +`hotspot_score` = `weighted_commits × complexity`. `hotspot_score_normalized` is 0–100 vs the corpus max in the result set. Triage with `snippet` before large refactors. diff --git a/templates/recipes/churn-complexity-hotspots.sql b/templates/recipes/churn-complexity-hotspots.sql new file mode 100644 index 00000000..290305a2 --- /dev/null +++ b/templates/recipes/churn-complexity-hotspots.sql @@ -0,0 +1,66 @@ +WITH params(row_limit, min_complexity, by_symbol) AS ( + SELECT ?, ?, ? +), +base AS ( + SELECT + fc.file_path, + s.name AS symbol_name, + s.kind AS symbol_kind, + s.line_start, + fc.weighted_commits, + fc.commit_count, + fc.churn_trend, + s.complexity, + ROUND(fc.weighted_commits * s.complexity, 2) AS hotspot_score + FROM file_churn fc + JOIN symbols s ON s.file_path = fc.file_path + WHERE s.complexity IS NOT NULL + AND s.complexity >= (SELECT min_complexity FROM params) +), +file_rows AS ( + SELECT + file_path, + NULL AS symbol_name, + NULL AS symbol_kind, + NULL AS line_start, + weighted_commits, + commit_count, + churn_trend, + MAX(complexity) AS max_complexity, + ROUND(AVG(complexity), 1) AS avg_complexity, + MAX(hotspot_score) AS hotspot_score + FROM base + GROUP BY file_path, weighted_commits, commit_count, churn_trend +), +symbol_rows AS ( + SELECT + file_path, + symbol_name, + symbol_kind, + line_start, + weighted_commits, + commit_count, + churn_trend, + complexity AS max_complexity, + complexity AS avg_complexity, + hotspot_score + FROM base +), +combined AS ( + SELECT * FROM file_rows WHERE (SELECT by_symbol FROM params) = 0 + UNION ALL + SELECT * FROM symbol_rows WHERE (SELECT by_symbol FROM params) != 0 +), +normalized AS ( + SELECT + c.*, + ROUND( + 100.0 * c.hotspot_score / NULLIF(MAX(c.hotspot_score) OVER (), 0), + 1 + ) AS hotspot_score_normalized + FROM combined c +) +SELECT * +FROM normalized +ORDER BY hotspot_score DESC, file_path, symbol_name +LIMIT (SELECT row_limit FROM params); From fca417940bb539aa51ef63c20a3e87181ec927ce Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 10 Jun 2026 16:54:44 +0300 Subject: [PATCH 2/8] harden: churn idle-skip correctness, docs lift, consumer surfaces Require populated file_churn and config fingerprint before idle skip; deletions-only index skips git log; ingest-churn needs index + inline schema help; delete shipped plan and fix inbound refs. --- .changeset/churn-complexity-hotspots.md | 2 +- docs/architecture.md | 2 +- docs/benchmark.md | 4 +- docs/glossary.md | 2 +- docs/golden-queries.md | 4 + docs/plans/churn-complexity-hotspots.md | 50 ------- docs/plans/substrate-extraction.md | 2 +- docs/roadmap.md | 2 +- docs/testing-coverage.md | 49 +++---- src/application/churn-ingest.test.ts | 126 +++++++++++++++++- src/application/churn-ingest.ts | 61 ++++++++- src/application/ingest-churn-run.ts | 21 ++- src/application/run-index.ts | 3 +- src/cli/bootstrap.ts | 4 + src/cli/cmd-ingest-churn.ts | 19 ++- src/db.ts | 2 + src/worker-pool.dist.test.ts | 2 + templates/agent-content/rule/00-full.md | 2 +- .../recipes/churn-complexity-hotspots.md | 8 +- 19 files changed, 265 insertions(+), 100 deletions(-) delete mode 100644 docs/plans/churn-complexity-hotspots.md diff --git a/.changeset/churn-complexity-hotspots.md b/.changeset/churn-complexity-hotspots.md index 4609d31c..0bb8a645 100644 --- a/.changeset/churn-complexity-hotspots.md +++ b/.changeset/churn-complexity-hotspots.md @@ -2,4 +2,4 @@ "@stainless-code/codemap": minor --- -Add churn × complexity hotspot ranking: `file_churn` from git on every index (incremental scoped refresh, idle HEAD cache), `codemap ingest-churn` + `churn.file` for non-git, bundled `churn-complexity-hotspots` recipe with file/symbol grain (`by_symbol`), raw + 0–100 normalized scores, and `churn_trend`. Outcome alias `hotspots` still maps to fan-in. +Add churn × complexity hotspot ranking: `file_churn` refreshed on every index from git history, with `codemap ingest-churn` and config `churn.file` for non-git repos. New `churn-complexity-hotspots` recipe ranks files or symbols (`by_symbol`) by change frequency × complexity with normalized 0–100 scores and `churn_trend`. Outcome alias `hotspots` still maps to fan-in. diff --git a/docs/architecture.md b/docs/architecture.md index 01581155..43cf3253 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -212,7 +212,7 @@ Three **mutually exclusive** CLI entry shapes; all converge on `applyDiffPayload **`src/application/session-lifecycle.ts`** — transport-specific start/stop rules for long-running `mcp` / `serve` processes (one-shot CLI unchanged). **`createStdioDisconnectMonitor`** (MCP only) exits the process when the agent host is actually gone: stdin EOF, stdout `EPIPE`, boot parent PID no longer alive (2s poll), or SIGINT/SIGTERM. The MCP SDK's stdio `transport.onclose` alone is insufficient — it fires only after an explicit `transport.close()`, not when the parent crashes without tearing down the pipe. **`createManagedWatchSession`** refcount-gates chokidar: MCP acquires one client before `connect` and **`forceStop`** drains the watcher on disconnect; HTTP acquires per authenticated request (after auth; **`GET /health`** excluded) and **`releaseClient`** stops the watcher when the count hits zero. **No MCP idle timeout:** `codemap mcp` does **not** exit after N minutes without tool calls while the stdio pipe stays open. IDE hosts spawn MCP once per session and do not reliably respawn it mid-conversation — an idle shutdown would break long pauses (human think time, reading, multi-step plans) with no recovery path. Orphan cleanup is handled by **disconnect detection**, not inactivity timers. **HTTP watch release grace (`HTTP_WATCH_RELEASE_GRACE_MS` = 5000):** distinct from idle timeout — only stops chokidar between stateless requests so the watcher is not started/stopped on every POST; the HTTP listener keeps running. **`GET /health`** liveness probes do not acquire a watch client (probes must not keep chokidar hot). Future **`MCP shared daemon per project`** could revisit opt-in idle policies with explicit client reconnect; not planned for stdio MCP today. -**Performance wiring:** **`--performance`** plumbs through **`RunIndexOptions.performance`** → **`indexFiles({ performance, collectMs })`**. `parse-worker-core.ts` records per-file **`parseMs`** on each `ParsedFile`; main thread times the eight phases (`collect`, `parse`, `insert`, `index_create`, `bindings`, `module_cycles`, `re_export_chains`, `heritage`) and assembles **`IndexPerformanceReport`** under `IndexRunStats.performance`. Note: `total_ms` is `indexFiles` wall-clock (parse + insert + DDL + bindings + cycles + re_exports + heritage), **not** end-to-end run wall — `collect_ms` happens before `indexFiles` and is reported separately. Env var **`CODEMAP_PERFORMANCE_JSON=`** dumps the report as JSON post-run (consumed by [`bun run check:perf-baseline`](./benchmark.md#perf-baseline-regression-guardrail) for local + weekly scheduled drift checks — not a PR merge gate). +**Performance wiring:** **`--performance`** plumbs through **`RunIndexOptions.performance`** → **`indexFiles({ performance, collectMs })`**. `parse-worker-core.ts` records per-file **`parseMs`** on each `ParsedFile`; main thread times the eight phases (`collect`, `parse`, `insert`, `index_create`, `bindings`, `module_cycles`, `re_export_chains`, `heritage`) and assembles **`IndexPerformanceReport`** under `IndexRunStats.performance`. Post-index **`refreshFileChurn`** records **`churn_ms`** separately (patched into the performance JSON when `CODEMAP_PERFORMANCE_JSON` is set). Note: `total_ms` is `indexFiles` wall-clock (parse + insert + DDL + bindings + cycles + re_exports + heritage), **not** end-to-end run wall — `collect_ms` and `churn_ms` happen outside `indexFiles` and are reported separately. Env var **`CODEMAP_PERFORMANCE_JSON=`** dumps the report as JSON post-run (consumed by [`bun run check:perf-baseline`](./benchmark.md#perf-baseline-regression-guardrail) for local + weekly scheduled drift checks — not a PR merge gate). **Agent templates:** `codemap agents init` writes thin pointer files (~18-line SKILL + ~25-line rule) to consumer disk; full content is served live by `codemap skill` / `codemap rule` (CLI) and `codemap://skill` / `codemap://rule` (MCP / HTTP) from `templates/agent-content//*.md`. Section files concatenate in lexical order; `*.gen.md` sections dispatch to renderers in `application/agent-content.ts` so recipe catalog + schema DDL auto-register. Pointer-version stamp (``) + once-per-process stderr nag (`maybeWarnStalePointers`) flag stale consumer templates; cure is `codemap agents init --force`. Full matrix: [agents.md](./agents.md). diff --git a/docs/benchmark.md b/docs/benchmark.md index ff6090d3..567b3449 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -207,9 +207,9 @@ Independent of the consumer-facing scenarios above, the repo carries a **per-pha ### Mechanism -1. `bun src/index.ts --full --performance` populates [`IndexPerformanceReport`](../src/application/types.ts) with `collect_ms` / `parse_ms` / `insert_ms` / `index_create_ms` / `bindings_ms` / `module_cycles_ms` / `re_export_chains_ms` / `heritage_ms` / `total_ms`. +1. `bun src/index.ts --full --performance` populates [`IndexPerformanceReport`](../src/application/types.ts) with `collect_ms` / `parse_ms` / `insert_ms` / `index_create_ms` / `bindings_ms` / `module_cycles_ms` / `re_export_chains_ms` / `heritage_ms` / `total_ms`, plus post-index **`churn_ms`** (git churn ingest; patched after `indexFiles` completes). 2. Setting `CODEMAP_PERFORMANCE_JSON=` dumps that report as JSON to `` after the run (no CLI flag added; env-var only). -3. [`scripts/check-perf-baseline.ts`](../scripts/check-perf-baseline.ts) (alias `bun run check:perf-baseline`) runs the indexer 3× on this repo, takes per-phase **medians**, and compares **`collect_ms`**, **`parse_ms`**, **`insert_ms`**, **`index_create_ms`**, **`bindings_ms`**, and **`total_ms`** to `fixtures/benchmark/perf-baseline.json`. Other `IndexPerformanceReport` fields (`module_cycles_ms`, `re_export_chains_ms`, `heritage_ms`, …) appear in `--performance` JSON only — not baseline-gated. +3. [`scripts/check-perf-baseline.ts`](../scripts/check-perf-baseline.ts) (alias `bun run check:perf-baseline`) runs the indexer 3× on this repo, takes per-phase **medians**, and compares **`collect_ms`**, **`parse_ms`**, **`insert_ms`**, **`index_create_ms`**, **`bindings_ms`**, **`churn_ms`**, and **`total_ms`** to `fixtures/benchmark/perf-baseline.json`. Other `IndexPerformanceReport` fields (`module_cycles_ms`, `re_export_chains_ms`, `heritage_ms`, …) appear in `--performance` JSON only — not baseline-gated. 4. **Local / scheduled only** — run before perf-sensitive PRs; [`.github/workflows/perf-baseline.yml`](../.github/workflows/perf-baseline.yml) fires weekly + `workflow_dispatch` for drift visibility. **Not** on the PR CI path (6 min × 3 runs + bimodal GHA runners → flaky merge gate). ### Why this is separate from `src/benchmark.ts` diff --git a/docs/glossary.md b/docs/glossary.md index 3111ea6e..98645a21 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -262,7 +262,7 @@ Number of edges _out of_ a file — `COUNT(*) FROM dependencies WHERE from_path ### `file_churn` (table) -Per-file git churn metrics refreshed on **every index pass** (incremental scoped recompute; idle HEAD cache). Git via `ingestFileChurnFromGit`; non-git via **`codemap ingest-churn`** or config **`churn.file`**. Config: `churn.halfLifeDays`, `churn.since` / `--churn-since`. Powers **`churn-complexity-hotspots`** (file or symbol grain, normalized score) — distinct from outcome alias **`hotspots`** → `fan-in`. +Per-file git churn metrics refreshed on **every index pass** (incremental scoped recompute; idle HEAD cache). Git via `ingestFileChurnFromGit`; non-git via **`codemap ingest-churn`** or config **`churn.file`**. Config: `churn.halfLifeDays`, `churn.since` / `--churn-since`. Column **`churn_trend`**: `accelerating` \| `stable` \| `cooling` when enough history exists. Powers **`churn-complexity-hotspots`** (file or symbol grain, normalized score) — distinct from outcome alias **`hotspots`** → `fan-in`. ### `files` (table) diff --git a/docs/golden-queries.md b/docs/golden-queries.md index af9b376c..2d9f79b2 100644 --- a/docs/golden-queries.md +++ b/docs/golden-queries.md @@ -78,6 +78,10 @@ Some bundled recipes add optional **`reason`** (TEXT) and **`evidence_json`** (T `coverage-confirmed-dead` adds **`confidence`** (`high` \| `medium`) on each row — **`high`** when static dead and ingested `coverage_pct = 0`; **`medium`** when static dead but the symbol has no ingested coverage row. Also **`reason`**, **`caller_count`**. Goldens: `coverage-confirmed-dead` (post-ingest mix) and `coverage-confirmed-dead-no-ingest` (`preSetup: clear-coverage`, `everyRowFieldEquals` on `confidence: medium`). +### Churn / hotspot columns (`churn-complexity-hotspots` recipe) + +`churn-complexity-hotspots` ranks indexed files or symbols by git churn × cyclomatic complexity. File grain (default): **`file_path`**, **`max_complexity`**, **`weighted_commits`**, **`commit_count`**, **`churn_trend`**, **`hotspot_score`**, **`hotspot_score_normalized`**. Symbol grain (`by_symbol=true`): per-symbol **`name`**, **`kind`**, **`line_start`**, **`cyclomatic_complexity`** plus file churn fields. Goldens: `churn-complexity-hotspots`, `churn-complexity-hotspots-by-symbol` (fixture churn seeded via setup step **`seed-file-churn`**). + ### Duplication columns (`duplicates` recipe) `duplicates` returns one row per function-shaped symbol in a **`body_hash`** collision group: **`name`**, **`kind`**, **`file_path`**, **`line_start`**, **`line_end`**, **`body_hash`**, **`body_line_count`**, **`duplicate_count`** (in-scope group size after `path_prefix` / `min_body_lines`). Substrate column **`symbols.body_hash`** is populated at index for function-shaped symbols (`function`, `method`, `getter`, `setter`) when `body_line_count >= 2`. Goldens: `duplicates` (includes `src/bench/duplicate-body-{a,b}.ts` pair). False positives possible when unrelated functions share control-flow skeleton or sync vs async/generator bodies match — triage with `snippet`. Recipe caps at **50 rows** (no truncation marker). diff --git a/docs/plans/churn-complexity-hotspots.md b/docs/plans/churn-complexity-hotspots.md deleted file mode 100644 index abc6f09b..00000000 --- a/docs/plans/churn-complexity-hotspots.md +++ /dev/null @@ -1,50 +0,0 @@ -# Churn × complexity hotspots — plan - -> **Status:** shipped (complete) · **Priority:** P2 -> -> **Roadmap:** [§ Core substrate & platform](../roadmap.md#core-substrate--platform) - ---- - -## Shipped - -| Layer | Delivered | -| ------------ | ---------------------------------------------------------------------------------------------- | -| **Moat B** | `file_churn` table; git ingest every index pass; incremental scoped recompute; idle HEAD cache | -| **Non-git** | `codemap ingest-churn ` + config `churn.file` fallback | -| **Moat A** | Recipe `churn-complexity-hotspots` — file (default) or symbol (`by_symbol=true`) grain | -| **Scores** | `hotspot_score` + `hotspot_score_normalized` (0–100 vs result-set max) | -| **Config** | `churn.halfLifeDays`, `churn.since`, `churn.file`; CLI `--churn-since` | -| **Trend** | `churn_trend`: accelerating \| stable \| cooling | -| **Agent AX** | Rule trigger; `refactor-priority` + `refactor` intent cards; golden + agent-eval | -| **Perf** | `churn_ms` in `--performance` JSON + perf baseline gate | -| **Alias** | `hotspots` → `fan-in` unchanged (Moat-A cap) | - -### Verification - -```bash -bun test src/application/churn-ingest.test.ts src/application/ingest-churn-run.test.ts src/file-churn.test.ts -bun run test:golden -bun src/index.ts ingest-churn fixtures/minimal/file-churn-seed.json -bun src/index.ts query --recipe churn-complexity-hotspots --params by_symbol=true --json -``` - ---- - -## Intentionally not shipped - -| Item | Why | -| ------------------------------------- | -------------------------------------------------------- | -| **Repurpose `hotspots` alias** | Moat-A alias cap; recipe is the churn×complexity surface | -| **Cross-repo absolute normalization** | `hotspot_score_normalized` is corpus-relative per query | - ---- - -## Key touchpoints - -| File | Role | -| ---------------------------------------------------------------------------------------------------------- | --------------------------------------------- | -| [`src/application/churn-ingest.ts`](../../src/application/churn-ingest.ts) | Git ingest, trend, incremental scope, refresh | -| [`src/application/ingest-churn-run.ts`](../../src/application/ingest-churn-run.ts) | JSON import | -| [`src/cli/cmd-ingest-churn.ts`](../../src/cli/cmd-ingest-churn.ts) | `ingest-churn` verb | -| [`templates/recipes/churn-complexity-hotspots.sql`](../../templates/recipes/churn-complexity-hotspots.sql) | Recipe | diff --git a/docs/plans/substrate-extraction.md b/docs/plans/substrate-extraction.md index 1d37e3c8..c34aaa79 100644 --- a/docs/plans/substrate-extraction.md +++ b/docs/plans/substrate-extraction.md @@ -393,7 +393,7 @@ New recipe candidates: `strict-mode-audit`; `missing-types-fields`; `monorepo-pa ### Tier 11 — Metrics expansion — **PARTIAL (2026-05-15)** -**Shipped:** `file_metrics` + per-symbol metric columns (see [`glossary.md`](../glossary.md)). **Open:** churn/git metrics → [`churn-complexity-hotspots.md`](./churn-complexity-hotspots.md). +**Shipped:** `file_metrics` + per-symbol metric columns (see [`glossary.md`](../glossary.md)). **Shipped:** `file_churn` + **`churn-complexity-hotspots`** recipe (see [architecture § `file_churn`](../architecture.md#file_churn--git-churn-metrics-per-indexed-file-strict)). ### Tier 12 — Module-graph topology — **PARTIAL (2026-05-15)** diff --git a/docs/roadmap.md b/docs/roadmap.md index e1d7362e..7513e951 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -111,7 +111,7 @@ Predicate-as-API only — enrich row shape and audit deltas; no standalone pass/ - [ ] **`history` table** (deferred — revisit-triggered) — temporal queries: "when did symbol X get `@deprecated`?", "coverage trend over last 50 commits", "files that became dead this week". `audit --base ` covers the most-common temporal question (PR-scoped diff) without schema growth, so the table earns its place only when bigger questions emerge. Two shapes (per-commit snapshots ~N × DB size; append-only event log heavier CTE walks); both pay an N-reindexes backfill cost (~30s per reindex). **Revisit triggers:** two consumers ship `jq`-based "audit-runs-over-time" workflows, OR `query_baselines` evolution becomes a recurring agent need. - [ ] **`codemap audit` verdict + thresholds** (v1.x) — `verdict: "pass" | "warn" | "fail"` driven by an `audit.deltas[].{added_max, action}` field on the config object (`.codemap/config.{ts,js,json}`). Triggers: two consumers ship `jq`-based threshold scripts with similar shapes, OR one consumer asks with a concrete config sketch. Until then, raw deltas + consumer-side `jq` is the CI exit-code idiom. **Likely accelerant:** the Marketplace Action (next item) shipping is the most plausible path to firing the trigger — once `- uses: stainless-code/codemap@v1` is the dominant CI path, real `jq` threshold scripts will surface. - [ ] **GitHub Marketplace Action — publish + listing finish** — core Action implementation is in-tree: root `action.yml`, `query --ci`, `audit --format sarif` / `--ci`, package-manager detection, dogfood smoke, and opt-in `pr-comment` summary renderer have shipped. Remaining work is the release/listing slice: `MARKETPLACE.md`, `v1.0.0` / floating `v1` tags, Marketplace setup, sacrificial-repo smoke, and making `action-smoke` blocking once the Action tag exists. Action version stream is independent of CLI version (`package.json` currently drives CLI/npm version; Action publishes at its own `v1.0.0`). Plan: [`plans/github-marketplace-action.md`](./plans/github-marketplace-action.md). Effort: S. -- [x] **Churn × complexity hotspots** — `file_churn` (git ingest every index + `ingest-churn` / `churn.file` fallback), recipe **`churn-complexity-hotspots`** (file/symbol grain, normalized score, trend). Alias `hotspots` → `fan-in`. Plan: [`plans/churn-complexity-hotspots.md`](./plans/churn-complexity-hotspots.md). +- [x] **Churn × complexity hotspots** — `file_churn` (git ingest every index + `ingest-churn` / `churn.file` fallback), recipe **`churn-complexity-hotspots`** (file/symbol grain, normalized score, trend). Alias `hotspots` → `fan-in`. Contract: [architecture § `file_churn`](./architecture.md#file_churn--git-churn-metrics-per-indexed-file-strict), [glossary § file_churn](./glossary.md#file_churn). - [x] **AST-hash duplication** — `symbols.body_hash` (canonical body AST, identifiers → `$id`, literals → kind, absent returns → `Literal:nullish`; function-shaped symbols; skip `body_line_count < 2`) + partial index + bundled `duplicates` recipe (per-symbol rows, CTE `GROUP BY`). **Different shape from token-level suffix-array dupes.** Contract: [architecture § `symbols` table](./architecture.md#symbols--functions-constants-classes-interfaces-types-enums-strict), [glossary § body_hash](./glossary.md#symbolsbody_hash--structural-duplicate-bodies). Effort: M. - [ ] **Falsifiable benchmark CI on named external fixtures** — structural-cost A/B (indexed queries vs `find` + `grep` + `Read`-loop discovery) on zod, fastify, vue-core, next.js. Numbers land in [`docs/benchmark.md`](./benchmark.md); headline figures surface in `MARKETPLACE.md` only after external runs land. Harness: [benchmark § Agent eval harness](./benchmark.md#agent-eval-harness) + external fixture extension; pair with **Agent eval: quality × tokens × wall** for scored completion metrics. **Partial:** manual [`.github/workflows/agent-eval-external.yml`](../.github/workflows/agent-eval-external.yml) for in-repo fixture paths (not zod/fastify/nightly). Effort: M. **Self-index regression guardrail shipped** (#96): `bun run check:perf-baseline` + weekly scheduled workflow (demoted from PR hard gate — GHA runner variance). - [ ] **In-repo test bench scale (optional)** — if `fixtures/minimal` outgrows one corpus: add committed `fixtures/bench/` or rename `minimal`→`bench`. Harness map: [`testing-coverage.md`](./testing-coverage.md), [`fixtures/README.md`](../fixtures/README.md). diff --git a/docs/testing-coverage.md b/docs/testing-coverage.md index a9c9e8c0..423c40b1 100644 --- a/docs/testing-coverage.md +++ b/docs/testing-coverage.md @@ -54,30 +54,31 @@ Every `templates/recipes/.sql` has **≥1** scenario in `fixtures/golden/sce Recipe goldens prove query _behavior_; these scenarios pin **persisted rows** on the minimal corpus. Pin-down ids enforced in CI: `SUBSTRATE_SCENARIO_BY_TABLE` in `scripts/query-golden-coverage-matrix.test.mjs` (plus every bundled recipe id). `index-table-stats` and `call-resolution-stats` are additional aggregate/residual scenarios on the same corpus. -| Table | Scenario id | -| ------------------------------------- | ------------------------------------------------------- | -| Aggregate counts (all indexed tables) | `index-table-stats` | -| `meta` | `meta-fts5-enabled`, `call-resolution-stats` (residual) | -| `source_fts` | `source-fts-row-count` | -| `file_metrics` | `file-metrics-complexity-fixture` | -| `scopes` | `scopes-product-card` | -| `references` | `references-product-card-perms` | -| `bindings` | `bindings-createClient` | -| `import_specifiers` | `import-specifiers-consumer` | -| `async_calls` | `async-calls-prefetch` | -| `decorators` | `decorators-sealed` | -| `dynamic_imports` | `dynamic-imports-prefetch` | -| `module_cycles` | `module-cycles-cache-store` | -| `re_export_chains` | `re-export-chains-product-card` | -| `runtime_markers` | `runtime-markers-env` | -| `function_params` | `function-params-createClient` | -| `boundary_rules` | `boundary-rules-ui-no-api` | -| `unresolved_calls` | `unresolved-call-sites` | -| `calls` (resolution) | `calls-createClient-resolved` | -| `try_catch` | `try-catch-rethrow-heuristics` | -| `jsdoc_tags` | `jsdoc-tags-createClient` | -| `suppressions` | `suppressions-orphan` | -| `coverage` | `coverage-rows-after-ingest` (+ killer recipes) | +| Table | Scenario id | +| ------------------------------------- | -------------------------------------------------------- | +| Aggregate counts (all indexed tables) | `index-table-stats` | +| `meta` | `meta-fts5-enabled`, `call-resolution-stats` (residual) | +| `source_fts` | `source-fts-row-count` | +| `file_metrics` | `file-metrics-complexity-fixture` | +| `file_churn` | `churn-complexity-hotspots` (seed via `seed-file-churn`) | +| `scopes` | `scopes-product-card` | +| `references` | `references-product-card-perms` | +| `bindings` | `bindings-createClient` | +| `import_specifiers` | `import-specifiers-consumer` | +| `async_calls` | `async-calls-prefetch` | +| `decorators` | `decorators-sealed` | +| `dynamic_imports` | `dynamic-imports-prefetch` | +| `module_cycles` | `module-cycles-cache-store` | +| `re_export_chains` | `re-export-chains-product-card` | +| `runtime_markers` | `runtime-markers-env` | +| `function_params` | `function-params-createClient` | +| `boundary_rules` | `boundary-rules-ui-no-api` | +| `unresolved_calls` | `unresolved-call-sites` | +| `calls` (resolution) | `calls-createClient-resolved` | +| `try_catch` | `try-catch-rethrow-heuristics` | +| `jsdoc_tags` | `jsdoc-tags-createClient` | +| `suppressions` | `suppressions-orphan` | +| `coverage` | `coverage-rows-after-ingest` (+ killer recipes) | Core graph tables (`files`, `symbols`, `imports`, …) are covered by many existing SQL/recipe scenarios — see `fixtures/golden/scenarios.json`. diff --git a/src/application/churn-ingest.test.ts b/src/application/churn-ingest.test.ts index 0bb4f26e..bc893b4e 100644 --- a/src/application/churn-ingest.test.ts +++ b/src/application/churn-ingest.test.ts @@ -4,9 +4,21 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { closeDb, createSchema, insertFile } from "../db"; +import { + closeDb, + createSchema, + getMeta, + insertFile, + META_CHURN_CONFIG_FINGERPRINT, + META_CHURN_INDEXED_COMMIT, + setMeta, +} from "../db"; import { openCodemapDatabase } from "../sqlite-db"; -import { computeChurnTrend, ingestFileChurnFromGit } from "./churn-ingest"; +import { + computeChurnTrend, + ingestFileChurnFromGit, + refreshFileChurn, +} from "./churn-ingest"; let projectRoot: string; @@ -220,3 +232,113 @@ describe("ingestFileChurnFromGit", () => { } }); }); + +describe("refreshFileChurn", () => { + beforeEach(() => { + projectRoot = mkdtempSync(join(tmpdir(), "codemap-churn-refresh-")); + git(["init", "-q", "-b", "main"]); + git(["config", "user.email", "t@example.com"]); + git(["config", "user.name", "T"]); + git(["config", "commit.gpgsign", "false"]); + mkdirSync(join(projectRoot, "src"), { recursive: true }); + writeFileSync(join(projectRoot, "src/a.ts"), "export const a = 1;\n"); + commitAll("seed"); + }); + + afterEach(() => { + rmSync(projectRoot, { recursive: true, force: true }); + }); + + it("idle skip requires populated file_churn even when HEAD meta matches", () => { + const head = spawnSync("git", ["rev-parse", "HEAD"], { + cwd: projectRoot, + encoding: "utf-8", + }) + .stdout.toString() + .trim(); + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/a.ts", + content_hash: "a", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + setMeta(db, META_CHURN_INDEXED_COMMIT, head); + setMeta(db, META_CHURN_CONFIG_FINGERPRINT, "90|"); + + const result = refreshFileChurn(db, { + projectRoot, + mode: "idle", + halfLifeDays: 90, + since: null, + quiet: true, + }); + expect(result.reason).not.toBe("skipped: HEAD unchanged"); + expect(result.ok).toBe(true); + expect(result.rowCount).toBeGreaterThan(0); + } finally { + closeDb(db); + } + }); + + it("idle skip when HEAD, rows, and config fingerprint match", () => { + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/a.ts", + content_hash: "a", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + ingestFileChurnFromGit(db, { projectRoot, quiet: true }); + const result = refreshFileChurn(db, { + projectRoot, + mode: "idle", + halfLifeDays: 90, + since: null, + quiet: true, + }); + expect(result.reason).toBe("skipped: HEAD unchanged"); + expect(result.elapsedMs).toBeLessThan(50); + } finally { + closeDb(db); + } + }); + + it("deletions mode stamps meta without running git log", () => { + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/a.ts", + content_hash: "a", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + ingestFileChurnFromGit(db, { projectRoot, quiet: true }); + const result = refreshFileChurn(db, { + projectRoot, + mode: "deletions", + halfLifeDays: 90, + since: null, + quiet: true, + }); + expect(result.reason).toBe("deletions: churn pruned via CASCADE"); + expect(getMeta(db, META_CHURN_INDEXED_COMMIT)).toBeTruthy(); + } finally { + closeDb(db); + } + }); +}); diff --git a/src/application/churn-ingest.ts b/src/application/churn-ingest.ts index ee4df153..8735029d 100644 --- a/src/application/churn-ingest.ts +++ b/src/application/churn-ingest.ts @@ -5,6 +5,7 @@ import { relative, resolve } from "node:path"; import { getMeta, mergeFileChurnForPaths, + META_CHURN_CONFIG_FINGERPRINT, META_CHURN_INDEXED_COMMIT, pruneFileChurnOrphans, replaceFileChurn, @@ -75,7 +76,7 @@ function indexedToGitPath(filePath: string, projectPrefix: string): string { export type ChurnTrend = "accelerating" | "stable" | "cooling"; -export type ChurnRefreshMode = "full" | "incremental" | "idle"; +export type ChurnRefreshMode = "full" | "incremental" | "idle" | "deletions"; export interface ChurnIngestResult { ok: boolean; @@ -216,9 +217,39 @@ function resolveGitHead(projectRoot: string): string | null { return head.length > 0 ? head : null; } -function stampChurnCommit(db: CodemapDatabase, projectRoot: string): void { +function churnConfigFingerprint( + halfLifeDays: number, + since: string | null, +): string { + return `${halfLifeDays}|${since ?? ""}`; +} + +function stampChurnMeta( + db: CodemapDatabase, + projectRoot: string, + halfLifeDays: number, + since: string | null, +): void { const head = resolveGitHead(projectRoot); if (head) setMeta(db, META_CHURN_INDEXED_COMMIT, head); + setMeta( + db, + META_CHURN_CONFIG_FINGERPRINT, + churnConfigFingerprint(halfLifeDays, since), + ); +} + +function canIdleSkipChurn( + db: CodemapDatabase, + head: string | null, + prevHead: string | null, + halfLifeDays: number, + since: string | null, +): boolean { + if (!head || !prevHead || head !== prevHead) return false; + if (countFileChurn(db) === 0) return false; + const fp = churnConfigFingerprint(halfLifeDays, since); + return getMeta(db, META_CHURN_CONFIG_FINGERPRINT) === fp; } function tryConfigChurnFallback( @@ -365,7 +396,7 @@ export function ingestFileChurnFromGit( `[churn] file_churn ${merge ? "merged" : "populated"}: ${rows.length} files (${rowCount} total)`, ); } - stampChurnCommit(db, projectRoot); + stampChurnMeta(db, projectRoot, halfLife, since); return finish({ ok: true, rowCount }); } @@ -385,10 +416,15 @@ export function refreshFileChurn( const quiet = options?.quiet ?? false; const mode = options?.mode ?? "full"; const projectRoot = options?.projectRoot ?? getProjectRoot(); + const halfLifeDays = options?.halfLifeDays ?? getChurnHalfLifeDays(); + const since = options?.since !== undefined ? options.since : getChurnSince(); const head = resolveGitHead(projectRoot); - const prevHead = getMeta(db, META_CHURN_INDEXED_COMMIT); + const prevHead = getMeta(db, META_CHURN_INDEXED_COMMIT) ?? null; - if (mode === "idle" && head && prevHead && head === prevHead) { + if ( + mode === "idle" && + canIdleSkipChurn(db, head, prevHead, halfLifeDays, since ?? null) + ) { const rowCount = countFileChurn(db); return { ok: true, @@ -398,10 +434,21 @@ export function refreshFileChurn( }; } + if (mode === "deletions") { + const rowCount = countFileChurn(db); + stampChurnMeta(db, projectRoot, halfLifeDays, since ?? null); + return { + ok: true, + rowCount, + elapsedMs: Math.round(performance.now() - t0), + reason: "deletions: churn pruned via CASCADE", + }; + } + const base = { projectRoot, - halfLifeDays: options?.halfLifeDays ?? getChurnHalfLifeDays(), - since: options?.since !== undefined ? options.since : getChurnSince(), + halfLifeDays, + since: since ?? null, quiet, }; diff --git a/src/application/ingest-churn-run.ts b/src/application/ingest-churn-run.ts index bfbee27d..01e2d7f0 100644 --- a/src/application/ingest-churn-run.ts +++ b/src/application/ingest-churn-run.ts @@ -3,6 +3,7 @@ import { existsSync, readFileSync } from "node:fs"; import { isAbsolute, resolve } from "node:path"; import { + META_CHURN_CONFIG_FINGERPRINT, META_CHURN_INDEXED_COMMIT, pruneFileChurnOrphans, replaceFileChurn, @@ -25,6 +26,16 @@ export interface IngestChurnRunError { export type IngestChurnRunResult = IngestChurnRunOk | IngestChurnRunError; +/** Strip inherited GIT_* so subprocess targets the project repo. */ +function gitSpawnEnv(): NodeJS.ProcessEnv { + const e: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(process.env)) { + if (k.startsWith("GIT_")) continue; + e[k] = v; + } + return e; +} + function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { if (!Array.isArray(raw)) { throw new TypeError("churn JSON must be an array of file_churn rows"); @@ -39,6 +50,9 @@ function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { if (typeof file_path !== "string" || file_path.length === 0) { throw new TypeError("file_path must be a non-empty string"); } + if (rows.some((existing) => existing.file_path === file_path)) { + throw new TypeError(`duplicate file_path in churn JSON: ${file_path}`); + } rows.push({ file_path, commit_count: Number(r.commit_count) || 0, @@ -108,10 +122,15 @@ export function ingestChurnFromJsonFile( pruneFileChurnOrphans(db); const headResult = spawnSync("git", ["rev-parse", "HEAD"], { cwd: options.projectRoot, + env: gitSpawnEnv(), }); if (headResult.status === 0) { const head = headResult.stdout.toString().trim(); - if (head) setMeta(db, META_CHURN_INDEXED_COMMIT, head); + if (head) { + setMeta(db, META_CHURN_INDEXED_COMMIT, head); + // JSON ingest has no half-life/since knobs — fingerprint marks manual import. + setMeta(db, META_CHURN_CONFIG_FINGERPRINT, "json|"); + } } return { diff --git a/src/application/run-index.ts b/src/application/run-index.ts index 57f6a8fd..77ad45ca 100644 --- a/src/application/run-index.ts +++ b/src/application/run-index.ts @@ -266,8 +266,7 @@ async function runCodemapIndexBody( stats: run.stats, }; } else if (diff.deleted.length > 0) { - churnMode = "incremental"; - churnChangedPaths = diff.deleted; + churnMode = "deletions"; deleteFilesFromIndex(db, diff.deleted, quiet); const callScope = expandHeritageResolveScope(db, diff.deleted); if (callScope.length > 0) { diff --git a/src/cli/bootstrap.ts b/src/cli/bootstrap.ts index 85b1b275..4529025f 100644 --- a/src/cli/bootstrap.ts +++ b/src/cli/bootstrap.ts @@ -79,6 +79,9 @@ Apply (substrate-shaped fix executor; diff-json row contract): Coverage ingest (Istanbul JSON or LCOV from any test runner): codemap ingest-coverage [--json] # path = file or dir; format auto-detected +Churn ingest (precomputed file_churn JSON for non-git repos or fixtures): + codemap ingest-churn [--json] + Other: codemap unlock [--force] Remove stale cross-process index lock codemap version @@ -150,6 +153,7 @@ export function validateIndexModeArgs(rest: string[]): void { if (rest[0] === "affected") return; if (rest[0] === "apply") return; if (rest[0] === "ingest-coverage") return; + if (rest[0] === "ingest-churn") return; if (rest[0] === "pr-comment") return; if (rest[0] === "trace") return; if (rest[0] === "explore") return; diff --git a/src/cli/cmd-ingest-churn.ts b/src/cli/cmd-ingest-churn.ts index 0779de6c..4a865122 100644 --- a/src/cli/cmd-ingest-churn.ts +++ b/src/cli/cmd-ingest-churn.ts @@ -1,5 +1,5 @@ import { ingestChurnFromJsonFile } from "../application/ingest-churn-run"; -import { closeDb, openDb } from "../db"; +import { closeDb, createSchema, openDb } from "../db"; import { bootstrapCodemap } from "./bootstrap-codemap"; interface IngestChurnOpts { @@ -14,9 +14,11 @@ export function printIngestChurnCmdHelp(): void { console.log(`Usage: codemap ingest-churn [--json] Import precomputed git churn metrics into \`file_churn\` for non-git -repositories or CI fixtures. JSON must be an array of rows matching the -\`file_churn\` schema (see docs/architecture.md). Only paths present in -the index are kept; unindexed paths are skipped. +repositories or CI fixtures. JSON must be an array of objects with +\`file_path\`, \`commit_count\`, \`weighted_commits\`, \`lines_added\`, +\`lines_removed\`, optional \`last_commit_at\` / \`churn_trend\`, and +\`computed_at\`. Run \`codemap\` (index) first — only indexed paths are +kept; unindexed paths are skipped. Args: Path to JSON file (relative to project root or absolute) @@ -79,6 +81,15 @@ export async function runIngestChurnCmd(opts: IngestChurnOpts): Promise { await bootstrapCodemap(opts); const db = openDb(); try { + createSchema(db); + const indexedCount = + db.query<{ n: number }>("SELECT COUNT(*) AS n FROM files").get()?.n ?? 0; + if (indexedCount === 0) { + console.error( + "codemap ingest-churn: no indexed files — run `codemap` or `codemap --full` first", + ); + process.exit(1); + } const result = ingestChurnFromJsonFile(db, { projectRoot: opts.root, path: opts.path, diff --git a/src/db.ts b/src/db.ts index 2ff1638d..573eff7c 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1750,6 +1750,8 @@ export function replaceFileChurn(db: CodemapDatabase, rows: FileChurnRow[]) { /** `meta` key: last `HEAD` when `file_churn` was refreshed (idle skip). */ export const META_CHURN_INDEXED_COMMIT = "churn_indexed_commit"; +/** `meta` key: `halfLifeDays|since` fingerprint for idle skip after config changes. */ +export const META_CHURN_CONFIG_FINGERPRINT = "churn_config_fingerprint"; /** Replace churn rows for `scopePaths` only; other paths are left unchanged. */ export function mergeFileChurnForPaths( diff --git a/src/worker-pool.dist.test.ts b/src/worker-pool.dist.test.ts index aae6c64f..e33a05b9 100644 --- a/src/worker-pool.dist.test.ts +++ b/src/worker-pool.dist.test.ts @@ -52,6 +52,8 @@ describe("node dist --full exit delay", () => { ...process.env, CODEMAP_ROOT: minimalRoot, CODEMAP_STATE_DIR: stateDir, + // Prevent git from walking above the fixture (parent monorepo history). + GIT_CEILING_DIRECTORIES: minimalRoot, }, stdout: "ignore", stderr: "pipe", diff --git a/templates/agent-content/rule/00-full.md b/templates/agent-content/rule/00-full.md index 94cca3be..a1df3b00 100644 --- a/templates/agent-content/rule/00-full.md +++ b/templates/agent-content/rule/00-full.md @@ -6,7 +6,7 @@ alwaysApply: true > **STOP.** Before you call Grep, Glob, SemanticSearch, or Read to answer a **structural** question about **this project** — query the Codemap SQLite index first. -This project is indexed by **Codemap** — a local SQLite database (default **`.codemap/index.db`**) of structure: `files`, `symbols`, `imports`, `exports`, `dependencies`, `calls`, `components`, `markers`, `type_members`, `type_heritage`, `import_specifiers`, `scopes`, `references`, `bindings`, `function_params`, `dynamic_imports`, `jsx_elements`, `jsx_attributes`, `async_calls`, `try_catch`, `decorators`, `jsdoc_tags`, `runtime_markers`, `test_suites`, `re_export_chains`, `module_cycles`, `file_metrics`, `css_variables`, `css_classes`, `css_keyframes`, `suppressions`, `boundary_rules`, and (after `codemap ingest-coverage `) `coverage`. Full DDL: `codemap query --json "SELECT sql FROM sqlite_schema WHERE type='table'"` or MCP resource `codemap://schema`. +This project is indexed by **Codemap** — a local SQLite database (default **`.codemap/index.db`**) of structure: `files`, `symbols`, `imports`, `exports`, `dependencies`, `calls`, `components`, `markers`, `type_members`, `type_heritage`, `import_specifiers`, `scopes`, `references`, `bindings`, `function_params`, `dynamic_imports`, `jsx_elements`, `jsx_attributes`, `async_calls`, `try_catch`, `decorators`, `jsdoc_tags`, `runtime_markers`, `test_suites`, `re_export_chains`, `module_cycles`, `file_metrics`, `file_churn` (every index pass or `codemap ingest-churn`), `css_variables`, `css_classes`, `css_keyframes`, `suppressions`, `boundary_rules`, and (after `codemap ingest-coverage `) `coverage`. Full DDL: `codemap query --json "SELECT sql FROM sqlite_schema WHERE type='table'"` or MCP resource `codemap://schema`. ## How to query diff --git a/templates/recipes/churn-complexity-hotspots.md b/templates/recipes/churn-complexity-hotspots.md index 947ff179..2b88abd3 100644 --- a/templates/recipes/churn-complexity-hotspots.md +++ b/templates/recipes/churn-complexity-hotspots.md @@ -21,7 +21,7 @@ params: Files or symbols ranked by **git churn × cyclomatic complexity**. Distinct from the outcome alias `codemap hotspots` (import **fan-in** via `fan-in` recipe). -Populated on every index pass from `git log --numstat` (config `churn.halfLifeDays`, `churn.since` / `--churn-since`). Non-git repos: `codemap ingest-churn ` or config `churn.file`. +Populated on every index pass from git history (config `churn.halfLifeDays`, `churn.since` / `--churn-since`). Non-git repos: `codemap ingest-churn ` or config `churn.file`. ```bash codemap query --recipe churn-complexity-hotspots @@ -30,4 +30,8 @@ codemap query --recipe churn-complexity-hotspots --params by_symbol=true codemap ingest-churn churn-metrics.json ``` -`hotspot_score` = `weighted_commits × complexity`. `hotspot_score_normalized` is 0–100 vs the corpus max in the result set. Triage with `snippet` before large refactors. +`hotspot_score` = `weighted_commits × complexity`. `hotspot_score_normalized` is 0–100 vs the corpus max in the result set. + +**Output columns:** file grain — `file_path`, `max_complexity`, `weighted_commits`, `commit_count`, `churn_trend`, scores. Symbol grain (`by_symbol=true`) — per-symbol `name`, `kind`, `line_start`, `cyclomatic_complexity`, plus file churn fields. `churn_trend` is `accelerating`, `stable`, or `cooling` when enough history exists. + +Triage with `snippet` before large refactors. From 71ddcd50d1e6c7bbe67270c3863a87c3ffcb8e16 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 10 Jun 2026 17:27:11 +0300 Subject: [PATCH 3/8] docs(agents): churn-hotspot AX parity across MCP, skill, and recipe actions Wire churn-complexity-hotspots into MCP playbook and recipe chains, add churn column guidance to rule/skill shards, per-row review actions on the recipe, and cross-links from refactor-risk recipes. --- templates/agent-content/mcp-instructions.md | 61 ++++++++++--------- templates/agent-content/rule/00-full.md | 2 + .../agent-content/skill/10-recipes-context.md | 3 +- .../agent-content/skill/40-query-patterns.md | 4 +- .../recipes/churn-complexity-hotspots.md | 4 ++ templates/recipes/high-complexity-untested.md | 2 + templates/recipes/refactor-risk-ranking.md | 2 + 7 files changed, 45 insertions(+), 33 deletions(-) diff --git a/templates/agent-content/mcp-instructions.md b/templates/agent-content/mcp-instructions.md index 44dd3cb7..41af231f 100644 --- a/templates/agent-content/mcp-instructions.md +++ b/templates/agent-content/mcp-instructions.md @@ -29,40 +29,41 @@ Key fields: `pending_sync` (watcher debounce queue or in-flight reindex), `commi ## Common tasks -| Goal | MCP tool | Recipe twin (`query_recipe`) | -| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Exact symbol lookup | **`show`** (`name`, optional `in`) | `find-symbol-definitions` | -| Field-qualified symbol discovery | **`show`** or **`snippet`** (`query` with `kind:` / `name:` / `path:` / `in:` + free text) | `find-symbol-by-kind` for kind-heavy patterns; CLI `codemap show --query '…' --print-sql` to inspect generated SQL (no MCP `print_sql` arg) | -| Kind / pattern lookup | **`query_recipe`** | `find-symbol-by-kind` | -| Source at symbol | **`snippet`** | same rows as `show` + disk text | -| Blast radius | **`impact`** (`target`, `direction`, `via`, `depth`) | `fan-in` for file hubs; symbol call graph via SQL or `impact` | -| Call path + snippets | **`trace`** (`from`, `to`, `via?`, `max_depth?`, `budget_chars?`) — adaptive snippet caps 15k/10k/6k when omitted | `call-path` | -| Type extends / implements chain | **`query_recipe`** | `type-ancestors`, `type-descendants` (`file_path` when homonyms; on `type-descendants` also scopes output to that file) | -| Multi-symbol survey | **`explore`** (`names`, `depth?`, `kind?`, `budget_chars?`) — row cap always adaptive (500/250/125); snippets 15k/10k/6k when `budget_chars` omitted | `symbol-neighborhood` (once per name) | -| One-hop symbol card | **`node`** (`name`, `kind?`, `in?`, `include_snippets?`, `budget_chars?`) — adaptive snippet caps when snippets enabled | `show` + `symbol-neighborhood` with `depth=1` | -| Affected tests | **`affected`** (`paths?`, `changed_since?`, `test_glob?`, `max_depth?`) | `affected-tests` (RS-delimit multiple paths in `query_recipe` params) | -| CI / SARIF | **`query_recipe`** + `format: "sarif"` | `deprecated-symbols`, `boundary-violations`, … | -| GitLab Code Quality | **`query_recipe`** + `format: "codeclimate"` | `boundary-violations`, … — locatable rows only; flat `minor` severity | -| CI badge / issue count | **`query_recipe`** + `format: "badge"` (+ `badge_style: "json"` for gates) | presentation only — triage via JSON rows / `--summary` | -| Ad-hoc SQL | **`query`** | — | -| N statements / one round-trip | **`query_batch`** | **`codemap query batch`** | -| Index freshness (index-level) | **`context`** (`index_freshness`) + tool metadata above | — | -| Per-file staleness | **`validate`** | — | -| Drift vs baseline | **`audit`** (`baseline_prefix` and/or per-delta `baselines`) or **`query`** / **`query_recipe`** + `baseline` (one-shot row diff vs `query_baselines`; incompatible with non-`json` `format` / `group_by`) | save via **`save_baseline`**; `summary: true` → count-only diff | -| PR merge-base drift | **`audit`** `base: ` (git committish; sha-keyed cache) | `attribution: introduced` (branch-new) \| `inherited` (pre-existing at merge base) on each `added` row; `jq '.deltas.deprecated.added[] \| select(.attribution == "introduced")'`; `summary: true` → `added_introduced` / `added_inherited` | -| Load coverage data | **`ingest_coverage`** (`path`, optional `runtime` for V8 dirs; auto-detects Istanbul `.json` / LCOV `.info`) | enables `worst-covered-exports`, `files-by-coverage`, `untested-and-dead`, **`coverage-confirmed-dead`** (`confidence: high`); **`high-crap-score`** measured override | -| Complex + undertested (no ingest) | **`query_recipe`** `high-crap-score` | graph-estimated 85/40/0% tiers — parse `coverage_source` before CI gates; prefer after **`ingest_coverage`** when possible | -| High-judgment recipe triage | **`query_recipe`** `unimported-exports`, `boundary-violations`, `deprecated-symbols` | rows include `reason` / `evidence_json` — cite before **`apply`** or deletion | -| Apply recipe diff rows | **`apply`** (`recipe`, `params?`, `dry_run?`, `yes?`, `force?`, `until_empty?`, `max_passes?`, `commit_message?`) | recipe must emit `{file_path, line_start, before_pattern, after_pattern}` rows; `yes: true` required for writes; non-`auto_fixable` recipes need `force: true` | -| Apply agent/codemod rows | **`apply_rows`** (`rows`, `dry_run?`, `yes?`) | same row contract; bypasses recipe `auto_fixable` / allowlist gates | -| Apply unified diff text | **`apply_diff_input`** (`diff_text`, `dry_run?`, `yes?`, `commit_message?`) | parses git-style hunks; same executor as `apply_rows` | +| Goal | MCP tool | Recipe twin (`query_recipe`) | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Exact symbol lookup | **`show`** (`name`, optional `in`) | `find-symbol-definitions` | +| Field-qualified symbol discovery | **`show`** or **`snippet`** (`query` with `kind:` / `name:` / `path:` / `in:` + free text) | `find-symbol-by-kind` for kind-heavy patterns; CLI `codemap show --query '…' --print-sql` to inspect generated SQL (no MCP `print_sql` arg) | +| Kind / pattern lookup | **`query_recipe`** | `find-symbol-by-kind` | +| Source at symbol | **`snippet`** | same rows as `show` + disk text | +| Blast radius | **`impact`** (`target`, `direction`, `via`, `depth`) | `fan-in` for file hubs; symbol call graph via SQL or `impact` | +| Call path + snippets | **`trace`** (`from`, `to`, `via?`, `max_depth?`, `budget_chars?`) — adaptive snippet caps 15k/10k/6k when omitted | `call-path` | +| Type extends / implements chain | **`query_recipe`** | `type-ancestors`, `type-descendants` (`file_path` when homonyms; on `type-descendants` also scopes output to that file) | +| Multi-symbol survey | **`explore`** (`names`, `depth?`, `kind?`, `budget_chars?`) — row cap always adaptive (500/250/125); snippets 15k/10k/6k when `budget_chars` omitted | `symbol-neighborhood` (once per name) | +| One-hop symbol card | **`node`** (`name`, `kind?`, `in?`, `include_snippets?`, `budget_chars?`) — adaptive snippet caps when snippets enabled | `show` + `symbol-neighborhood` with `depth=1` | +| Affected tests | **`affected`** (`paths?`, `changed_since?`, `test_glob?`, `max_depth?`) | `affected-tests` (RS-delimit multiple paths in `query_recipe` params) | +| CI / SARIF | **`query_recipe`** + `format: "sarif"` | `deprecated-symbols`, `boundary-violations`, … | +| GitLab Code Quality | **`query_recipe`** + `format: "codeclimate"` | `boundary-violations`, … — locatable rows only; flat `minor` severity | +| CI badge / issue count | **`query_recipe`** + `format: "badge"` (+ `badge_style: "json"` for gates) | presentation only — triage via JSON rows / `--summary` | +| Ad-hoc SQL | **`query`** | — | +| N statements / one round-trip | **`query_batch`** | **`codemap query batch`** | +| Index freshness (index-level) | **`context`** (`index_freshness`) + tool metadata above | — | +| Per-file staleness | **`validate`** | — | +| Drift vs baseline | **`audit`** (`baseline_prefix` and/or per-delta `baselines`) or **`query`** / **`query_recipe`** + `baseline` (one-shot row diff vs `query_baselines`; incompatible with non-`json` `format` / `group_by`) | save via **`save_baseline`**; `summary: true` → count-only diff | +| PR merge-base drift | **`audit`** `base: ` (git committish; sha-keyed cache) | `attribution: introduced` (branch-new) \| `inherited` (pre-existing at merge base) on each `added` row; `jq '.deltas.deprecated.added[] \| select(.attribution == "introduced")'`; `summary: true` → `added_introduced` / `added_inherited` | +| Load coverage data | **`ingest_coverage`** (`path`, optional `runtime` for V8 dirs; auto-detects Istanbul `.json` / LCOV `.info`) | enables `worst-covered-exports`, `files-by-coverage`, `untested-and-dead`, **`coverage-confirmed-dead`** (`confidence: high`); **`high-crap-score`** measured override | +| Complex + undertested (no ingest) | **`query_recipe`** `high-crap-score` | graph-estimated 85/40/0% tiers — parse `coverage_source` before CI gates; prefer after **`ingest_coverage`** when possible | +| Churn × complexity refactor targets | **`query_recipe`** `churn-complexity-hotspots` (`by_symbol?`, `min_complexity?`, `row_limit?`) | git `file_churn` refreshed every index; non-git: CLI `ingest-churn` or config `churn.file` — **not** the `hotspots` outcome alias (`fan-in`) | +| High-judgment recipe triage | **`query_recipe`** `unimported-exports`, `boundary-violations`, `deprecated-symbols` | rows include `reason` / `evidence_json` — cite before **`apply`** or deletion | +| Apply recipe diff rows | **`apply`** (`recipe`, `params?`, `dry_run?`, `yes?`, `force?`, `until_empty?`, `max_passes?`, `commit_message?`) | recipe must emit `{file_path, line_start, before_pattern, after_pattern}` rows; `yes: true` required for writes; non-`auto_fixable` recipes need `force: true` | +| Apply agent/codemod rows | **`apply_rows`** (`rows`, `dry_run?`, `yes?`) | same row contract; bypasses recipe `auto_fixable` / allowlist gates | +| Apply unified diff text | **`apply_diff_input`** (`diff_text`, `dry_run?`, `yes?`, `commit_message?`) | parses git-style hunks; same executor as `apply_rows` | ## Chains - Rename: `find-symbol-definitions` → `find-symbol-references` (both via **`query_recipe`**). - Call path: **`trace`** (`from`, `to`) or **`query_recipe`** `call-path`; add snippets via **`trace`** / **`node`** / **`explore`** (adaptive snippet caps 15k/10k/6k; explore row cap 500/250/125 always adaptive) or **`snippet`** per row. Dependency hops may return `snippets_skipped_reason` — fall back to **`query_recipe`** + **`snippet`** per hop. - Type hierarchy: **`query_recipe`** `type-ancestors` / `type-descendants`; pass `file_path` when symbol names collide across files. On **`type-descendants`**, `file_path` also limits results to descendants defined in that file. -- Refactor risk: `fan-in` + `refactor-risk-ranking`. +- Refactor risk: `churn-complexity-hotspots` → `fan-in` + `refactor-risk-ranking` → **`snippet`** per row (alias `hotspots` = import fan-in only — not churn×complexity). - Edit path: **`show`** → **`snippet`**; if `stale: true`, line range may have drifted. - Apply path: **`query_recipe`** (or audit baseline `added`) with `format: "diff-json"` → **`apply`** `dry_run: true` → **`apply`** `yes: true` (+ `force: true` when recipe is not `auto_fixable`). Pre-built rows: **`apply_rows`**; unified diff: **`apply_diff_input`**. Fixpoint: **`apply`** `until_empty: true`; git: **`commit_message`** on **`apply`** / **`apply_diff_input`**. **`rename-preview` homonyms:** pass `define_in` in `params` (definition `file_path` anchor — not the same as `in_file`). @@ -75,6 +76,6 @@ Key fields: `pending_sync` (watcher debounce queue or in-flight reindex), `commi ## Recipe ids cited here -`find-symbol-definitions`, `find-symbol-by-kind`, `find-symbol-references`, `fan-in`, `call-path`, `symbol-neighborhood`, `type-ancestors`, `type-descendants`, `affected-tests`, `deprecated-symbols`, `boundary-violations`, `unimported-exports`, `coverage-confirmed-dead`, `high-crap-score`, `refactor-risk-ranking`, `calls-including-heuristic` (opt-in; requires `synthesis.heuristicCalls: true` in config). Default call-graph recipes exclude `provenance = 'heuristic'`. Others: list via **`codemap://recipes`** before **`query_recipe`**. +`find-symbol-definitions`, `find-symbol-by-kind`, `find-symbol-references`, `fan-in`, `call-path`, `symbol-neighborhood`, `type-ancestors`, `type-descendants`, `affected-tests`, `deprecated-symbols`, `boundary-violations`, `unimported-exports`, `coverage-confirmed-dead`, `high-crap-score`, `churn-complexity-hotspots`, `refactor-risk-ranking`, `calls-including-heuristic` (opt-in; requires `synthesis.heuristicCalls: true` in config). Default call-graph recipes exclude `provenance = 'heuristic'`. Others: list via **`codemap://recipes`** before **`query_recipe`**. - + diff --git a/templates/agent-content/rule/00-full.md b/templates/agent-content/rule/00-full.md index a1df3b00..c1a79dd6 100644 --- a/templates/agent-content/rule/00-full.md +++ b/templates/agent-content/rule/00-full.md @@ -24,6 +24,8 @@ codemap query --recipes-json # canonical list of every bundled + p **Coverage columns:** `high-crap-score` rows add **`coverage_source`** (`measured` \| `estimated`) and **`effective_coverage_pct`** — measured when `ingest-coverage` has a symbol row; else graph tiers 85/40/0% from test reachability (heuristic, not execution). +**Churn / hotspot columns:** `churn-complexity-hotspots` rows add **`hotspot_score`**, **`hotspot_score_normalized`**, **`churn_trend`** — distinct from outcome alias **`hotspots`** → `fan-in`. Non-git: `ingest-churn` / `churn.file`. + **Confidence columns:** `coverage-confirmed-dead` rows add **`confidence`** (`high` \| `medium`) — `high` when static dead and ingested `coverage_pct = 0`; `medium` when dead but unmeasured. Parse before deletion. **Audit attribution:** `codemap audit --base ` (and MCP/HTTP `audit` with `base`) tags each `added` row with **`attribution: introduced | inherited`** — branch-new vs pre-existing at merge base. Filter actionable PR deltas with `jq '.deltas.deprecated.added[] | select(.attribution == "introduced")'`. diff --git a/templates/agent-content/skill/10-recipes-context.md b/templates/agent-content/skill/10-recipes-context.md index e143daf1..41cba3b5 100644 --- a/templates/agent-content/skill/10-recipes-context.md +++ b/templates/agent-content/skill/10-recipes-context.md @@ -2,7 +2,7 @@ Replace placeholders (`'...'`) with your module path, file glob, or symbol name. -**Outcome aliases:** **`codemap dead-code`** · **`deprecated`** · **`boundaries`** · **`hotspots`** · **`coverage-gaps`** — thin wrappers over `query --recipe `. Every `query` flag passes through (`--json`, `--format sarif|codeclimate|badge`, `--badge-style markdown|json`, `--ci`, `--summary`, `--changed-since`, `--group-by`, `--params`, `--save-baseline`, `--baseline`). Run **`codemap --help`** for the wrapped recipe id. Capped at 5 to avoid sprawl. **Write alias (outside the cap):** **`codemap rename`** → **`apply rename-preview`** (homonym-safe via `--define-in` / `define_in` in params). +**Outcome aliases:** **`codemap dead-code`** · **`deprecated`** · **`boundaries`** · **`hotspots`** · **`coverage-gaps`** — thin wrappers over `query --recipe `. Every `query` flag passes through (`--json`, `--format sarif|codeclimate|badge`, `--badge-style markdown|json`, `--ci`, `--summary`, `--changed-since`, `--group-by`, `--params`, `--save-baseline`, `--baseline`). Run **`codemap --help`** for the wrapped recipe id. Capped at 5 to avoid sprawl. **`hotspots` → `fan-in` (import hubs)** — for **change-frequency × complexity** refactor targets use recipe **`churn-complexity-hotspots`** (not the alias). **Write alias (outside the cap):** **`codemap rename`** → **`apply rename-preview`** (homonym-safe via `--define-in` / `define_in` in params). **Suppressions (opt-in):** `// codemap-ignore-next-line ` and `// codemap-ignore-file ` (also `#`, `--`, ` ## Recipe catalog diff --git a/templates/agent-content/skill/30-schema.gen.md b/templates/agent-content/skill/30-schema.gen.md index 22d1eeed..7878218e 100644 --- a/templates/agent-content/skill/30-schema.gen.md +++ b/templates/agent-content/skill/30-schema.gen.md @@ -1,9 +1,7 @@ ## Schema reference diff --git a/templates/agent-content/skill/50-maintenance.md b/templates/agent-content/skill/50-maintenance.md index cd59f23f..f92651d4 100644 --- a/templates/agent-content/skill/50-maintenance.md +++ b/templates/agent-content/skill/50-maintenance.md @@ -23,4 +23,4 @@ codemap validate --json Same flags as **`npx @stainless-code/codemap`**, **`pnpm dlx @stainless-code/codemap`**, etc. **`codemap --root /path/to/project`** indexes another working tree. -**Full-text search (opt-in):** pass **`--with-fts`** on index runs, or set **`fts5: true`** in `.codemap/config.ts` — populates the `source_fts` virtual table for `show --query` / `snippet --query` / MCP `show` / `snippet` when `with_fts: true`. Default OFF until measurement closes [FTS default-on evaluation](../../docs/plans/fts-default-on-evaluation.md). +**Full-text search (opt-in):** pass **`--with-fts`** on index runs, or set **`fts5: true`** in `.codemap/config.ts` — populates the `source_fts` virtual table for `show --query` / `snippet --query` / MCP `show` / `snippet` when `with_fts: true`. Default OFF on index runs; enable when you need FTS-backed field search. From 2953ed30fcd8f92da886392801c61b55fa492d53 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 10 Jun 2026 17:53:06 +0300 Subject: [PATCH 6/8] harden: churn ingest correctness and index-table-stats golden order Do not wipe file_churn on git log failure; fall back to full churn refresh when config fingerprint drifts during incremental index. Run index-table-stats golden before churn seed scenarios (file_churn: 46). --- .../golden/minimal/index-table-stats.json | 2 +- fixtures/golden/scenarios.json | 10 ++--- src/application/churn-ingest.test.ts | 37 +++++++++++++++++++ src/application/churn-ingest.ts | 6 ++- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/fixtures/golden/minimal/index-table-stats.json b/fixtures/golden/minimal/index-table-stats.json index a9858745..220d5987 100644 --- a/fixtures/golden/minimal/index-table-stats.json +++ b/fixtures/golden/minimal/index-table-stats.json @@ -33,6 +33,6 @@ "jsx_attributes": 10, "boundary_rules": 1, "suppressions": 1, - "file_churn": 3 + "file_churn": 46 } ] diff --git a/fixtures/golden/scenarios.json b/fixtures/golden/scenarios.json index 682e8661..2e202b69 100644 --- a/fixtures/golden/scenarios.json +++ b/fixtures/golden/scenarios.json @@ -44,6 +44,11 @@ "prompt": "Row counts for main tables (same SQL as --recipe index-summary)", "recipe": "index-summary" }, + { + "id": "index-table-stats", + "prompt": "Row counts for all indexed substrate tables (mirrors fetchTableStats + post-pass tables)", + "sql": "SELECT (SELECT COUNT(*) FROM files) AS files, (SELECT COUNT(*) FROM symbols) AS symbols, (SELECT COUNT(*) FROM imports) AS imports, (SELECT COUNT(*) FROM exports) AS exports, (SELECT COUNT(*) FROM components) AS components, (SELECT COUNT(*) FROM dependencies) AS dependencies, (SELECT COUNT(*) FROM markers) AS markers, (SELECT COUNT(*) FROM type_members) AS type_members, (SELECT COUNT(*) FROM type_heritage) AS type_heritage, (SELECT COUNT(*) FROM calls) AS calls, (SELECT COUNT(*) FROM css_variables) AS css_vars, (SELECT COUNT(*) FROM css_classes) AS css_classes, (SELECT COUNT(*) FROM css_keyframes) AS css_keyframes, (SELECT COUNT(*) FROM scopes) AS scopes, (SELECT COUNT(*) FROM \"references\") AS ref_count, (SELECT COUNT(*) FROM bindings) AS bindings, (SELECT COUNT(*) FROM import_specifiers) AS import_specifiers, (SELECT COUNT(*) FROM function_params) AS function_params, (SELECT COUNT(*) FROM runtime_markers) AS runtime_markers, (SELECT COUNT(*) FROM test_suites) AS test_suites, (SELECT COUNT(*) FROM re_export_chains) AS re_export_chains, (SELECT COUNT(*) FROM module_cycles) AS module_cycles, (SELECT COUNT(*) FROM dynamic_imports) AS dynamic_imports, (SELECT COUNT(*) FROM file_metrics) AS file_metrics, (SELECT COUNT(*) FROM unresolved_calls) AS unresolved_calls, (SELECT COUNT(*) FROM jsx_elements) AS jsx_elements, (SELECT COUNT(*) FROM async_calls) AS async_calls, (SELECT COUNT(*) FROM decorators) AS decorators, (SELECT COUNT(*) FROM try_catch) AS try_catch, (SELECT COUNT(*) FROM jsdoc_tags) AS jsdoc_tags, (SELECT COUNT(*) FROM jsx_attributes) AS jsx_attributes, (SELECT COUNT(*) FROM boundary_rules) AS boundary_rules, (SELECT COUNT(*) FROM suppressions) AS suppressions, (SELECT COUNT(*) FROM file_churn) AS file_churn" + }, { "id": "imports-consumer-alias", "prompt": "Which alias import does consumer.ts use for the API client?", @@ -676,11 +681,6 @@ "prompt": "JSX attribute substrate on ProductCard", "sql": "SELECT a.name, a.value_kind FROM jsx_attributes a JOIN jsx_elements e ON e.id = a.element_id WHERE e.file_path = 'src/components/shop/ProductCard.tsx' ORDER BY a.name" }, - { - "id": "index-table-stats", - "prompt": "Row counts for all indexed substrate tables (mirrors fetchTableStats + post-pass tables)", - "sql": "SELECT (SELECT COUNT(*) FROM files) AS files, (SELECT COUNT(*) FROM symbols) AS symbols, (SELECT COUNT(*) FROM imports) AS imports, (SELECT COUNT(*) FROM exports) AS exports, (SELECT COUNT(*) FROM components) AS components, (SELECT COUNT(*) FROM dependencies) AS dependencies, (SELECT COUNT(*) FROM markers) AS markers, (SELECT COUNT(*) FROM type_members) AS type_members, (SELECT COUNT(*) FROM type_heritage) AS type_heritage, (SELECT COUNT(*) FROM calls) AS calls, (SELECT COUNT(*) FROM css_variables) AS css_vars, (SELECT COUNT(*) FROM css_classes) AS css_classes, (SELECT COUNT(*) FROM css_keyframes) AS css_keyframes, (SELECT COUNT(*) FROM scopes) AS scopes, (SELECT COUNT(*) FROM \"references\") AS ref_count, (SELECT COUNT(*) FROM bindings) AS bindings, (SELECT COUNT(*) FROM import_specifiers) AS import_specifiers, (SELECT COUNT(*) FROM function_params) AS function_params, (SELECT COUNT(*) FROM runtime_markers) AS runtime_markers, (SELECT COUNT(*) FROM test_suites) AS test_suites, (SELECT COUNT(*) FROM re_export_chains) AS re_export_chains, (SELECT COUNT(*) FROM module_cycles) AS module_cycles, (SELECT COUNT(*) FROM dynamic_imports) AS dynamic_imports, (SELECT COUNT(*) FROM file_metrics) AS file_metrics, (SELECT COUNT(*) FROM unresolved_calls) AS unresolved_calls, (SELECT COUNT(*) FROM jsx_elements) AS jsx_elements, (SELECT COUNT(*) FROM async_calls) AS async_calls, (SELECT COUNT(*) FROM decorators) AS decorators, (SELECT COUNT(*) FROM try_catch) AS try_catch, (SELECT COUNT(*) FROM jsdoc_tags) AS jsdoc_tags, (SELECT COUNT(*) FROM jsx_attributes) AS jsx_attributes, (SELECT COUNT(*) FROM boundary_rules) AS boundary_rules, (SELECT COUNT(*) FROM suppressions) AS suppressions, (SELECT COUNT(*) FROM file_churn) AS file_churn" - }, { "id": "meta-fts5-enabled", "prompt": "meta.fts5_enabled after indexing minimal with fts5: true", diff --git a/src/application/churn-ingest.test.ts b/src/application/churn-ingest.test.ts index 09230aa4..015e908c 100644 --- a/src/application/churn-ingest.test.ts +++ b/src/application/churn-ingest.test.ts @@ -429,6 +429,43 @@ describe("refreshFileChurn", () => { } }); + it("incremental scope falls back to full refresh when config fingerprint drifts", () => { + const db = openCodemapDatabase(":memory:"); + try { + createSchema(db); + insertFile(db, { + path: "src/a.ts", + content_hash: "a", + size: 10, + line_count: 1, + language: "typescript", + last_modified: 1, + indexed_at: 1, + }); + setMeta(db, META_CHURN_CONFIG_FINGERPRINT, "30|v1.0.0"); + ingestFileChurnFromGit(db, { projectRoot, quiet: true }); + const fullCount = db + .query<{ c: number }>("SELECT COUNT(*) AS c FROM file_churn") + .get()?.c; + const scoped = refreshFileChurn(db, { + projectRoot, + mode: "incremental", + changedPaths: ["src/a.ts"], + halfLifeDays: 90, + since: null, + quiet: true, + }); + expect(scoped.ok).toBe(true); + const afterCount = db + .query<{ c: number }>("SELECT COUNT(*) AS c FROM file_churn") + .get()?.c; + expect(afterCount).toBe(fullCount); + expect(getMeta(db, META_CHURN_CONFIG_FINGERPRINT)).toBe("90|"); + } finally { + closeDb(db); + } + }); + it("deletions mode stamps meta without running git log", () => { const db = openCodemapDatabase(":memory:"); try { diff --git a/src/application/churn-ingest.ts b/src/application/churn-ingest.ts index 3fbc2a4a..6e5bedde 100644 --- a/src/application/churn-ingest.ts +++ b/src/application/churn-ingest.ts @@ -367,7 +367,6 @@ export function ingestFileChurnFromGit( maxBuffer: 64 * 1024 * 1024, }); if (log.status !== 0) { - if (!merge) replaceFileChurn(db, []); const reason = `skipped: git log failed (${log.stderr?.toString().trim() || "unknown"})`; if (!quiet) console.error(`[churn] ${reason}`); const fallback = tryConfigChurnFallback(db, projectRoot, quiet, finish); @@ -488,10 +487,13 @@ export function refreshFileChurn( quiet, }; + const fp = churnConfigFingerprint(halfLifeDays, since ?? null); + const storedFp = getMeta(db, META_CHURN_CONFIG_FINGERPRINT) ?? null; if ( mode === "incremental" && options?.changedPaths && - options.changedPaths.length > 0 + options.changedPaths.length > 0 && + storedFp === fp ) { return ingestFileChurnFromGit(db, { ...base, From 51778cff1bb156cfd76843d8af7d79f210350413 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 10 Jun 2026 18:00:24 +0300 Subject: [PATCH 7/8] =?UTF-8?q?test:=20close=20CodeRabbit=20churn=20nits?= =?UTF-8?q?=20=E2=80=94=20config=20path,=20help,=20golden=20guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover churn.file absolute resolution, mark computed_at optional in ingest-churn help, guard file_churn in index-table-stats matrix, and reuse parseChurnJsonPayload for golden seed validation. --- scripts/query-golden-coverage-matrix.test.mjs | 1 + scripts/query-golden/run-setup.ts | 15 ++++++++++++--- src/application/ingest-churn-run.ts | 2 +- src/cli/cmd-ingest-churn.ts | 4 ++-- src/config.test.ts | 7 +++++++ 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/query-golden-coverage-matrix.test.mjs b/scripts/query-golden-coverage-matrix.test.mjs index d4ac2128..cc03bbb0 100644 --- a/scripts/query-golden-coverage-matrix.test.mjs +++ b/scripts/query-golden-coverage-matrix.test.mjs @@ -79,6 +79,7 @@ describe("golden coverage matrix", () => { expect(scenarioIds.has("index-table-stats")).toBe(true); const scenario = scenarios.find((s) => s.id === "index-table-stats"); expect(scenario?.sql).toContain("FROM file_metrics"); + expect(scenario?.sql).toContain("FROM file_churn"); expect(scenario?.sql).toContain("FROM unresolved_calls"); expect(scenario?.sql).toContain('FROM "references"'); }); diff --git a/scripts/query-golden/run-setup.ts b/scripts/query-golden/run-setup.ts index b4fef0bf..b0e27065 100644 --- a/scripts/query-golden/run-setup.ts +++ b/scripts/query-golden/run-setup.ts @@ -5,6 +5,7 @@ import { ingestIstanbul, ingestLcov, } from "../../src/application/coverage-engine"; +import { parseChurnJsonPayload } from "../../src/application/ingest-churn-run"; import { closeDb, openDb, replaceFileChurn } from "../../src/db"; import type { FileChurnRow } from "../../src/db"; import type { GoldenSetupStep } from "./schema"; @@ -42,9 +43,17 @@ export function runGoldenSetup( `query-golden setup: missing file-churn seed ${absPath}`, ); } - const rows = JSON.parse( - readFileSync(absPath, "utf-8"), - ) as FileChurnRow[]; + let rows: FileChurnRow[]; + try { + rows = parseChurnJsonPayload( + JSON.parse(readFileSync(absPath, "utf-8")) as unknown, + ); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error( + `query-golden setup: invalid file-churn seed ${absPath}: ${msg}`, + ); + } replaceFileChurn(db, rows); continue; } diff --git a/src/application/ingest-churn-run.ts b/src/application/ingest-churn-run.ts index f9d6f14b..b94f6bc9 100644 --- a/src/application/ingest-churn-run.ts +++ b/src/application/ingest-churn-run.ts @@ -36,7 +36,7 @@ function gitSpawnEnv(): NodeJS.ProcessEnv { return e; } -function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { +export function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { if (!Array.isArray(raw)) { throw new TypeError("churn JSON must be an array of file_churn rows"); } diff --git a/src/cli/cmd-ingest-churn.ts b/src/cli/cmd-ingest-churn.ts index 4a865122..c5ffbf9e 100644 --- a/src/cli/cmd-ingest-churn.ts +++ b/src/cli/cmd-ingest-churn.ts @@ -16,8 +16,8 @@ export function printIngestChurnCmdHelp(): void { Import precomputed git churn metrics into \`file_churn\` for non-git repositories or CI fixtures. JSON must be an array of objects with \`file_path\`, \`commit_count\`, \`weighted_commits\`, \`lines_added\`, -\`lines_removed\`, optional \`last_commit_at\` / \`churn_trend\`, and -\`computed_at\`. Run \`codemap\` (index) first — only indexed paths are +\`lines_removed\`, optional \`last_commit_at\` / \`churn_trend\` / +\`computed_at\` (defaults to current time). Run \`codemap\` (index) first — only indexed paths are kept; unindexed paths are skipped. Args: diff --git a/src/config.test.ts b/src/config.test.ts index 0d4f35eb..bcc0abd1 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -135,6 +135,13 @@ describe("resolveCodemapConfig", () => { expect(r.churn.since).toBe("abc123"); }); + it("resolves churn.file to absolute path", () => { + const r = resolveCodemapConfig(dir, { + churn: { file: "churn-data.json" }, + }); + expect(r.churn.file).toBe(join(dir, "churn-data.json")); + }); + it("defaults boundaries to []", () => { const r = resolveCodemapConfig(dir, undefined); expect(r.boundaries).toEqual([]); From c8f037c6fdf04c114ce54b8d14a301b9f5b16662 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 10 Jun 2026 18:03:55 +0300 Subject: [PATCH 8/8] harden: churn ingest transactions, trend validation, consumer parity Wrap replaceFileChurn/mergeFileChurnForPaths in transactions; validate churn_trend on JSON ingest; align README, recipe, agent-content, and CLI help for churn.file override and hotspots alias distinction. --- README.md | 4 +++- src/application/ingest-churn-run.test.ts | 18 ++++++++++++++- src/application/ingest-churn-run.ts | 22 +++++++++++++++---- src/cli/cmd-ingest-churn.ts | 13 ++++++----- src/db.ts | 18 ++++++++++----- templates/agent-content/mcp-instructions.md | 2 +- .../agent-content/skill/10-recipes-context.md | 6 ++--- .../recipes/churn-complexity-hotspots.md | 2 +- 8 files changed, 63 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index faa54d64..317b8bf7 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ codemap validate --json # detect stale / mi codemap context --compact --for "refactor auth" # JSON envelope + intent-matched recipes codemap ingest-coverage coverage/coverage-final.json --json # Istanbul / LCOV (auto-detected) → coverage table; joins with symbols NODE_V8_COVERAGE=.cov bun test && codemap ingest-coverage .cov --runtime --json # V8 protocol (per-process dumps); local-only +codemap ingest-churn metrics/churn.json --json # precomputed file_churn → churn-complexity-hotspots (non-git / CI) +codemap query --json --recipe churn-complexity-hotspots # change-frequency × complexity (not the hotspots alias) codemap agents init # scaffold .agents/ rules + skills codemap agents init --mcp # PM-aware project MCP config (see docs/agents.md) codemap apply rename-preview --params old=foo,new=bar --dry-run # preview recipe-driven edits (substrate executor) @@ -86,7 +88,7 @@ codemap query --json --recipe fan-out-sample codemap dead-code --json # → query --recipe untested-and-dead codemap deprecated --ci # → query --recipe deprecated-symbols --ci codemap boundaries --format sarif > boundary-findings.sarif # → query --recipe boundary-violations --format sarif -codemap hotspots --json --group-by directory # → query --recipe fan-in --json --group-by directory +codemap hotspots --json --group-by directory # → query --recipe fan-in (import hubs — not churn×complexity) codemap coverage-gaps --json --summary # → query --recipe worst-covered-exports --json --summary # Parametrised recipes validate params from .md frontmatter before SQL binding. codemap query --json --recipe find-symbol-by-kind --params kind=function,name_pattern=%Query% diff --git a/src/application/ingest-churn-run.test.ts b/src/application/ingest-churn-run.test.ts index ac3a5fd3..42420b34 100644 --- a/src/application/ingest-churn-run.test.ts +++ b/src/application/ingest-churn-run.test.ts @@ -5,7 +5,10 @@ import { join } from "node:path"; import { closeDb, createSchema, insertFile } from "../db"; import { openCodemapDatabase } from "../sqlite-db"; -import { ingestChurnFromJsonFile } from "./ingest-churn-run"; +import { + ingestChurnFromJsonFile, + parseChurnJsonPayload, +} from "./ingest-churn-run"; describe("ingestChurnFromJsonFile", () => { it("loads indexed paths and skips unindexed rows", () => { @@ -70,6 +73,19 @@ describe("ingestChurnFromJsonFile", () => { } }); + it("rejects invalid churn_trend values", () => { + expect(() => + parseChurnJsonPayload([ + { + file_path: "src/a.ts", + commit_count: 1, + weighted_commits: 1, + churn_trend: "spiking", + }, + ]), + ).toThrow(/churn_trend must be accelerating/); + }); + it("rejects empty JSON without wiping file_churn", () => { const root = mkdtempSync(join(tmpdir(), "codemap-ingest-churn-empty-")); try { diff --git a/src/application/ingest-churn-run.ts b/src/application/ingest-churn-run.ts index b94f6bc9..eec0ba78 100644 --- a/src/application/ingest-churn-run.ts +++ b/src/application/ingest-churn-run.ts @@ -36,6 +36,23 @@ function gitSpawnEnv(): NodeJS.ProcessEnv { return e; } +const CHURN_TREND_VALUES = new Set(["accelerating", "stable", "cooling"]); + +function parseChurnTrendField(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (typeof value !== "string") { + throw new TypeError( + "churn_trend must be accelerating, stable, cooling, or null", + ); + } + if (!CHURN_TREND_VALUES.has(value)) { + throw new TypeError( + `churn_trend must be accelerating, stable, or cooling (got ${value})`, + ); + } + return value; +} + export function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { if (!Array.isArray(raw)) { throw new TypeError("churn JSON must be an array of file_churn rows"); @@ -63,10 +80,7 @@ export function parseChurnJsonPayload(raw: unknown): FileChurnRow[] { r.last_commit_at === null || r.last_commit_at === undefined ? null : String(r.last_commit_at), - churn_trend: - r.churn_trend === null || r.churn_trend === undefined - ? null - : String(r.churn_trend), + churn_trend: parseChurnTrendField(r.churn_trend), computed_at: typeof r.computed_at === "string" ? r.computed_at diff --git a/src/cli/cmd-ingest-churn.ts b/src/cli/cmd-ingest-churn.ts index c5ffbf9e..d5f27687 100644 --- a/src/cli/cmd-ingest-churn.ts +++ b/src/cli/cmd-ingest-churn.ts @@ -14,11 +14,14 @@ export function printIngestChurnCmdHelp(): void { console.log(`Usage: codemap ingest-churn [--json] Import precomputed git churn metrics into \`file_churn\` for non-git -repositories or CI fixtures. JSON must be an array of objects with -\`file_path\`, \`commit_count\`, \`weighted_commits\`, \`lines_added\`, -\`lines_removed\`, optional \`last_commit_at\` / \`churn_trend\` / -\`computed_at\` (defaults to current time). Run \`codemap\` (index) first — only indexed paths are -kept; unindexed paths are skipped. +repositories or CI fixtures (same JSON shape as config \`churn.file\`). +Replaces all \`file_churn\` rows — include every indexed path you want +retained. Enables \`churn-complexity-hotspots\`. JSON must be an array of +objects with \`file_path\`, \`commit_count\`, \`weighted_commits\`, optional +\`lines_added\` / \`lines_removed\` (default 0), \`last_commit_at\` / +\`churn_trend\` (\`accelerating\` \| \`stable\` \| \`cooling\`) / +\`computed_at\` (defaults to current time). Run \`codemap\` (index) first — +only indexed paths are kept; unindexed paths are skipped. Args: Path to JSON file (relative to project root or absolute) diff --git a/src/db.ts b/src/db.ts index 573eff7c..c889a609 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1744,8 +1744,11 @@ export function insertFileChurn(db: CodemapDatabase, rows: FileChurnRow[]) { /** Replace all churn rows (full-rebuild git ingest or golden seed-file-churn). */ export function replaceFileChurn(db: CodemapDatabase, rows: FileChurnRow[]) { - db.run("DELETE FROM file_churn"); - insertFileChurn(db, rows); + const persist = db.transaction(() => { + db.run("DELETE FROM file_churn"); + insertFileChurn(db, rows); + }); + persist(); } /** `meta` key: last `HEAD` when `file_churn` was refreshed (idle skip). */ @@ -1759,10 +1762,13 @@ export function mergeFileChurnForPaths( rows: FileChurnRow[], scopePaths: Iterable, ) { - for (const p of scopePaths) { - db.run("DELETE FROM file_churn WHERE file_path = ?", [p]); - } - insertFileChurn(db, rows); + const persist = db.transaction(() => { + for (const p of scopePaths) { + db.run("DELETE FROM file_churn WHERE file_path = ?", [p]); + } + insertFileChurn(db, rows); + }); + persist(); } /** Drop churn rows whose `file_path` is no longer indexed. */ diff --git a/templates/agent-content/mcp-instructions.md b/templates/agent-content/mcp-instructions.md index cfec594a..e287ee9b 100644 --- a/templates/agent-content/mcp-instructions.md +++ b/templates/agent-content/mcp-instructions.md @@ -53,7 +53,7 @@ Key fields: `pending_sync` (watcher debounce queue or in-flight reindex), `commi | Load coverage data | **`ingest_coverage`** (`path`, optional `runtime` for V8 dirs; auto-detects Istanbul `.json` / LCOV `.info`) | enables `worst-covered-exports`, `files-by-coverage`, `untested-and-dead`, **`coverage-confirmed-dead`** (`confidence: high`); **`high-crap-score`** measured override | | Load churn data (non-git / fixture) | **`ingest_churn`** (`path`) | precomputed `file_churn` JSON — CLI twin `codemap ingest-churn`; enables `churn-complexity-hotspots` when git history is unavailable | | Complex + undertested (no ingest) | **`query_recipe`** `high-crap-score` | graph-estimated 85/40/0% tiers — parse `coverage_source` before CI gates; prefer after **`ingest_coverage`** when possible | -| Churn × complexity refactor targets | **`query_recipe`** `churn-complexity-hotspots` (`by_symbol?`, `min_complexity?`, `row_limit?`, `path_prefix?`) | git `file_churn` refreshed every index; non-git: **`ingest_churn`** / CLI `ingest-churn` or config `churn.file` — **not** the `hotspots` outcome alias (`fan-in`); empty `file_churn` → `context` `churn_hint` | +| Churn × complexity refactor targets | **`query_recipe`** `churn-complexity-hotspots` (`by_symbol?`, `min_complexity?`, `row_limit?`, `path_prefix?`) | git `file_churn` by default each index; config **`churn.file`** replaces git when set; fixtures: **`ingest_churn`** / `ingest-churn` — **not** the `hotspots` alias (`fan-in`); empty `file_churn` → `context` `churn_hint` | | High-judgment recipe triage | **`query_recipe`** `unimported-exports`, `boundary-violations`, `deprecated-symbols` | rows include `reason` / `evidence_json` — cite before **`apply`** or deletion | | Apply recipe diff rows | **`apply`** (`recipe`, `params?`, `dry_run?`, `yes?`, `force?`, `until_empty?`, `max_passes?`, `commit_message?`) | recipe must emit `{file_path, line_start, before_pattern, after_pattern}` rows; `yes: true` required for writes; non-`auto_fixable` recipes need `force: true` | | Apply agent/codemod rows | **`apply_rows`** (`rows`, `dry_run?`, `yes?`) | same row contract; bypasses recipe `auto_fixable` / allowlist gates | diff --git a/templates/agent-content/skill/10-recipes-context.md b/templates/agent-content/skill/10-recipes-context.md index abd00854..62231299 100644 --- a/templates/agent-content/skill/10-recipes-context.md +++ b/templates/agent-content/skill/10-recipes-context.md @@ -19,7 +19,7 @@ Replace placeholders (`'...'`) with your module path, file glob, or symbol name. - **Evidence columns** — high-judgment recipes (`boundary-violations`, `deprecated-symbols`, `unimported-exports`, …) may add **`reason`** and **`evidence_json`** on each row — factual detection path (consumer hop, re-export chain, or unresolved-import blind spot on `unimported-exports`); parse before `apply` or deletion. - **Coverage columns** — `high-crap-score` adds **`coverage_source`** (`measured` \| `estimated`) and **`effective_coverage_pct`**; `estimated` is graph reachability, not execution — prefer `ingest-coverage` before CI gates. - **Confidence columns** — `coverage-confirmed-dead` adds **`confidence`** (`high` \| `medium`), **`reason`**, **`caller_count`** on static dead exports — `high` only after `ingest-coverage` shows 0% on the symbol. -- **Churn / hotspot columns** — `churn-complexity-hotspots` adds **`hotspot_score`**, **`hotspot_score_normalized`** (0–100 vs result-set max), **`churn_trend`** (`accelerating` \| `stable` \| `cooling`), plus file or symbol grain via **`by_symbol`**. Substrate **`file_churn`** refreshes every index from git; non-git: **`codemap ingest-churn`** or config **`churn.file`**. Pair with **`refactor-risk-ranking`** and **`snippet`** before large edits. +- **Churn / hotspot columns** — `churn-complexity-hotspots` adds **`hotspot_score`**, **`hotspot_score_normalized`** (0–100 vs result-set max), **`churn_trend`** (`accelerating` \| `stable` \| `cooling`), plus file or symbol grain via **`by_symbol`**. Substrate **`file_churn`**: git refresh every index by default; config **`churn.file`** replaces git when set; **`codemap ingest-churn`** / **`ingest_churn`** for fixtures (full table replace). Tune via **`churn.halfLifeDays`**, **`churn.since`**, **`--churn-since`**. Pair with **`refactor-risk-ranking`** and **`snippet`** before large edits. - **Per-row recipe `actions`** — recipes that define an **`actions: [{type, auto_fixable?, description?, command?}]`** template append it to every row in **`--json`** output (recipe-only; ad-hoc SQL never carries actions). Rendered **`command`** lines substitute `{{param}}` from bound recipe params — param **names vary by recipe** (`old`/`new` on `rename-preview`; `old_source`/`new_source` on `migrate-import-source`; `symbol`/`replacement` on `migrate-deprecated`; see each `.md` frontmatter). Under `--baseline`, actions attach to the **`added`** rows only (the rows the agent should act on). Inspect via **`--recipes-json`**. - **Boundary violations (config-driven)** — declare `boundaries: [{name, from_glob, to_glob, action?}]` in `.codemap/config.ts` and run `codemap query --recipe boundary-violations [--format sarif|codeclimate|badge]`. GitLab CI: `--format codeclimate`; README/CI summary: `--format badge`. The `action` field defaults to `"deny"` (the only shape v1 surfaces); rules are reconciled into the `boundary_rules` table on every index pass and joined against `dependencies` via SQLite `GLOB`. - **Project-local recipes** — drop **`.sql`** (and optional **`.md`** for description body, params, and actions) into **`/recipes/`** (default `.codemap/recipes/`; honors `--state-dir` / `CODEMAP_STATE_DIR`) to make team-internal SQL a first-class CLI verb. `--recipes-json` and the `codemap://recipes` MCP resource list project recipes alongside bundled ones with **`source: "bundled" | "project"`** discriminating them. Project recipes win on id collision; entries that override a bundled id carry **`shadows: true`** so agents reading the catalog at session start know when a recipe behaves differently from the documented bundled version. `.md` supports YAML frontmatter for `params:` and per-row `actions:` — **block-list shape only** (loader's hand-rolled parser; no inline-flow `[{...}]`). Param types: `string | number | boolean`; pass values with `--params key=value[,key=value]` (repeatable; last value wins). Example: `codemap query --json --recipe find-symbol-by-kind --params kind=function,name_pattern=%Query%`. Validation: SQL is rejected at load time if it starts with DML/DDL (DELETE/DROP/UPDATE/etc.); params validate before SQL binding; runtime `PRAGMA query_only=1` is the parser-proof backstop. `/index.db` is gitignored; **`/recipes/` is NOT** — recipes are git-tracked source code authored for human review. @@ -46,7 +46,7 @@ Each emitted delta carries its own `base` metadata so mixed-baseline audits are - **`save_baseline`** — polymorphic `{name, sql? | recipe?}` (exactly one of `sql` / `recipe`). - **`list_baselines`** — no args; returns the array `codemap query --baselines --json` would print. - **`drop_baseline`** — `{name}` → `{dropped}` on success; structured `{error}` on unknown name (MCP sets `isError: true`). -- **`context`** — `{compact?, intent?, include_snippets?}`. CLI: `codemap context [--include-snippets]`. Session-start project envelope with `start_here` shortcuts (one call replaces 4-5 `query`s). `include_snippets` adds one-line export previews on hub leaders (capped to adaptive `signature_max_chars`; may set `stale`/`missing`); no-op when `compact: true`. Whitespace-only `intent` is treated as no intent. Prefer `start_here.hub_leaders` over legacy `hubs` for signatures — `hubs` keeps the full bundled `fan-in` recipe limit for backward compatibility. `sample_markers` count scales down on repos >500 / >5000 files. +- **`context`** — `{compact?, intent?, include_snippets?}`. CLI: `codemap context [--include-snippets]`. Session-start project envelope with `start_here` shortcuts (one call replaces 4-5 `query`s). `index_summary.file_churn` row count; **`churn_hint`** when empty (steers to index, **`ingest-churn`**, or **`churn.file`**). `include_snippets` adds one-line export previews on hub leaders (capped to adaptive `signature_max_chars`; may set `stale`/`missing`); no-op when `compact: true`. Whitespace-only `intent` is treated as no intent. Prefer `start_here.hub_leaders` over legacy `hubs` for signatures — `hubs` keeps the full bundled `fan-in` recipe limit for backward compatibility. `sample_markers` count scales down on repos >500 / >5000 files. - **`validate`** — `{paths?: string[]}`. SHA-256 vs `files.content_hash`; returns only out-of-sync rows (`stale` / `missing` / `unindexed` — fresh paths are omitted). - **`show`** — `{name, kind?, in?}` or `{query, with_fts?}`. Exact symbol lookup or field-qualified search (`kind:`, `name:`, `path:`, `in:` + free text) → `{matches, disambiguation?, warning?}`. CLI: `codemap show --query '…' [--print-sql]`. - **`snippet`** — same as `show` (`{name, kind?, in?}` or `{query, with_fts?}`) but each match also carries `source` (file text) + `stale` / `missing` flags → `{matches, disambiguation?, warning?}`. No reindex side-effects. @@ -59,7 +59,7 @@ Each emitted delta carries its own `base` metadata so mixed-baseline audits are - **`apply_rows`** — `{rows, dry_run?, yes?}`. Same executor with caller-supplied rows (no recipe policy gates). CLI twin: `codemap apply --rows -|`. - **`apply_diff_input`** — `{diff_text, dry_run?, yes?, commit_message?}`. Unified diff → row contract (git-style `-`/`+` hunks). CLI twin: `codemap apply --diff-input `. - **`ingest_coverage`** — `{path, runtime?}`. Load Istanbul / LCOV / V8 coverage into the index (CLI twin: `codemap ingest-coverage --json`). Enables coverage recipes (`worst-covered-exports`, …). -- **`ingest_churn`** — `{path}`. Load precomputed `file_churn` JSON (CLI twin: `codemap ingest-churn`). Enables `churn-complexity-hotspots` for non-git repos and CI fixtures. +- **`ingest_churn`** — `{path}`. Load precomputed `file_churn` JSON (CLI twin: `codemap ingest-churn`); replaces all churn rows. Enables `churn-complexity-hotspots` for non-git repos and CI fixtures. **Apply workflow (discover → preview → apply):** diff --git a/templates/recipes/churn-complexity-hotspots.md b/templates/recipes/churn-complexity-hotspots.md index 1c6557d7..4212d8d6 100644 --- a/templates/recipes/churn-complexity-hotspots.md +++ b/templates/recipes/churn-complexity-hotspots.md @@ -30,7 +30,7 @@ params: Files or symbols ranked by **git churn × cyclomatic complexity**. Distinct from the outcome alias `codemap hotspots` (import **fan-in** via `fan-in` recipe). -Populated on every index pass from git history (config `churn.halfLifeDays`, `churn.since` / `--churn-since`). Non-git repos: `codemap ingest-churn ` or config `churn.file`. +Default: git refresh every index (`churn.halfLifeDays`, `churn.since` / `--churn-since`). When config `churn.file` is set, that JSON loads on every index and **skips** git log (including in git repos). Manual twin: `codemap ingest-churn ` for non-git repos and CI fixtures. ```bash codemap query --recipe churn-complexity-hotspots