From 33704fe979e9ddc40745659d13638a247f3731cb Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Mon, 14 Sep 2026 14:03:29 +0800 Subject: [PATCH] emrg: keep hand-written MEMORY.md rows through an index load/save MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MemoryIndex.to_markdown rendered only from parsed _IndexEntrys, so every store write (create/update/delete → _load_index → add_entry → _save_index) rewrote the whole index from what the parser understood. What it does not understand is most of a hand-kept index: the document title, `>` notes, per-row prose after the dates, a row listing several links, any heading that is not `## `. Rows were re-filed under an invented `## reference`. Measured on this repo's own indexes: 22389 chars → 2628 and 2388 chars → 349. The index is not a store-private file — agents append rows to it directly, and it is the artifact embedded into every system prompt. Fix: from_text keeps the source document plus the line each entry came from; to_markdown walks that document — unknown lines verbatim, an entry line verbatim unless the store rewrote that entry (raw empty) or the line exceeds INDEX_TITLE_MAX_CHARS, and only genuinely new entries appended under their `## type` heading. Unknown types are filed last instead of being dropped. Render is pure: no state changes on to_markdown. Tests: byte-identical round trip of a hand-written index, what-the-model- cannot-represent kept, load→load stable, SessionMemoryStore.create keeps hand-written rows, remove_entry drops only its own row, a rewritten entry renders once from its fields. Mutating the verbatim branch back to re-render-everything fails 3 of them. Full suite 1962 passed, 1 skipped. --- emrg/memory.py | 162 ++++++++++++++++++++++++++++++++----------- tests/test_memory.py | 95 +++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 41 deletions(-) diff --git a/emrg/memory.py b/emrg/memory.py index 755bed95..1c1d8d0a 100644 --- a/emrg/memory.py +++ b/emrg/memory.py @@ -91,6 +91,11 @@ def _truncate_index_title(title: str, max_len: int | None = None) -> str: INDEX_COUNT_WARN = 100 # >N memory files → consolidation recommended INDEX_SIZE_WARN = 50 * 1024 # >50KB MEMORY.md → consolidation recommended +# Order of the `## type` sections when the index has to be rendered from +# entries alone (a rebuild, or entries the document never had). The parser +# accepts any `## ` heading; this only decides where new rows file. +INDEX_TYPE_ORDER = ("user", "feedback", "project", "reference", "decision", "task") + # ── MemoryFile ───────────────────────────────────────────────────── @@ -302,6 +307,11 @@ class _IndexEntry: created_at: str = "" event_at: str = "" updated_at: str = "" + # The index line this entry was parsed from, byte for byte. Kept so a + # hand-written index survives a load → save round trip: the renderer emits + # the source line verbatim unless the store itself rewrote the entry. + # Empty for entries the store created (they have no source line). + raw: str = "" class MemoryIndex: @@ -316,10 +326,26 @@ class MemoryIndex: ## decision - [Use httpx](decision-use-httpx.md) [superseded] — rec: 2026-07-14, evt: 2026-07-03 + + Round-trip fidelity: MEMORY.md is a file agents hand-edit (they append rows + directly), so it holds more than this model represents — a document title, + ``>`` notes, per-row prose, a row listing several links. ``from_text`` + therefore keeps the source document and the line each entry came from, and + ``to_markdown`` walks that document instead of re-rendering it from parsed + fields: lines it does not understand come out verbatim, an entry line comes + out verbatim unless the store rewrote that entry, and only genuinely new + entries are appended under their ``## type`` heading. Loading an untouched + index and saving it back returns the same text (normalised to one trailing + newline) — no row, note or heading is lost. """ def __init__(self, entries: list[_IndexEntry] | None = None): self.entries: list[_IndexEntry] = entries or [] + # Document skeleton: the index file's own lines in order, and the + # filename each entry line parsed to. A fresh index (nothing loaded) + # starts from just the title, so every entry counts as new. + self._lines: list[str] = ["# Memory Index", ""] + self._src: dict[int, str] = {} # ── Mutation ─────────────────────────────────────────────── @@ -357,46 +383,91 @@ def remove_entry(self, filename: str) -> None: # ── Rendering ────────────────────────────────────────────── def to_markdown(self) -> str: - """Render the full index as markdown.""" - lines = ["# Memory Index", ""] - - # Group by type - by_type: dict[str, list[_IndexEntry]] = {} - for e in self.entries: - by_type.setdefault(e.type, []).append(e) - - # Stable type ordering - type_order = ["user", "feedback", "project", "reference", "decision", "task"] - for t in type_order: - if t not in by_type: + """Render the index, preserving everything this model cannot hold. + + Walks the source document (``_lines``): non-entry lines are emitted as + written, and an entry line is emitted verbatim unless the store rewrote + that entry (``raw`` empty) or the line exceeds the index line cap. + Entries the document never had are appended under their ``## type`` + heading. Re-rendering every line from parsed fields — what this used to + do — dropped a document title, ``>`` notes, per-row prose and any row + listing several links, i.e. any load → save destroyed the hand-written + index: measured on this repo's own indexes, 22389 chars came back as + 2628 and 2388 chars as 349. + """ + live = {e.filename: e for e in self.entries} + out: list[str] = [] + placed: set[str] = set() + + for i, line in enumerate(self._lines): + filename = self._src.get(i) + if filename is None: + out.append(line) # title, `>` notes, headings, prose — as written continue - lines.append(f"## {t}") - for e in by_type[t]: - status_tag = f" [{e.status}]" if e.status != "active" else "" - rec = f"rec: {_short_date(e.created_at)}" if e.created_at else "" - evt = f"evt: {_short_date(e.event_at)}" if e.event_at else "" - date_part = ", ".join(p for p in [rec, evt] if p) - raw_line = f"- [{e.title}]({e.filename}){status_tag} — {date_part}" - if len(raw_line) > INDEX_TITLE_MAX_CHARS: - # Rant 2026-08-23T08:04:26 — render-time fallback for legacy - # dirty index data (write-time truncation wasn't in place). - # Keep the filename so the detail file stays reachable. - logger.warning( - "memory index line exceeds %d chars (title=%d chars) — truncating", - INDEX_TITLE_MAX_CHARS, len(e.title), - ) - other = len(f"- []({e.filename}){status_tag} — {date_part}") - budget = max(1, INDEX_TITLE_MAX_CHARS - other) - line = ( - f"- [{_truncate_index_title(e.title, budget)}]" - f"({e.filename}){status_tag} — {date_part}" - ) - else: - line = raw_line - lines.append(line) - lines.append("") + entry = live.get(filename) + if entry is None: + continue # the store removed this entry + if filename in placed and not entry.raw: + continue # a rewritten entry renders once, however many rows it had + placed.add(filename) + if entry.raw and len(entry.raw) <= INDEX_TITLE_MAX_CHARS: + out.append(entry.raw) + else: + out.append(self._render_entry(entry)) + + fresh = [e for e in self.entries if e.filename not in placed] + # Leftover types (not in INDEX_TYPE_ORDER) are filed last rather than + # dropped — an unknown type is a rendering question, not data loss. + leftovers = sorted({e.type for e in fresh} - set(INDEX_TYPE_ORDER)) + for t in INDEX_TYPE_ORDER + tuple(leftovers): + group = [e for e in fresh if e.type == t] + if group: + self._append_under(out, t, [self._render_entry(e) for e in group]) + + return "\n".join(out).strip() + "\n" + + @staticmethod + def _render_entry(e: _IndexEntry) -> str: + """One entry as a fresh index line (bounded by INDEX_TITLE_MAX_CHARS).""" + status_tag = f" [{e.status}]" if e.status != "active" else "" + rec = f"rec: {_short_date(e.created_at)}" if e.created_at else "" + evt = f"evt: {_short_date(e.event_at)}" if e.event_at else "" + date_part = ", ".join(p for p in [rec, evt] if p) + line = f"- [{e.title}]({e.filename}){status_tag} — {date_part}" + if len(line) <= INDEX_TITLE_MAX_CHARS: + return line + # Rant 2026-08-23T08:04:26 — render-time fallback for legacy dirty + # index data (write-time truncation wasn't in place). Keep the filename + # so the detail file stays reachable. + logger.warning( + "memory index line exceeds %d chars (title=%d chars) — truncating", + INDEX_TITLE_MAX_CHARS, len(e.title), + ) + other = len(f"- []({e.filename}){status_tag} — {date_part}") + budget = max(1, INDEX_TITLE_MAX_CHARS - other) + return ( + f"- [{_truncate_index_title(e.title, budget)}]" + f"({e.filename}){status_tag} — {date_part}" + ) - return "\n".join(lines).strip() + "\n" + @staticmethod + def _append_under(out: list[str], type_name: str, lines: list[str]) -> None: + """Insert rendered entry lines into ``out`` under `## type_name`.""" + heading = f"## {type_name}" + if heading not in out: + if out and out[-1].strip(): + out.append("") + out.extend([heading, *lines, ""]) + return + start = out.index(heading) + end = len(out) + for j in range(start + 1, len(out)): + if out[j].startswith("## "): + end = j + break + while end > start + 1 and not out[end - 1].strip(): + end -= 1 # keep the section's trailing blank line + out[end:end] = lines def save(self, path: Path) -> None: """Write the index to disk.""" @@ -421,11 +492,17 @@ def from_file(cls, path: Path) -> MemoryIndex: @classmethod def from_text(cls, text: str) -> MemoryIndex: - """Parse MEMORY.md content into entries.""" + """Parse MEMORY.md content into entries, keeping the document itself.""" + idx = cls() + lines = text.split("\n") + if lines and lines[-1] == "": + lines.pop() # a trailing newline is not a line of its own + if lines: + idx._lines = lines entries: list[_IndexEntry] = [] current_type = "reference" - for line in text.split("\n"): + for i, line in enumerate(lines): stripped = line.strip() # Detect type heading: ## user @@ -463,10 +540,13 @@ def from_text(cls, text: str) -> MemoryIndex: created_at=created_at, event_at=event_at, updated_at=created_at, + raw=line, ) ) + idx._src[i] = filename - return cls(entries) + idx.entries = entries + return idx def _normalize_date(d: str) -> str: diff --git a/tests/test_memory.py b/tests/test_memory.py index 571352a4..dfab2737 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -317,3 +317,98 @@ def test_save_and_load(self, temp_cwd): loaded = MemoryIndex.from_file(path) assert len(loaded.entries) == 1 assert loaded.entries[0].filename == mem.filename + + +# A hand-written index, as agents actually keep them: a document title, `>` +# notes, a row with prose after the dates, a row listing several links, and a +# heading that is not a `## ` section. +HAND_WRITTEN_INDEX = """# 项目记忆索引 + +> 历史记录已于 2026-08-23 归档清理。 +> 本项目记忆从零开始重建。 + +- [Steady state](evolution-state.md) — active · 最近更新 cyc20260914-131523：master `f15e1b88`。 + +### Session Memory + +- [cycle-20260914-131523](cycle-20260914-131523.md) — 本周期：宿主令复述一律删除。 +- [cycle-20260914-125119](cycle-20260914-125119.md) / [cycle-20260914-122910](cycle-20260914-122910.md) — 更早周期。 +""" + + +class TestMemoryIndexRoundTripFidelity: + """The index is hand-edited; a store write must not rewrite it. + + Before this: `to_markdown` rendered only from parsed `_IndexEntry`s, so a + load → save dropped the document title, every `>` note and every row's + prose, and filed surviving rows under an invented `## reference` — the + evolution index (22389 chars, 50 rows) came back as 2628 chars. + """ + + def test_untouched_index_round_trips_byte_for_byte(self): + idx = MemoryIndex.from_text(HAND_WRITTEN_INDEX) + assert idx.to_markdown() == HAND_WRITTEN_INDEX + + def test_round_trip_keeps_what_the_model_cannot_represent(self): + out = MemoryIndex.from_text(HAND_WRITTEN_INDEX).to_markdown() + for kept in [ + "# 项目记忆索引", # document title + "> 历史记录已于", # `>` note + "### Session Memory", # unknown heading + "最近更新 cyc20260914-131523", # per-row prose + "本周期：宿主令复述一律删除", # per-row prose + "cycle-20260914-122910.md", # 2nd link in a multi-link row + ]: + assert kept in out, f"lost on round trip: {kept}" + + def test_loading_an_untouched_index_twice_is_stable(self): + once = MemoryIndex.from_text(HAND_WRITTEN_INDEX).to_markdown() + twice = MemoryIndex.from_text(once).to_markdown() + assert twice == once + + def test_store_create_preserves_hand_written_rows(self, temp_cwd): + """The store's write path: create → load index → add_entry → save.""" + store = SessionMemoryStore(temp_cwd) + store.index_path.write_text(HAND_WRITTEN_INDEX, encoding="utf-8") + + mem = store.create("task", "A new memory", "body text") + + after = store.index_path.read_text(encoding="utf-8") + assert mem.filename in after, "the new memory is not in the index" + assert "### Session Memory" in after + assert "> 历史记录已于" in after + assert "最近更新 cyc20260914-131523" in after + # The pre-existing hand-written rows are still there, verbatim. + assert "cycle-20260914-131523.md" in after + assert "cycle-20260914-125119.md" in after + + def test_removed_entry_drops_only_its_own_row(self): + idx = MemoryIndex.from_text(HAND_WRITTEN_INDEX) + idx.remove_entry("cycle-20260914-131523.md") + out = idx.to_markdown() + assert "cycle-20260914-131523.md" not in out + assert "### Session Memory" in out + assert "cycle-20260914-125119.md" in out + + def test_rewritten_entry_renders_once_from_its_fields(self): + """An entry the store rewrote loses its stale line — once, not twice.""" + text = ( + "# Memory Index\n\n" + "## task\n" + "- [Fix bug](task-fix-bug.md) — rec: 2026-07-14, evt: 2026-07-10\n" + "- [Other](task-other.md) — rec: 2026-07-14, evt: 2026-07-10\n" + ) + idx = MemoryIndex.from_text(text) + idx.add_entry( + MemoryFile( + id="1111", + type="task", + title="Fix bug", + created_at="2026-07-20T00:00:00Z", + ) + ) + out = idx.to_markdown() + assert out.count("task-fix-bug.md") == 1 + assert "2026-07-20" in out, "the rewritten row kept its stale dates" + # The untouched row is still its original line, verbatim. + assert "- [Other](task-other.md) — rec: 2026-07-14, evt: 2026-07-10" in out