host: cut the memory review's text on a code point boundary and make clip()'s reason optional

changelog.d/1918-clip-reason-optional.fixed.md

@@ -0,0 +1 @@
+The memory review now keeps whole characters in what it shows the model and writes to the log ([#1918](https://github.com/ThinkFlowLab/FlyBot/issues/1918)): the two bounds in `core/src/host/memory.js` — the 800-unit context line and the 3,000-unit two-ended user line — go through the shared boundary rule, so a cut that lands inside an emoji steps back instead of leaving half a pair, and the tail it keeps is never longer than the budget it had. The exported `clip` in `core/src/host/text.js` also makes its reason optional: a two-argument call now reads `… (cut)` instead of the literal `… (cut; undefined)`, while a call that passes a reason keeps `… (cut; <reason>)` byte for byte.

core/src/host/memory.js

@@ -28,6 +28,7 @@ import { BlockAssembler } from '@deepseek-ai/dsh-llm'
 import { currentParticipants } from './room-membership.js'
 import { defineTool } from './tool-schema.js'
 import { codePoints, splitRef, unquote } from './text.js'
+import { codePointStart, cut } from '../shared/text-boundary.js'
 import { output } from './tools.js'
 import { retiredBot } from './store.js'
 import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js'
@@ -1143,9 +1144,15 @@ export function install(rt) {
     }
     return cursors
   }
-  const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} …(cut)` : text)
+  const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text)
   // A long message often ends with what the user wants of it, so both ends stay.
-  const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text)
+  const ends = (text, max) => {
+    if (text.length <= max) return text
+    const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3)))
+    // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back.
+    const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length))
+    return `${head.text} …(middle cut)… ${tail}`
+  }
 
   // Lines say who spoke. The user's lines are kept longest: what they say is what
   // the review keeps; the rest is there to make it understood.

core/src/host/text.js

@@ -56,7 +56,9 @@ export const estimateTokens = (text) => {
   for (const char of text) tokens += char.codePointAt(0) > 0x2e7f ? 1 : 0.34
   return Math.ceil(tokens)
 }
-export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut; ${rest})` : text)
+// #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never
+// "… (cut; undefined)"; a caller that gives one keeps "… (cut; <reason>)" byte for byte.
+export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text)
 
 export const createClock = (timeZone) => {
   const clockFormat = new Intl.DateTimeFormat('en-CA', {

core/test/memory.test.js

@@ -534,6 +534,75 @@ async function reviewed(call) {
   throw new Error('the memory review did not finish')
 }
 
+// #1918: both local cuts of the review keep whole characters. The Bot line goes through the
+// two-argument cut at REVIEW_OTHER_LINE_MAX, the user's through the two-ended one at
+// REVIEW_USER_LINE_MAX; a boundary that lands inside a surrogate pair must step back, never
+// leave half of it in the prompt.
+test('a long Bot line cut for the review context keeps whole characters (#1918)', async (t) => {
+  const long = `${'x'.repeat(791)}😀tail`
+  const history = {
+    b2: [
+      { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } },
+      { event: { seq: 21, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: long }] } } } },
+      { event: { seq: 22, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Now keep it.' }] } } },
+    ],
+  }
+  const replies = [JSON.stringify({ changes: [] })]
+  const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } })
+  t.after(() => stop())
+  await call('state', {})
+  ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+  await reviewed(call)
+  const prompt = requests[0].messages[0].content[0].text
+  assert.equal(prompt.isWellFormed(), true)
+  assert.equal(prompt.includes('\ufffd'), false)
+  const shown = prompt.split('\n').find(one => one.startsWith('Writer: x'))
+  assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`)
+  assert.ok(shown.endsWith('…(cut)'), shown.slice(-20))
+  assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length))
+})
+
+test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => {
+  const MAX = 3000
+  const render = async (text) => {
+    const history = {
+      b2: [
+        { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } },
+        { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } },
+      ],
+    }
+    const replies = [JSON.stringify({ changes: [] })]
+    const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } })
+    await call('state', {})
+    ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+    await reviewed(call)
+    await stop()
+    return requests[0].messages[0].content[0].text
+  }
+  // `ends` is handed the rendered line, not the bare message: `'User: '` is six units, so the
+  // 2000-unit head budget (ceil(3000 * 2/3)) spends them first and cuts at unit 1994 of the
+  // message. The first sample puts the emoji's halves at message units 1993-1994, right on
+  // that cut, so an unfixed slice keeps the high half; the second puts them at the tail start
+  // (3008 - 1000 = 2008), where an unfixed tail begins on the low half. The whole kept line
+  // must stay on the budget within a few units either way, so a cut that throws it away fails.
+  for (const text of [`${'a'.repeat(1993)}😀${'d'.repeat(1006)}`, `${'e'.repeat(2001)}😀${'f'.repeat(999)}`]) {
+    const prompt = await render(text)
+    assert.equal(prompt.isWellFormed(), true, text.slice(0, 6))
+    assert.equal(prompt.includes('\ufffd'), false)
+    const line = prompt.split('\n').find(one => one.startsWith('User: a') || one.startsWith('User: e'))
+    assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`)
+    const marker = ' …(middle cut)… '
+    const at = line.indexOf(marker)
+    assert.ok(at > 0, line.slice(0, 40))
+    const keptHead = line.slice('User: '.length, at)
+    const keptTail = line.slice(at + marker.length)
+    assert.ok(keptHead.length + keptTail.length <= MAX, `over: ${keptHead.length} + ${keptTail.length}`)
+    assert.ok(keptHead.length + keptTail.length > MAX - 10, `short: ${keptHead.length} + ${keptTail.length}`)
+    assert.deepEqual(Buffer.from(keptHead, 'utf8'), Buffer.from(text.slice(0, keptHead.length), 'utf8'))
+    assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8'))
+  }
+})
+
 test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => {
   const history = {
     b2: [

core/test/text.test.js

@@ -76,6 +76,20 @@ test('clip cuts long text and says where the rest is', () => {
 })
 
 const sameBytes = (kept, prefix) => Buffer.from(kept, 'utf8').equals(Buffer.from(prefix, 'utf8'))
+// #1918: the reason is optional. A two-argument call used to print "(cut; undefined)".
+test('clip says only that it cut when it is given no reason (#1918)', () => {
+  const cut = clip('x'.repeat(900), 800)
+  assert.equal(cut.includes('undefined'), false)
+  assert.equal(cut, `${'x'.repeat(800)}… (cut)`)
+})
+
+test('clip keeps the reason it is given byte for byte (#1918)', () => {
+  assert.equal(clip('x'.repeat(900), 800, 'see part 2'), `${'x'.repeat(800)}… (cut; see part 2)`)
+  assert.equal(clip('abcdef', 3, 'see part 2'), 'abc… (cut; see part 2)')
+  assert.equal(clip('abc', 5, 'x'), 'abc')
+  assert.equal(clip('abc', 5), 'abc')
+})
+
 
 test('clip never leaves half of a surrogate pair behind', () => {
   const text = `${'a'.repeat(3999)}😀${'z'.repeat(100)}`

docs/subsystems/core-host.md

@@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core
 | --- | --- |
 | `core/index.js` | Server entry. |
 | `core/src/host/` | Host-side helpers (non-hooks). |
-| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/<date>-<slug>.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. |
+| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/<date>-<slug>.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. |
 | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. |
 | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. |
 | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `<owner>'s Chief` / `<name> · <owner>'s Bot`, `<name> · <owner>`, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). |