From c6972cc7ec1969ea55c082ec0f3e223b2f89dfd7 Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:54:33 +0800 Subject: [PATCH] Cut the memory review's text on a code point boundary (#1918) --- .../1918-clip-reason-optional.fixed.md | 1 + core/src/host/memory.js | 11 +- core/src/host/text.js | 4 +- core/test/memory.test.js | 69 ++++++++++ core/test/text.test.js | 14 +++ docs/subsystems/core-host.md | 2 +- patches/ds-bot/0001-core-fixes.patch | 119 +++++++++++++++++- 7 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 changelog.d/1918-clip-reason-optional.fixed.md diff --git a/changelog.d/1918-clip-reason-optional.fixed.md b/changelog.d/1918-clip-reason-optional.fixed.md new file mode 100644 index 000000000..6e5a7c389 --- /dev/null +++ b/changelog.d/1918-clip-reason-optional.fixed.md @@ -0,0 +1 @@ +The memory review now keeps whole characters in what it shows the model and writes to the log ([#1918](https://github.com/ThinkFlowLab/FlyBot/issues/1918)): the two bounds in `core/src/host/memory.js` — the 800-unit context line and the 3,000-unit two-ended user line — go through the shared boundary rule, so a cut that lands inside an emoji steps back instead of leaving half a pair, and the tail it keeps is never longer than the budget it had. The exported `clip` in `core/src/host/text.js` also makes its reason optional: a two-argument call now reads `… (cut)` instead of the literal `… (cut; undefined)`, while a call that passes a reason keeps `… (cut; )` byte for byte. diff --git a/core/src/host/memory.js b/core/src/host/memory.js index 17bbc968e..8ea47c4eb 100644 --- a/core/src/host/memory.js +++ b/core/src/host/memory.js @@ -28,6 +28,7 @@ import { BlockAssembler } from '@deepseek-ai/dsh-llm' import { currentParticipants } from './room-membership.js' import { defineTool } from './tool-schema.js' import { codePoints, splitRef, unquote } from './text.js' +import { codePointStart, cut } from '../shared/text-boundary.js' import { output } from './tools.js' import { retiredBot } from './store.js' import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -1143,9 +1144,15 @@ export function install(rt) { } return cursors } - const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} …(cut)` : text) + const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text) // A long message often ends with what the user wants of it, so both ends stay. - const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text) + const ends = (text, max) => { + if (text.length <= max) return text + const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3))) + // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back. + const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length)) + return `${head.text} …(middle cut)… ${tail}` + } // Lines say who spoke. The user's lines are kept longest: what they say is what // the review keeps; the rest is there to make it understood. diff --git a/core/src/host/text.js b/core/src/host/text.js index c2f5e7878..ad6e627db 100644 --- a/core/src/host/text.js +++ b/core/src/host/text.js @@ -56,7 +56,9 @@ export const estimateTokens = (text) => { for (const char of text) tokens += char.codePointAt(0) > 0x2e7f ? 1 : 0.34 return Math.ceil(tokens) } -export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut; ${rest})` : text) +// #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never +// "… (cut; undefined)"; a caller that gives one keeps "… (cut; )" byte for byte. +export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text) export const createClock = (timeZone) => { const clockFormat = new Intl.DateTimeFormat('en-CA', { diff --git a/core/test/memory.test.js b/core/test/memory.test.js index b32718226..c4da9feb9 100644 --- a/core/test/memory.test.js +++ b/core/test/memory.test.js @@ -534,6 +534,75 @@ async function reviewed(call) { throw new Error('the memory review did not finish') } +// #1918: both local cuts of the review keep whole characters. The Bot line goes through the +// two-argument cut at REVIEW_OTHER_LINE_MAX, the user's through the two-ended one at +// REVIEW_USER_LINE_MAX; a boundary that lands inside a surrogate pair must step back, never +// leave half of it in the prompt. +test('a long Bot line cut for the review context keeps whole characters (#1918)', async (t) => { + const long = `${'x'.repeat(791)}😀tail` + const history = { + b2: [ + { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, + { event: { seq: 21, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: long }] } } } }, + { event: { seq: 22, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Now keep it.' }] } } }, + ], + } + const replies = [JSON.stringify({ changes: [] })] + const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) + t.after(() => stop()) + await call('state', {}) + ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) + await reviewed(call) + const prompt = requests[0].messages[0].content[0].text + assert.equal(prompt.isWellFormed(), true) + assert.equal(prompt.includes('\ufffd'), false) + const shown = prompt.split('\n').find(one => one.startsWith('Writer: x')) + assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`) + assert.ok(shown.endsWith('…(cut)'), shown.slice(-20)) + assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length)) +}) + +test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => { + const MAX = 3000 + const render = async (text) => { + const history = { + b2: [ + { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, + { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } }, + ], + } + const replies = [JSON.stringify({ changes: [] })] + const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) + await call('state', {}) + ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) + await reviewed(call) + await stop() + return requests[0].messages[0].content[0].text + } + // `ends` is handed the rendered line, not the bare message: `'User: '` is six units, so the + // 2000-unit head budget (ceil(3000 * 2/3)) spends them first and cuts at unit 1994 of the + // message. The first sample puts the emoji's halves at message units 1993-1994, right on + // that cut, so an unfixed slice keeps the high half; the second puts them at the tail start + // (3008 - 1000 = 2008), where an unfixed tail begins on the low half. The whole kept line + // must stay on the budget within a few units either way, so a cut that throws it away fails. + for (const text of [`${'a'.repeat(1993)}😀${'d'.repeat(1006)}`, `${'e'.repeat(2001)}😀${'f'.repeat(999)}`]) { + const prompt = await render(text) + assert.equal(prompt.isWellFormed(), true, text.slice(0, 6)) + assert.equal(prompt.includes('\ufffd'), false) + const line = prompt.split('\n').find(one => one.startsWith('User: a') || one.startsWith('User: e')) + assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`) + const marker = ' …(middle cut)… ' + const at = line.indexOf(marker) + assert.ok(at > 0, line.slice(0, 40)) + const keptHead = line.slice('User: '.length, at) + const keptTail = line.slice(at + marker.length) + assert.ok(keptHead.length + keptTail.length <= MAX, `over: ${keptHead.length} + ${keptTail.length}`) + assert.ok(keptHead.length + keptTail.length > MAX - 10, `short: ${keptHead.length} + ${keptTail.length}`) + assert.deepEqual(Buffer.from(keptHead, 'utf8'), Buffer.from(text.slice(0, keptHead.length), 'utf8')) + assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8')) + } +}) + test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => { const history = { b2: [ diff --git a/core/test/text.test.js b/core/test/text.test.js index 763a9eb13..0b8b27ac0 100644 --- a/core/test/text.test.js +++ b/core/test/text.test.js @@ -76,6 +76,20 @@ test('clip cuts long text and says where the rest is', () => { }) const sameBytes = (kept, prefix) => Buffer.from(kept, 'utf8').equals(Buffer.from(prefix, 'utf8')) +// #1918: the reason is optional. A two-argument call used to print "(cut; undefined)". +test('clip says only that it cut when it is given no reason (#1918)', () => { + const cut = clip('x'.repeat(900), 800) + assert.equal(cut.includes('undefined'), false) + assert.equal(cut, `${'x'.repeat(800)}… (cut)`) +}) + +test('clip keeps the reason it is given byte for byte (#1918)', () => { + assert.equal(clip('x'.repeat(900), 800, 'see part 2'), `${'x'.repeat(800)}… (cut; see part 2)`) + assert.equal(clip('abcdef', 3, 'see part 2'), 'abc… (cut; see part 2)') + assert.equal(clip('abc', 5, 'x'), 'abc') + assert.equal(clip('abc', 5), 'abc') +}) + test('clip never leaves half of a surrogate pair behind', () => { const text = `${'a'.repeat(3999)}😀${'z'.repeat(100)}` diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 8e6f6b69b..ff5015c6a 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it — the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) — each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `… (cut)` instead of `… (cut; undefined)` while a call with a reason is unchanged. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` · 's Bot`, ` · `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index 106d3f175..d55c29beb 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4696,7 +4696,7 @@ diff --git a/src/host/images.js b/src/host/images.js diff --git a/src/host/memory.js b/src/host/memory.js --- a/src/host/memory.js +++ b/src/host/memory.js -@@ -25,9 +25,12 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSyn +@@ -25,9 +25,13 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSyn import { appendFile, mkdir, rename, unlink, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { BlockAssembler } from '@deepseek-ai/dsh-llm' @@ -4705,6 +4705,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js +import { currentParticipants } from './room-membership.js' +import { defineTool } from './tool-schema.js' +import { codePoints, splitRef, unquote } from './text.js' ++import { codePointStart, cut } from '../shared/text-boundary.js' import { output } from './tools.js' +import { retiredBot } from './store.js' +import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -4925,6 +4926,24 @@ diff --git a/src/host/memory.js b/src/host/memory.js limits: MEMORY_LIMITS, summary: entries.length > 0 && saved !== undefined ? { text: saved.text, at: saved.at } : null, fresh: entries.length === 0 || saved?.hash === hash, +@@ -1083,9 +1083,15 @@ export function install(rt) { + } + return cursors + } +- const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} …(cut)` : text) ++ const clip = (text, max) => (text.length > max ? `${cut(text, max).text} …(cut)` : text) + // A long message often ends with what the user wants of it, so both ends stay. +- const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} …(middle cut)… ${text.slice(-Math.floor(max / 3))}` : text) ++ const ends = (text, max) => { ++ if (text.length <= max) return text ++ const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3))) ++ // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back. ++ const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length)) ++ return `${head.text} …(middle cut)… ${tail}` ++ } + + // Lines say who spoke. The user's lines are kept longest: what they say is what + // the review keeps; the rest is there to make it understood. @@ -1123,17 +1123,28 @@ export function install(rt) { // What is new in a group, from its log: the member's Session holds the same @@ -6356,12 +6375,14 @@ diff --git a/src/host/text.js b/src/host/text.js } // An upper bound: a CJK character is about one token, other text about a third. -@@ -36,7 +36,7 @@ export const estimateTokens = (text) => { +@@ -36,7 +36,9 @@ export const estimateTokens = (text) => { for (const char of text) tokens += char.codePointAt(0) > 0x2e7f ? 1 : 0.34 return Math.ceil(tokens) } -export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, max)}… (cut; ${rest})` : text) -+export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut; ${rest})` : text) ++// #1918: the reason is optional — a caller with nothing to add gets "… (cut)", never ++// "… (cut; undefined)"; a caller that gives one keeps "… (cut; )" byte for byte. ++export const clip = (text, max, rest) => (text.length > max ? `${text.slice(0, codePointEnd(text, max))}… (cut${rest === undefined ? '' : `; ${rest}`})` : text) export const createClock = (timeZone) => { const clockFormat = new Intl.DateTimeFormat('en-CA', { @@ -10376,7 +10397,7 @@ diff --git a/test/memory.test.js b/test/memory.test.js const done = (await call('memory-view', { botId: 'b2' })).value assert.deepEqual([done.summary?.text, done.error], ['## Overview\nYou like tea.', null]) assert.deepEqual(routes(requests).slice(2), ['deepseek/flash@2048']) -@@ -519,7 +519,7 @@ test('a host that only reads the team shows memory but changes none of it', asyn +@@ -519,12 +519,81 @@ test('a host that only reads the team shows memory but changes none of it', asyn // Until no review waits or runs. async function reviewed(call) { for (let tries = 0; tries < 200; tries += 1) { @@ -10385,6 +10406,80 @@ diff --git a/test/memory.test.js b/test/memory.test.js if ((await call('state', {})).value.memoryReviews === 0) return } throw new Error('the memory review did not finish') + } + ++// #1918: both local cuts of the review keep whole characters. The Bot line goes through the ++// two-argument cut at REVIEW_OTHER_LINE_MAX, the user's through the two-ended one at ++// REVIEW_USER_LINE_MAX; a boundary that lands inside a surrogate pair must step back, never ++// leave half of it in the prompt. ++test('a long Bot line cut for the review context keeps whole characters (#1918)', async (t) => { ++ const long = `${'x'.repeat(791)}😀tail` ++ const history = { ++ b2: [ ++ { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, ++ { event: { seq: 21, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: long }] } } } }, ++ { event: { seq: 22, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Now keep it.' }] } } }, ++ ], ++ } ++ const replies = [JSON.stringify({ changes: [] })] ++ const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) ++ t.after(() => stop()) ++ await call('state', {}) ++ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) ++ await reviewed(call) ++ const prompt = requests[0].messages[0].content[0].text ++ assert.equal(prompt.isWellFormed(), true) ++ assert.equal(prompt.includes('\ufffd'), false) ++ const shown = prompt.split('\n').find(one => one.startsWith('Writer: x')) ++ assert.ok(shown !== undefined, `the Bot line is in the review context:\n${prompt}`) ++ assert.ok(shown.endsWith('…(cut)'), shown.slice(-20)) ++ assert.ok(shown.length <= 'Writer: '.length + 800 + ' …(cut)'.length, String(shown.length)) ++}) ++ ++test('a long user line cut for the review keeps whole characters at both ends (#1918)', async () => { ++ const MAX = 3000 ++ const render = async (text) => { ++ const history = { ++ b2: [ ++ { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, ++ { event: { seq: 21, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text }] } } }, ++ ], ++ } ++ const replies = [JSON.stringify({ changes: [] })] ++ const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) ++ await call('state', {}) ++ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) ++ await reviewed(call) ++ await stop() ++ return requests[0].messages[0].content[0].text ++ } ++ // `ends` is handed the rendered line, not the bare message: `'User: '` is six units, so the ++ // 2000-unit head budget (ceil(3000 * 2/3)) spends them first and cuts at unit 1994 of the ++ // message. The first sample puts the emoji's halves at message units 1993-1994, right on ++ // that cut, so an unfixed slice keeps the high half; the second puts them at the tail start ++ // (3008 - 1000 = 2008), where an unfixed tail begins on the low half. The whole kept line ++ // must stay on the budget within a few units either way, so a cut that throws it away fails. ++ for (const text of [`${'a'.repeat(1993)}😀${'d'.repeat(1006)}`, `${'e'.repeat(2001)}😀${'f'.repeat(999)}`]) { ++ const prompt = await render(text) ++ assert.equal(prompt.isWellFormed(), true, text.slice(0, 6)) ++ assert.equal(prompt.includes('\ufffd'), false) ++ const line = prompt.split('\n').find(one => one.startsWith('User: a') || one.startsWith('User: e')) ++ assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`) ++ const marker = ' …(middle cut)… ' ++ const at = line.indexOf(marker) ++ assert.ok(at > 0, line.slice(0, 40)) ++ const keptHead = line.slice('User: '.length, at) ++ const keptTail = line.slice(at + marker.length) ++ assert.ok(keptHead.length + keptTail.length <= MAX, `over: ${keptHead.length} + ${keptTail.length}`) ++ assert.ok(keptHead.length + keptTail.length > MAX - 10, `short: ${keptHead.length} + ${keptTail.length}`) ++ assert.deepEqual(Buffer.from(keptHead, 'utf8'), Buffer.from(text.slice(0, keptHead.length), 'utf8')) ++ assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8')) ++ } ++}) ++ + test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => { + const history = { + b2: [ @@ -994,7 +994,7 @@ test('a turn whose Session moves to a fresh part before its review is still revi test('config.memoryReview false leaves memory to the Bots\' own remember calls', async (t) => { const { ctx, call, section, requests, stop } = start({ config: { memoryReview: false, memoryReviewDelayMs: 0 } }) @@ -11971,11 +12066,25 @@ diff --git a/test/text.test.js b/test/text.test.js test('textOf joins the text blocks only', () => { assert.equal(textOf([{ type: 'text', text: ' a ' }, { type: 'image' }, { type: 'text', text: 'b' }, { type: 'text', text: 3 }]), 'a \nb') -@@ -63,6 +63,41 @@ test('clip cuts long text and says where the rest is', () => { +@@ -63,6 +63,55 @@ test('clip cuts long text and says where the rest is', () => { assert.equal(clip('abcdef', 3, 'see part 2'), 'abc… (cut; see part 2)') }) +const sameBytes = (kept, prefix) => Buffer.from(kept, 'utf8').equals(Buffer.from(prefix, 'utf8')) ++// #1918: the reason is optional. A two-argument call used to print "(cut; undefined)". ++test('clip says only that it cut when it is given no reason (#1918)', () => { ++ const cut = clip('x'.repeat(900), 800) ++ assert.equal(cut.includes('undefined'), false) ++ assert.equal(cut, `${'x'.repeat(800)}… (cut)`) ++}) ++ ++test('clip keeps the reason it is given byte for byte (#1918)', () => { ++ assert.equal(clip('x'.repeat(900), 800, 'see part 2'), `${'x'.repeat(800)}… (cut; see part 2)`) ++ assert.equal(clip('abcdef', 3, 'see part 2'), 'abc… (cut; see part 2)') ++ assert.equal(clip('abc', 5, 'x'), 'abc') ++ assert.equal(clip('abc', 5), 'abc') ++}) ++ + +test('clip never leaves half of a surrogate pair behind', () => { + const text = `${'a'.repeat(3999)}😀${'z'.repeat(100)}`