core-host: cut the decision and meeting archives on a code point boundary

changelog.d/1912-archive-cut-boundary.fixed.md

@@ -0,0 +1 @@
+A decision or a meeting archive no longer stores half of an emoji ([#1912](https://github.com/ThinkFlowLab/FlyBot/issues/1912)): the host's one `cut` helper stops a bounded prefix before the surrogate pair it would split, and every archive writer now uses it instead of `slice(0, max)` — `decisions.jsonl`, `decisions/<date>-<slug>.md`, and the claim line, diagnostics report, activity-feed label and browser text beside them — so the stored record round-trips through UTF-8 as the whole character instead of U+FFFD.

core/src/host/browser.js

@@ -3,6 +3,7 @@
 // the app window, not here, so `read_browser` reaches it through the window: the client
 // holds a long poll open (`browser-wait`) that reports which Bots have a page open and
 // carries read requests back; the client answers with `browser-result`.
+import { cut } from './text.js'
 import { defineTool } from './tool-schema.js'
 import { output } from './tools.js'
 
@@ -131,7 +132,7 @@ export function createBrowserChannel({ now = Date.now, waitMs = WAIT_MS, staleMs
   return { wait, result, read, openPages, dispose }
 }
 
-const clean = (value, max) => String(value ?? '').replace(/\s+/g, ' ').trim().slice(0, max)
+const clean = (value, max) => cut(String(value ?? '').replace(/\s+/g, ' ').trim(), max).text
 
 /**
  * Where a part of the text that starts at `start` ends: at a blank line, else a line break,

core/src/host/decisions.js

@@ -10,6 +10,7 @@
 // can show it without reading the file.
 import { appendFile, mkdir, open, readFile, rename, stat, writeFile } from 'node:fs/promises'
 import { dirname, join } from 'node:path'
+import { cut } from './text.js'
 import { defineTool } from './tool-schema.js'
 import { looksSecret } from './memory.js'
 import { output } from './tools.js'
@@ -23,7 +24,7 @@ const KEPT = 2_000
 const MAX_BYTES = 1_000_000
 const NAME_SAFE = /^[\w-]{1,128}$/
 
-const oneLine = (text, max) => String(text ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max)
+const oneLine = (text, max) => cut(String(text ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/\s+/g, ' ').trim(), max).text
 
 export function install(rt) {
   if (rt.config?.decisions !== true) {

core/src/host/diagnostics.js

@@ -10,6 +10,7 @@ import { arch, homedir, platform, release } from 'node:os'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { format } from 'node:util'
+import { cut } from './text.js'
 
 export const KEEP = 200
 const LOG_MAX_BYTES = 1 << 20
@@ -47,7 +48,8 @@ export function cleanText(text, { redact = value => value, home = homedir() } =
     const doubled = home.replace(/\\/g, '\\\\')
     if (doubled !== home) out = out.replace(new RegExp(quote(doubled), 'g'), '~')
   }
-  return out.length > TEXT_MAX ? `${out.slice(0, TEXT_MAX)}…` : out
+  const kept = cut(out, TEXT_MAX)
+  return kept.cut ? `${kept.text}…` : out
 }
 
 const errorText = error => `${error.name && error.name !== 'Error' ? `${error.name}: ` : ''}${error.message}`

core/src/host/flybot-feed.js

@@ -8,6 +8,7 @@
 // user pasted or a tool printed. Strings that came from outside the team (a tool name
 // from an MCP server, a remote handle, an error code) are cleaned here.
 import { SOURCE_KIND } from './constants.js'
+import { cut } from './text.js'
 import { normRemoteHandle, remoteBotTitle } from '../shared/remote-label.js'
 
 export const FEED_EVENTS = Object.freeze(['turn.start', 'turn.end', 'tool.start', 'tool.end', 'exchange.delivered', 'decision'])
@@ -22,7 +23,7 @@ const CONTROLS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g
 export const cleanLabel = (value, max = 80) => {
   let out = String(value ?? '').replace(CONTROLS, ' ')
   for (let previous; previous !== out;) { previous = out; out = out.replace(/<\s*\/?\s*untrusted-?/gi, '') }
-  return out.replace(/\s+/g, ' ').trim().slice(0, max)
+  return cut(out.replace(/\s+/g, ' ').trim(), max).text
 }
 // An error code is an enumerated value (RATE_LIMIT, AUTH): upper-case words only, so free text or a key put in the field is not kept.
 const cleanCode = code => (typeof code === 'string' && /^[A-Z][A-Z0-9_]{0,39}$/.test(code) ? code : 'UNKNOWN')

core/src/host/groups.js

@@ -21,7 +21,7 @@ import { createUserMessage } from '@deepseek-ai/dsh-llm'
 import { defineTool } from './tool-schema.js'
 import { ROOM_LATE_ROUNDS, SOURCE_KIND } from './constants.js'
 import { OWNER, OTHER, SCHEDULED, combine } from './turn-origin.js'
-import { escapeRegExp, textOf } from './text.js'
+import { cut, escapeRegExp, textOf } from './text.js'
 import { mentionSpans } from '../shared/mentions.js'
 import { GROUP_REF, output } from './tools.js'
 import { failureLine } from './turns.js'
@@ -43,7 +43,7 @@ export const CLAIM_CUE_REPEAT_MS = 30 * 60 * 1000
 // that requires the line is (#1253), not because the room is assumed to speak Chinese.
 const CLAIM_CUE_REF_MAX = 60
 const CLAIM_CUE_STEP_MAX = 100
-const claimLine = (value, max) => String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max)
+const claimLine = (value, max) => cut(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim(), max).text
 export function claimCueText(task) {
   const title = claimLine(task?.title, 300)
   if (title === '') return ''

core/src/host/meetings.js

@@ -8,6 +8,7 @@
 import { createHash, randomBytes } from 'node:crypto'
 import { link, mkdir, open, readFile, rm } from 'node:fs/promises'
 import { join } from 'node:path'
+import { cut } from './text.js'
 import { defineTool } from './tool-schema.js'
 import { output } from './tools.js'
 import { workDirFor } from './project-state.js'
@@ -23,7 +24,7 @@ const NAME_MAX = 80
 const PARTICIPANTS_MAX = 32
 const OPEN_STATES = new Set(['open', 'deliberating'])
 
-const oneLine = (value, max) => String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, max)
+const oneLine = (value, max) => cut(String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim(), max).text
 // A slug keeps letters and digits of every script, so two different Chinese questions do not both
 // fall back to `meeting` and collide on one day's file.
 const slugOf = value => oneLine(value, 80).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, '').slice(0, 80)

core/src/host/text.js

@@ -43,6 +43,10 @@ const isLowSurrogate = code => code >= 0xdc00 && code <= 0xdfff
 const codePointStart = (text, at) => (at > 0 && isLowSurrogate(text.charCodeAt(at)) && isHighSurrogate(text.charCodeAt(at - 1)) ? at - 1 : at)
 const codePointEnd = (text, end) => (end > 0 && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) ? end - 1 : end)
 
+// A bounded prefix that never splits a surrogate pair (#1912): every other host module cuts its own
+// archive text, and a half pair is stored as U+FFFD. `cut` says whether anything was dropped.
+export const cut = (text, max) => ({ text: text.slice(0, codePointEnd(text, max)), cut: text.length > max })
+
 // The part of a long message around the first match, so a search hit stays visible.
 export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => {
   if (text.length <= max) return text

core/test/decisions.test.js

@@ -84,3 +84,16 @@ test('#603: a byte cap bounds an oversized log left by an older build', async (t
   assert.equal(log.at(-1).option, 'after the byte cap')
   assert.equal(log.length, 2)
 })
+
+test('#1912: an option cut at an emoji boundary stays well formed in the log', async (t) => {
+  const { home, tool, stop } = await boot({ decisions: true })
+  t.after(stop)
+  // The 200-unit cut lands inside the emoji, so a code-unit slice stores half a surrogate pair.
+  const option = `${'a'.repeat(199)}😀tail`
+  await tool('record_decision', { option, choice: 'used' })
+  const text = readFileSync(join(home, 'decisions.jsonl'), 'utf8')
+  const [entry] = text.trim().split('\n').map(line => JSON.parse(line))
+  assert.equal(entry.option.isWellFormed(), true, `the stored option must stay well formed, got: ${JSON.stringify(entry.option.slice(-4))}`)
+  assert.equal(entry.option.length <= 200, true, `the stored option must stay within 200 units, got ${entry.option.length}`)
+  assert.equal(option.startsWith(entry.option), true, 'the stored option must be a prefix of the recorded one')
+})

core/test/meetings-persist.test.js

@@ -3,7 +3,7 @@
 // second decide is refused, with no second decision file and no second card), and an open meeting
 // keeps its recorded positions.
 import assert from 'node:assert/strict'
-import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { test } from 'node:test'
@@ -85,3 +85,22 @@ test('P-209: an open meeting and a recorded position survive a restart', async (
   assert.match(read, /\[deliberating\]/, `an open meeting must survive a restart, got: ${read.split('\n')[0]}`)
   assert.match(read, /- Chief: agree — the existing store/, `the position must survive, got: ${read}`)
 })
+
+test('#1912: a decision cut at an emoji boundary writes no replacement character into the archive', async (t) => {
+  const { dir, meeting } = await setup(t)
+  const text = await meeting({ action: 'open', question: 'Which storage shape should Stage 4 use?', participants: ['Chief', 'Dev', 'Reviewer'] })
+  const id = text.split(' ')[0]
+  await positionAll(meeting, id)
+  // The 600-unit cut lands inside the emoji, so a code-unit slice keeps half of a surrogate pair.
+  const decision = `${'a'.repeat(599)}😀tail`
+  await meeting({ action: 'decide', meeting: id, owner: 'Chief', decision, evidence: [{ pr: 1912 }] })
+  const [file] = readdirSync(join(dir, 'decisions')).filter(name => name.endsWith('.md'))
+  const bytes = readFileSync(join(dir, 'decisions', file))
+  const written = bytes.toString('utf8')
+  assert.equal(written.includes('\ufffd'), false, 'the archive must not hold U+FFFD')
+  assert.equal(Buffer.from(written, 'utf8').equals(bytes), true, 'the archive must round-trip through UTF-8 unchanged')
+  const lines = written.split('\n')
+  const kept = lines[lines.indexOf('## Decision') + 2]
+  assert.equal(decision.startsWith(kept), true, `the kept body must be a prefix of the decision, got: ${JSON.stringify(kept)}`)
+  assert.equal(kept.length <= 600, true, `the kept body must stay within 600 units, got ${kept.length}`)
+})

docs/subsystems/core-host.md

@@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core
 | --- | --- |
 | `core/index.js` | Server entry. |
 | `core/src/host/` | Host-side helpers (non-hooks). |
-| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). |
+| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit — one character costs one whatever its UTF-16 width — and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it — the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/<date>-<slug>.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` — so a decision or a meeting record holds the whole character instead of U+FFFD. |
 | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative — an addressed Bot answers as before. |
 | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. |
 | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `<owner>'s Chief` / `<name> · <owner>'s Bot`, `<name> · <owner>`, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). |

docs/subsystems/plugin-runtime.md

@@ -35,7 +35,7 @@ FlyBot's plugin API surface on top of the patched core: host hooks, client slots
 | File | Role |
 | --- | --- |
 | `core/src/host/hooks.js` | Host hook points: send middleware, transports, grants, the owner-bound held-send approvals (#52), task hold/decide/delete seams (#54), and `bot.update({ name?, hidden? })` for plugin rename/hide (#452). A hold's waiting status is the shared rule `src/shared/approval-waiting.js` `isApprovalWaiting()`: `decide()`, `heldOf()` (the chat card), `PENDING_MAX`, the in-flight filter and the `APPROVALS_KEPT` cleanup all ask it instead of comparing to `pending`, so a `pending_approval` hold is decidable, drawn as waiting, counted and never cleaned up mid-wait (#754). It also exposes `flybot.approvals.stats()` — per-Bot, per-status counts of held sends, read-only (#396/#57 direction). A dispatch that fails keeps its error on the record, and an error the sender marks `noRetry: true` — a refusal nothing can change, such as a friend’s Bot no longer shared — reaches the chat view as `retry: false` so the card draws no Retry (#954); every other failure stays retryable. Whether a failure can still be retried is one rule in `src/shared/held-retry.js` (#968): `canRetry(record)` reads the flags or, for an older record, the stored sentence, and `approvals.list` marks a failed terminal approval `retry: false` so a dock card and the chat card answer the same question the same way. `rooms.list()` copies `sharedFrom.remoteRoomId`, `ticket` and `inviterHandle` when a room has them, so a mention of an inbound seat can name the host's room id inside the share line (#1126). |
-| `core/src/host/flybot-feed.js` | Activity feed: folds a Bot Session's `session/event`s into `flybot.on('turn.start' \| 'turn.end' \| 'tool.start' \| 'tool.end' \| 'exchange.delivered' \| 'decision')` (#241). `turn.end` also carries the turn's token counts (summed from its `assistant/message` usage, floored and capped, counts only) under `tokens` (#357). `decision` is fired by `decisions.js` (#396), not folded from a Session event. |
+| `core/src/host/flybot-feed.js` | Activity feed: folds a Bot Session's `session/event`s into `flybot.on('turn.start' \| 'turn.end' \| 'tool.start' \| 'tool.end' \| 'exchange.delivered' \| 'decision')` (#241). `turn.end` also carries the turn's token counts (summed from its `assistant/message` usage, floored and capped, counts only) under `tokens` (#357). `decision` is fired by `decisions.js` (#396), not folded from a Session event. Since #1912 `cleanLabel` bounds its text through `cut` in `text.js`, so a cleaned tool name or remote handle never ends on half of a surrogate pair. |
 | `core/src/client/flybot-slots.js` | Client slots, including the approval card queue (`flybot.approvals`, `flybot/decide`) and `flybot.sidebar.structure` (`parentOf`, optional `include`, `menuItems`, `nodeActions`). Every `flybot.sidebar.section` entry is wrapped in `div.bt-flybot-section`, and a wrapper a section leaves empty (its component renders null) draws no separator: the `:empty` rule zeroes that margin, border and padding, keyed on emptiness rather than on a section id, so the next null-rendering section is covered too (#818; `core/test/flybot-section-empty.test.js` renders the host entry and pins both rules). Its Approve / Reject and the held-reply card both ask `approval-waiting.js` `isApprovalWaiting()` (`pending` and `pending_approval` wait), so a hold carrying the spelling Bot Social's own outbound hold uses stays answerable (#734). Its dock card draws **Retry** only while `src/shared/held-retry.js` `canRetry(approval)` allows it — a terminal failure asks one shared predicate instead of `status === 'failed'` (#968) — and it publishes `globalThis.__flybot__.canRetry` so a classic-script plugin's own dock asks the same rule rather than keeping a second copy. |
 | `core/src/client/flybot-registry.js` | Registry. A friend's Bot row in the @ menu carries `remoteMentionLabel` — the one name its bubble and the sidebar preview wear — so the menu inserts the name the user reads, while the row keeps the handle as its value and the send still routes by handle (#967). |
 | `core/types/flybot.d.ts` | Types (`BotView.hidden`, `bot.update`, …). |