host: an outside string that arrives ill-formed reaches visible text (peer name header, room-share notice)

changelog.d/1937-peer-name-well-formed.fixed.md

@@ -0,0 +1 @@
+A peer name that arrives with half of a surrogate pair — a string JSON accepts but is not well formed — is now made well formed before it is cleaned, so the message header a person reads (`[Message from …]`), the `from="…"` attribute of an untrusted-message fence and the recorded `senderName` carry the replacement character instead of a lone half that would not survive a UTF-8 round trip. The same repair now happens when a shared room's envelope is parsed: the notice, a roster entry's name and owner name and a copied chat line are handed on well formed, so the room's standing prompt and the guest's copy of a line no longer carry a broken character. The 60-character name bound, the notice's 2,000-character bound and their code point boundaries are unchanged ([#1937](https://github.com/ThinkFlowLab/FlyBot/issues/1937)).

core/src/host/delivery.js

@@ -34,9 +34,11 @@ export const unwrapUntrusted = (text) => {
   return match ? match[1] : value
 }
 // A name from outside the team, safe on one line of trusted text: no control characters,
-// newlines, brackets or quotes, at most 60 characters.
+// newlines, brackets or quotes, at most 60 characters. JSON admits a lone surrogate, so the
+// input is made well formed first: a name that arrives with half a pair would otherwise reach
+// the header a person reads, where it renders as U+FFFD and cannot survive a UTF-8 round trip.
 export const cleanName = (name, fallback = 'a remote Bot') => {
-  const text = cut(String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"<>`]/g, '').replace(/\s+/g, ' ').trim(), 60).text
+  const text = cut(String(name ?? '').toWellFormed().replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"<>`]/g, '').replace(/\s+/g, ' ').trim(), 60).text
   return text || fallback
 }
 export const fenceUntrusted = (name, text) => `<untrusted-message from="${cleanName(name)}">\n${stripUntrusted(text ?? '')}\n</untrusted-message>`

core/src/shared/room-share.js

@@ -222,6 +222,25 @@ export function roomInviteUrl(handle, ticket, origin = 'https://social.botawaken
   return `${String(origin || '').replace(/\/+$/, '')}/r/${who}/${clean}`
 }
 
+// #1937: the strings a peer sends in here are its own words — the room's notice, a roster entry's
+// names, the chat lines a join copies — and JSON carries an unpaired surrogate escape through as it
+// came, so JSON.parse hands back a string that is not well formed and the notice reaches the room's
+// standing prompt in that shape. Repair the fields this parser passes on, in place: one unit becomes
+// one U+FFFD, so no bound, cut or path around them moves.
+function wellFormedFields(entry, keys) {
+  if (!entry || typeof entry !== 'object') return
+  for (const key of keys) {
+    if (typeof entry[key] === 'string') entry[key] = entry[key].toWellFormed()
+  }
+}
+
+function wellFormedShare(data) {
+  wellFormedFields(data, ['notice'])
+  if (Array.isArray(data.roster)) data.roster.forEach(entry => wellFormedFields(entry, ['name', 'ownerName']))
+  if (Array.isArray(data.history)) data.history.forEach(line => wellFormedFields(line, ['who', 'text']))
+  return data
+}
+
 export function parseRoomShare(text) {
   const raw = String(text ?? '')
   const at = raw.indexOf(ROOM_SHARE_MARK)
@@ -246,7 +265,7 @@ export function parseRoomShare(text) {
     if (data.op === 'ask') return TICKET_SHAPE.test(String(data.ticket ?? '')) ? data : undefined
     if (data.op === 'invite') return TICKET_SHAPE.test(String(data.ticket ?? '')) && data.roomId ? data : undefined
     if (!data.roomId) return undefined
-    return data
+    return wellFormedShare(data)
   } catch {
     return undefined
   }

core/test/fence.test.js

@@ -48,3 +48,30 @@ test('a peer name at the 60 bound keeps the whole character (#P282b)', () => {
   assert.ok(fenced.isWellFormed(), 'the fence header is not well formed')
   assert.ok(fenced.startsWith(`<untrusted-message from="${'a'.repeat(59)}">`))
 })
+
+// A name from outside arrives through JSON, where "\ud83d" is a legal string that is not well
+// formed: the lone half used to travel through cleanName into the header a person reads.
+test('a peer name that arrives with a lone surrogate is made well formed', () => {
+  const broken = 'Ann\ud83dX'
+  assert.equal(broken.isWellFormed(), false, 'the fixture is the broken input')
+  const name = cleanName(broken)
+  assert.equal(name.isWellFormed(), true, `not well formed: ${JSON.stringify(name)}`)
+  assert.equal(name, 'Ann\uFFFDX', 'the lone half becomes the replacement character, in place')
+  assert.ok(name.length <= 60)
+  assert.doesNotMatch(name, /\p{Cc}/u, 'no control character survives the clean')
+})
+
+test('the line a broken peer name reaches stays well formed on every byte', () => {
+  const broken = 'Ann\ud83dX'
+  const fenced = fenceUntrusted(broken, 'hello')
+  const header = fenced.split('\n')[0]
+  assert.equal(header.isWellFormed(), true, `not well formed: ${JSON.stringify(header)}`)
+  assert.equal(header, '<untrusted-message from="Ann\uFFFDX">')
+  assert.equal(Buffer.from(header, 'utf8').toString('utf8'), header, 'the header survives a UTF-8 round trip')
+  // The first line a message is shown with, and the source name beside it, both read from the
+  // same cleanName, so neither can carry the lone half.
+  const shown = `[Message from ${cleanName(broken)}]`
+  assert.equal(shown.isWellFormed(), true, `not well formed: ${JSON.stringify(shown)}`)
+  assert.equal(shown, '[Message from Ann\uFFFDX]')
+  assert.equal(Buffer.from(shown, 'utf8').toString('utf8'), shown)
+})

core/test/remote-room.test.js

@@ -890,6 +890,53 @@ test('a room-share envelope clips peer names on a code point boundary (#1923)',
   }
 })
 
+// #1937: the envelope carries a peer's own words — the room's notice, a roster entry's name and its
+// owner's name, and the chat lines a join copies. JSON accepts a lone surrogate escape, so a peer can
+// put one in each of those strings, and JSON.parse hands them back in the same broken shape; the
+// notice then reaches the room's standing prompt and a line reaches the guest's copy. The parser
+// repairs what it hands on, in place: one unit in, one unit out, no bound moved.
+test('a room-share envelope hands on well formed notice, roster and history text (#1937)', () => {
+  const notice = `${'a'.repeat(19)}\ud83d tail`
+  const name = 'Ann\ud83dX'
+  const ownerName = 'Owner\ud83dY'
+  const who = 'W\ud83dho'
+  const line = 'hello \ud83d world'
+  const packed = encodeRoomShare({
+    op: 'join', roomId: 'war', name: 'War room', inviterName: 'Alice',
+    inviterHome: '@alice.home', inviterHandle: '@chief.alice.home', invitedHandle: '@coder.bob.home', seq: 6,
+    roster: [{ handle: '@ann.home', name, ownerName }],
+    notice,
+    history: [{ who, text: line, seq: 2 }],
+  })
+  // The wire text is well formed already: JSON.stringify escapes an unpaired surrogate. The break
+  // happens on the receiving side, in JSON.parse — this test pins the string the parser hands on.
+  assert.equal(packed.isWellFormed(), true, 'the wire text is well formed')
+  assert.equal(packed.includes('\ud83d'), false, 'the lone surrogate travels as an escape, not as a bare unit')
+  const parsed = parseRoomShare(packed)
+  assert.equal(parsed.op, 'join')
+  const cases = [
+    ['notice', parsed.notice, notice],
+    ['roster[0].name', parsed.roster[0].name, name],
+    ['roster[0].ownerName', parsed.roster[0].ownerName, ownerName],
+    ['history[0].who', parsed.history[0].who, who],
+    ['history[0].text', parsed.history[0].text, line],
+  ]
+  for (const [field, kept, sent] of cases) {
+    assert.equal(typeof kept, 'string', `${field} is in the envelope: ${JSON.stringify(parsed)}`)
+    assert.equal(kept.isWellFormed(), true, `${field} is not well formed: ${JSON.stringify(kept)}`)
+    assert.equal(kept, sent.replace(/\ud83d/g, '\uFFFD'), `${field} replaces the lone surrogate with U+FFFD and keeps everything else`)
+    assert.equal(kept.length, sent.length, `${field} keeps the length it arrived with: ${kept.length} vs ${sent.length}`)
+  }
+  // A receiving home re-sends what it read: a second pass changes nothing.
+  const again = parseRoomShare(encodeRoomShare(parsed))
+  assert.equal(again.notice, notice.replace(/\ud83d/g, '\uFFFD'))
+  assert.equal(again.notice.isWellFormed(), true)
+  // The shortest shape of the same arrival: one unpaired half between two letters.
+  const repaired = parseRoomShare(encodeRoomShare({ op: 'join', roomId: 'war', seq: 8, notice: 'Ann\ud83dX' }))
+  assert.equal(repaired.notice, 'Ann\uFFFDX')
+  assert.equal(repaired.notice.isWellFormed(), true)
+})
+
 // A join copies chat lines into the envelope it sends to another owner, and its own bound is a
 // code-unit one: a line whose 2,000th unit is half of a surrogate pair reached the guest as half of
 // a character. The copied line stops before the pair instead.

patches/ds-bot/0005-social-seams.patch

@@ -985,7 +985,7 @@ diff --git a/src/host/channel.js b/src/host/channel.js
 diff --git a/src/host/delivery.js b/src/host/delivery.js
 --- a/src/host/delivery.js
 +++ b/src/host/delivery.js
-@@ -2,6 +2,62 @@
+@@ -2,6 +2,64 @@
  import { randomUUID } from 'node:crypto'
  import { createUserMessage } from '@deepseek-ai/dsh-llm'
  import { SOURCE_KIND } from './constants.js'
@@ -1021,9 +1021,11 @@ diff --git a/src/host/delivery.js b/src/host/delivery.js
 +  return match ? match[1] : value
 +}
 +// A name from outside the team, safe on one line of trusted text: no control characters,
-+// newlines, brackets or quotes, at most 60 characters.
++// newlines, brackets or quotes, at most 60 characters. JSON admits a lone surrogate, so the
++// input is made well formed first: a name that arrives with half a pair would otherwise reach
++// the header a person reads, where it renders as U+FFFD and cannot survive a UTF-8 round trip.
 +export const cleanName = (name, fallback = 'a remote Bot') => {
-+  const text = cut(String(name ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"<>`]/g, '').replace(/\s+/g, ' ').trim(), 60).text
++  const text = cut(String(name ?? '').toWellFormed().replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, ' ').replace(/[[\]"<>`]/g, '').replace(/\s+/g, ' ').trim(), 60).text
 +  return text || fallback
 +}
 +export const fenceUntrusted = (name, text) => `<untrusted-message from="${cleanName(name)}">\n${stripUntrusted(text ?? '')}\n</untrusted-message>`

patches/ds-bot/files/src/shared/room-share.js

@@ -222,6 +222,25 @@ export function roomInviteUrl(handle, ticket, origin = 'https://social.botawaken
   return `${String(origin || '').replace(/\/+$/, '')}/r/${who}/${clean}`
 }
 
+// #1937: the strings a peer sends in here are its own words — the room's notice, a roster entry's
+// names, the chat lines a join copies — and JSON carries an unpaired surrogate escape through as it
+// came, so JSON.parse hands back a string that is not well formed and the notice reaches the room's
+// standing prompt in that shape. Repair the fields this parser passes on, in place: one unit becomes
+// one U+FFFD, so no bound, cut or path around them moves.
+function wellFormedFields(entry, keys) {
+  if (!entry || typeof entry !== 'object') return
+  for (const key of keys) {
+    if (typeof entry[key] === 'string') entry[key] = entry[key].toWellFormed()
+  }
+}
+
+function wellFormedShare(data) {
+  wellFormedFields(data, ['notice'])
+  if (Array.isArray(data.roster)) data.roster.forEach(entry => wellFormedFields(entry, ['name', 'ownerName']))
+  if (Array.isArray(data.history)) data.history.forEach(line => wellFormedFields(line, ['who', 'text']))
+  return data
+}
+
 export function parseRoomShare(text) {
   const raw = String(text ?? '')
   const at = raw.indexOf(ROOM_SHARE_MARK)
@@ -246,7 +265,7 @@ export function parseRoomShare(text) {
     if (data.op === 'ask') return TICKET_SHAPE.test(String(data.ticket ?? '')) ? data : undefined
     if (data.op === 'invite') return TICKET_SHAPE.test(String(data.ticket ?? '')) && data.roomId ? data : undefined
     if (!data.roomId) return undefined
-    return data
+    return wellFormedShare(data)
   } catch {
     return undefined
   }

patches/ds-bot/files/test/fence.test.js

@@ -48,3 +48,30 @@ test('a peer name at the 60 bound keeps the whole character (#P282b)', () => {
   assert.ok(fenced.isWellFormed(), 'the fence header is not well formed')
   assert.ok(fenced.startsWith(`<untrusted-message from="${'a'.repeat(59)}">`))
 })
+
+// A name from outside arrives through JSON, where "\ud83d" is a legal string that is not well
+// formed: the lone half used to travel through cleanName into the header a person reads.
+test('a peer name that arrives with a lone surrogate is made well formed', () => {
+  const broken = 'Ann\ud83dX'
+  assert.equal(broken.isWellFormed(), false, 'the fixture is the broken input')
+  const name = cleanName(broken)
+  assert.equal(name.isWellFormed(), true, `not well formed: ${JSON.stringify(name)}`)
+  assert.equal(name, 'Ann\uFFFDX', 'the lone half becomes the replacement character, in place')
+  assert.ok(name.length <= 60)
+  assert.doesNotMatch(name, /\p{Cc}/u, 'no control character survives the clean')
+})
+
+test('the line a broken peer name reaches stays well formed on every byte', () => {
+  const broken = 'Ann\ud83dX'
+  const fenced = fenceUntrusted(broken, 'hello')
+  const header = fenced.split('\n')[0]
+  assert.equal(header.isWellFormed(), true, `not well formed: ${JSON.stringify(header)}`)
+  assert.equal(header, '<untrusted-message from="Ann\uFFFDX">')
+  assert.equal(Buffer.from(header, 'utf8').toString('utf8'), header, 'the header survives a UTF-8 round trip')
+  // The first line a message is shown with, and the source name beside it, both read from the
+  // same cleanName, so neither can carry the lone half.
+  const shown = `[Message from ${cleanName(broken)}]`
+  assert.equal(shown.isWellFormed(), true, `not well formed: ${JSON.stringify(shown)}`)
+  assert.equal(shown, '[Message from Ann\uFFFDX]')
+  assert.equal(Buffer.from(shown, 'utf8').toString('utf8'), shown)
+})

patches/ds-bot/files/test/remote-room.test.js

@@ -890,6 +890,53 @@ test('a room-share envelope clips peer names on a code point boundary (#1923)',
   }
 })
 
+// #1937: the envelope carries a peer's own words — the room's notice, a roster entry's name and its
+// owner's name, and the chat lines a join copies. JSON accepts a lone surrogate escape, so a peer can
+// put one in each of those strings, and JSON.parse hands them back in the same broken shape; the
+// notice then reaches the room's standing prompt and a line reaches the guest's copy. The parser
+// repairs what it hands on, in place: one unit in, one unit out, no bound moved.
+test('a room-share envelope hands on well formed notice, roster and history text (#1937)', () => {
+  const notice = `${'a'.repeat(19)}\ud83d tail`
+  const name = 'Ann\ud83dX'
+  const ownerName = 'Owner\ud83dY'
+  const who = 'W\ud83dho'
+  const line = 'hello \ud83d world'
+  const packed = encodeRoomShare({
+    op: 'join', roomId: 'war', name: 'War room', inviterName: 'Alice',
+    inviterHome: '@alice.home', inviterHandle: '@chief.alice.home', invitedHandle: '@coder.bob.home', seq: 6,
+    roster: [{ handle: '@ann.home', name, ownerName }],
+    notice,
+    history: [{ who, text: line, seq: 2 }],
+  })
+  // The wire text is well formed already: JSON.stringify escapes an unpaired surrogate. The break
+  // happens on the receiving side, in JSON.parse — this test pins the string the parser hands on.
+  assert.equal(packed.isWellFormed(), true, 'the wire text is well formed')
+  assert.equal(packed.includes('\ud83d'), false, 'the lone surrogate travels as an escape, not as a bare unit')
+  const parsed = parseRoomShare(packed)
+  assert.equal(parsed.op, 'join')
+  const cases = [
+    ['notice', parsed.notice, notice],
+    ['roster[0].name', parsed.roster[0].name, name],
+    ['roster[0].ownerName', parsed.roster[0].ownerName, ownerName],
+    ['history[0].who', parsed.history[0].who, who],
+    ['history[0].text', parsed.history[0].text, line],
+  ]
+  for (const [field, kept, sent] of cases) {
+    assert.equal(typeof kept, 'string', `${field} is in the envelope: ${JSON.stringify(parsed)}`)
+    assert.equal(kept.isWellFormed(), true, `${field} is not well formed: ${JSON.stringify(kept)}`)
+    assert.equal(kept, sent.replace(/\ud83d/g, '\uFFFD'), `${field} replaces the lone surrogate with U+FFFD and keeps everything else`)
+    assert.equal(kept.length, sent.length, `${field} keeps the length it arrived with: ${kept.length} vs ${sent.length}`)
+  }
+  // A receiving home re-sends what it read: a second pass changes nothing.
+  const again = parseRoomShare(encodeRoomShare(parsed))
+  assert.equal(again.notice, notice.replace(/\ud83d/g, '\uFFFD'))
+  assert.equal(again.notice.isWellFormed(), true)
+  // The shortest shape of the same arrival: one unpaired half between two letters.
+  const repaired = parseRoomShare(encodeRoomShare({ op: 'join', roomId: 'war', seq: 8, notice: 'Ann\ud83dX' }))
+  assert.equal(repaired.notice, 'Ann\uFFFDX')
+  assert.equal(repaired.notice.isWellFormed(), true)
+})
+
 // A join copies chat lines into the envelope it sends to another owner, and its own bound is a
 // code-unit one: a line whose 2,000th unit is half of a surrogate pair reached the guest as half of
 // a character. The copied line stops before the pair instead.