Writing: keep secrets out of Personal History and the next-word predictor

Sources/TranscriptedWriting/Runtime/SaveMyWriting/WritingDayFileRecorder.swift

@@ -6,9 +6,12 @@ import Foundation
 /// Save my writing inside the app: keyboard events in, Markdown day files
 /// out. Gates every batch (Save my writing on, the current history and
 /// consent, the app scope), composes entries, and appends each closed entry
-/// to `<writing folder>/Writing_<date>.md`. One serial queue owns the composer
-/// and every write, so `flush()` at quit and `deleteAll()` never race an
-/// append. Not part of Tilde.
+/// to `<writing folder>/Writing_<date>.md`. It's also the only way into
+/// Personal History: the events of each entry that closed with nothing to
+/// scrub go to `releaseToPersonalHistory`, and an entry with a secret in it
+/// never does. One serial queue owns the composer and every write, so
+/// `flush()` at quit and `deleteAll()` never race an append. Not part of
+/// Tilde.
 final class WritingDayFileRecorder: @unchecked Sendable {
     /// Read fresh for every batch and every write, like Tilde's settings.
     struct Gate: Equatable, Sendable {
@@ -76,6 +79,7 @@ final class WritingDayFileRecorder: @unchecked Sendable {
     private let didWrite: @Sendable (URL) -> Void
     private let writeFailed: @Sendable () -> Void
     private let writeProblemStarted: @Sendable (WritingDayFileStore.StoreError) -> Void
+    private let releaseToPersonalHistory: @Sendable ([PersonalHistoryEvent]) -> Void
     private var composer: WritingEntryComposer
     private var rememberedEventIDs: Set<String> = []
     private var rememberedEventOrder: [String] = []
@@ -94,6 +98,8 @@ final class WritingDayFileRecorder: @unchecked Sendable {
     /// `writeFailed` runs on every failed append. `writeProblemStarted` runs
     /// once per problem: on the first failure after a success (or before any
     /// write), not again until a write succeeds and another fails.
+    /// `releaseToPersonalHistory` gets cleared events in keyboard order, on
+    /// the recorder's queue; it must hand them off, not wait on them.
     init(
         directory: @escaping @Sendable () -> URL,
         gate: @escaping @Sendable () -> Gate,
@@ -106,7 +112,8 @@ final class WritingDayFileRecorder: @unchecked Sendable {
         },
         didWrite: @escaping @Sendable (URL) -> Void = { _ in },
         writeFailed: @escaping @Sendable () -> Void = {},
-        writeProblemStarted: @escaping @Sendable (WritingDayFileStore.StoreError) -> Void = { _ in }
+        writeProblemStarted: @escaping @Sendable (WritingDayFileStore.StoreError) -> Void = { _ in },
+        releaseToPersonalHistory: @escaping @Sendable ([PersonalHistoryEvent]) -> Void = { _ in }
     ) {
         self.directory = directory
         self.gate = gate
@@ -117,6 +124,7 @@ final class WritingDayFileRecorder: @unchecked Sendable {
         self.didWrite = didWrite
         self.writeFailed = writeFailed
         self.writeProblemStarted = writeProblemStarted
+        self.releaseToPersonalHistory = releaseToPersonalHistory
         composer = WritingEntryComposer { milliseconds in
             WritingDayFileFormatter.entryID(
                 forMilliseconds: milliseconds,
@@ -139,12 +147,18 @@ final class WritingDayFileRecorder: @unchecked Sendable {
 
     /// Writes the open entry once it has been idle for 2 minutes.
     func closeIdleEntries() {
-        queue.sync { write(composer.closeIdle(now: now())) }
+        queue.sync {
+            write(composer.closeIdle(now: now()))
+            releaseClearedHistory()
+        }
     }
 
     /// Writes whatever is open. The app calls it at quit.
     func flush() {
-        queue.sync { write(composer.closeAll()) }
+        queue.sync {
+            write(composer.closeAll())
+            releaseClearedHistory()
+        }
     }
 
     /// Delete all writing: drops the open entry and anything unwritten, then
@@ -202,6 +216,25 @@ final class WritingDayFileRecorder: @unchecked Sendable {
         }
         guard !admitted.isEmpty else { return }
         write(composer.ingest(admitted, receivedAt: now()))
+        releaseClearedHistory()
+    }
+
+    /// Hands Personal History the events of entries that closed clean,
+    /// re-checked against the gate like a write: turning Save my writing off
+    /// or narrowing the scope drops them here too.
+    private func releaseClearedHistory() {
+        let cleared = composer.takeClearedHistory()
+        guard !cleared.isEmpty else { return }
+        let gate = gate()
+        let admitted = cleared.filter {
+            gate.admits(
+                appBundleIdentifier: $0.appBundleIdentifier,
+                historyIdentifier: $0.historyIdentifier,
+                consentIdentifier: $0.consentIdentifier
+            )
+        }
+        guard !admitted.isEmpty else { return }
+        releaseToPersonalHistory(admitted)
     }
 
     private func remember(_ eventID: String) -> Bool {
@@ -272,22 +305,72 @@ final class WritingDayFileRecorder: @unchecked Sendable {
     }
 }
 
-/// What the socket server ingests: every Personal History batch goes to
-/// Tilde's controller (the encrypted log and the predictor) and to the day
-/// files. The app scope is re-checked here the way Tilde's app re-checks its
-/// exclusions; the keyboard applied it already. Not part of Tilde.
+/// What the socket server ingests. Every batch goes to the day files'
+/// recorder, which is also the way into Tilde's controller (the encrypted
+/// log and the predictor): an entry reaches it once it has closed with
+/// nothing to scrub (`WritingDayFileRecorder`, `PersonalHistoryRelay`).
+/// In Tilde a batch went to the controller as it arrived. The app scope is
+/// re-checked here the way Tilde's app re-checks its exclusions; the
+/// keyboard applied it already. Not part of Tilde.
 struct WritingHistoryIngest: PersonalHistoryIngesting {
-    let personalHistory: any PersonalHistoryIngesting
     let dayFiles: WritingDayFileRecorder
     let appScope: @Sendable () -> WritingAppScope
 
+    /// `true` once the batch is composed: the keyboard doesn't resend it.
+    /// Personal History takes it later, when its entry closes.
     func ingest(_ events: [PersonalHistoryEvent]) async -> Bool {
         guard PersonalHistoryEvent.validBatch(events) else { return false }
         let scope = appScope()
         let inScope = events.filter { scope.includes($0.appBundleIdentifier) }
         // Acknowledged and never kept, like an excluded app in Tilde.
         guard !inScope.isEmpty else { return true }
         await dayFiles.ingest(inScope)
-        return await personalHistory.ingest(inScope)
+        return true
+    }
+}
+
+/// Hands the events Save my writing cleared to Personal History, in the
+/// order they cleared, in batches the controller takes. A batch the
+/// controller refuses (storage down) is not retried: the controller's
+/// storage health already shows it isn't saving. Not part of Tilde.
+final class PersonalHistoryRelay: @unchecked Sendable {
+    private let personalHistory: any PersonalHistoryIngesting
+    private let lock = NSLock()
+    private var tail = OrderedAsyncTaskTail()
+
+    init(personalHistory: any PersonalHistoryIngesting) {
+        self.personalHistory = personalHistory
+    }
+
+    /// Returns at once; the batches go out in call order.
+    func send(_ events: [PersonalHistoryEvent]) {
+        let batches = Self.batches(events)
+        guard !batches.isEmpty else { return }
+        let personalHistory = personalHistory
+        lock.withLock {
+            _ = tail.enqueue {
+                for batch in batches { _ = await personalHistory.ingest(batch) }
+            }
+        }
+    }
+
+    /// Waits until everything sent so far has been handed over.
+    func drain() async {
+        let marker = lock.withLock { tail.enqueue {} }
+        _ = await marker.result
+    }
+
+    /// Consecutive batches the controller accepts, in order.
+    private static func batches(_ events: [PersonalHistoryEvent]) -> [[PersonalHistoryEvent]] {
+        var batches: [[PersonalHistoryEvent]] = []
+        var remaining = events[...]
+        while !remaining.isEmpty {
+            let batch = PersonalHistoryEvent.boundedBatchPrefix(Array(remaining))
+            // Every event fits a batch on its own; this only guards the loop.
+            guard !batch.isEmpty else { break }
+            batches.append(batch)
+            remaining = remaining.dropFirst(batch.count)
+        }
+        return batches
     }
 }

Sources/TranscriptedWriting/Runtime/SaveMyWriting/WritingEntryComposer.swift

@@ -16,8 +16,13 @@ import Foundation
 /// no letters, like a card or OTP box) don't count toward the 3 words, so a
 /// card typed over four boxes stays one entry the scrubber can see whole. Every closed entry goes through
 /// `WritingSecretScrubber` before it's returned; an entry that was nothing
-/// but a secret isn't returned at all. Pure: the caller passes the clock.
-/// Not part of Tilde.
+/// but a secret isn't returned at all.
+///
+/// Personal History (the encrypted log and the next-word predictor) gets the
+/// keyboard's events only through here, once their entry has closed:
+/// `takeClearedHistory()`. An entry the scrubber redacts anything from, as
+/// saved or as typed before Backspace, gives Personal History nothing. Pure:
+/// the caller passes the clock. Not part of Tilde.
 struct WritingEntryComposer {
     static let idleGapMilliseconds: Int64 = 120_000
     static let minimumCharacters = 2
@@ -51,6 +56,9 @@ struct WritingEntryComposer {
         let firstTimestampMilliseconds: Int64
         var lastActivityMilliseconds: Int64
         var pieces: [Piece] = []
+        /// The typed and accepted events as the keyboard sent them, for
+        /// Personal History. Never deletions: Personal History doesn't take them.
+        var historyEvents: [PersonalHistoryEvent] = []
 
         var text: String { pieces.map(\.text).joined() }
 
@@ -103,6 +111,7 @@ struct WritingEntryComposer {
     private let makeEntryID: @Sendable (Int64) -> String
     private var open: OpenEntry?
     private var recent: [ContextKey: RecentLines] = [:]
+    private var clearedHistory: [PersonalHistoryEvent] = []
 
     /// `makeEntryID` gets the first keystroke's time in milliseconds.
     init(makeEntryID: @escaping @Sendable (Int64) -> String) {
@@ -111,6 +120,15 @@ struct WritingEntryComposer {
 
     var hasOpenEntry: Bool { open != nil }
 
+    /// The typed and accepted events of every entry closed since the last
+    /// call, in keyboard order, and forgets them. Never deletions. Entries
+    /// too short to save still count; an entry that held a secret adds
+    /// nothing.
+    mutating func takeClearedHistory() -> [PersonalHistoryEvent] {
+        defer { clearedHistory.removeAll() }
+        return clearedHistory
+    }
+
     /// Takes one batch in keyboard order. `receivedAt` counts as activity
     /// for the entry the batch ends in: the keyboard sends a batch shortly
     /// after the last key in it, while an event's own timestamp is its first
@@ -168,9 +186,11 @@ struct WritingEntryComposer {
     }
 
     /// Drops the open entry unsaved: Save my writing went off, or delete all.
+    /// Personal History gets none of it either.
     mutating func discardOpenEntry() {
         open = nil
         recent.removeAll()
+        clearedHistory.removeAll()
     }
 
     private mutating func closeOpen() -> Entry? {
@@ -186,13 +206,15 @@ struct WritingEntryComposer {
             ),
             for: Self.contextKey(entry)
         )
-        guard typed.count >= Self.minimumCharacters else { return nil }
         let scrubbed = WritingSecretScrubber.scrub(
             typed,
             appBundleIdentifier: entry.appBundleIdentifier,
             precedingLines: context
         )
-        guard !scrubbed.isOnlyRedactions else { return nil }
+        if !Self.holdsSecret(entry, typed: typed, scrubbed: scrubbed, context: context) {
+            clearedHistory += entry.historyEvents
+        }
+        guard typed.count >= Self.minimumCharacters, !scrubbed.isOnlyRedactions else { return nil }
         let text = scrubbed.clean.trimmingCharacters(in: .whitespacesAndNewlines)
         guard text.count >= Self.minimumCharacters else { return nil }
         let wordCount = Self.wordCount(text)
@@ -212,6 +234,40 @@ struct WritingEntryComposer {
         )
     }
 
+    /// Whether the scrubber redacts anything from the entry as saved, or from
+    /// it as the keyboard sent it. The sent text matters when Backspace took
+    /// a secret back out: the saved entry no longer has it, but Personal
+    /// History would store every event, the deleted text included. The
+    /// scrubber changes text only to redact, so any change counts.
+    private static func holdsSecret(
+        _ entry: OpenEntry,
+        typed: String,
+        scrubbed: WritingSecretScrubber.Result,
+        context: [String]
+    ) -> Bool {
+        guard scrubbed.clean == typed else { return true }
+        let sent = sentText(entry.historyEvents)
+        guard sent != typed else { return false }
+        return WritingSecretScrubber.scrub(
+            sent,
+            appBundleIdentifier: entry.appBundleIdentifier,
+            precedingLines: context
+        ).clean != sent
+    }
+
+    /// The events' text the way Personal History keeps it: one line per
+    /// keyboard segment, and each Backspace there starts a new segment.
+    private static func sentText(_ events: [PersonalHistoryEvent]) -> String {
+        var text = ""
+        var session: String?
+        for event in events {
+            if let session, session != event.sessionIdentifier, !text.isEmpty { text += "\n" }
+            session = event.sessionIdentifier
+            text += event.text
+        }
+        return text.trimmingCharacters(in: .whitespacesAndNewlines)
+    }
+
     /// The previous entry's tail when it was the same app and history and
     /// ended within the idle gap of this entry's first keystroke.
     private func precedingLines(for entry: OpenEntry) -> [String] {
@@ -267,8 +323,12 @@ struct WritingEntryComposer {
 
     private static func apply(_ event: PersonalHistoryEvent, to entry: inout OpenEntry) {
         switch event.source {
-        case .typed: entry.append(event.text, accepted: false)
-        case .acceptedSuggestion: entry.append(event.text, accepted: true)
+        case .typed:
+            entry.append(event.text, accepted: false)
+            entry.historyEvents.append(event)
+        case .acceptedSuggestion:
+            entry.append(event.text, accepted: true)
+            entry.historyEvents.append(event)
         case .deletion: entry.deleteLast(event.deletedCharacters ?? 0)
         }
     }

Sources/Writing/WritingController.swift

@@ -323,18 +323,19 @@ final class WritingController {
             port: TildeProductProfile.current.llamaServerPort,
             modelFileProvider: { models.manager.verifiedInstalledModelFile() }
         )
+        let personalHistoryController = PersonalHistoryController(
+            store: EncryptedPersonalHistoryStore(),
+            settings: settings,
+            diagnostics: .shared
+        )
         let runtime = Runtime(
             models: models,
             llamaServerHost: llamaServerHost,
             scaffoldPrewarmer: ScaffoldPrewarmer(
                 baseURL: llamaServerHost.baseURL,
                 accessKey: llamaServerHost.accessKey
             ),
-            personalHistoryController: PersonalHistoryController(
-                store: EncryptedPersonalHistoryStore(),
-                settings: settings,
-                diagnostics: .shared
-            ),
+            personalHistoryController: personalHistoryController,
             // Screen Memory serves Autocomplete only: with it off, nothing
             // on screen is read, even with Screen Recording granted.
             screenCaptureService: ScreenCaptureService(
@@ -346,9 +347,12 @@ final class WritingController {
                 },
                 excludedApps: { Self.settings().personalHistoryExcludedApps }
             ),
+            // Personal History takes only entries Save my writing cleared of
+            // secrets, so it's reached through the day files, not the socket.
             dayFiles: WritingDayFileWriter(
                 directory: writingDirectory,
                 preferences: { Self.preferences() },
+                personalHistory: personalHistoryController,
                 problemStarted: { [weak self] error in
                     self?.log("WRITING | save my writing: day file write failed (\(error))")
                 }
@@ -840,7 +844,6 @@ final class WritingController {
             runtime: runtime.llamaServerHost,
             personalHistory: WritingPausableIngest(
                 base: WritingHistoryIngest(
-                    personalHistory: runtime.personalHistoryController,
                     dayFiles: runtime.dayFiles.recorder,
                     appScope: { Self.preferences().appScope }
                 ),