From 5115caa030082dd5d7b2b96d94c496a344156b3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:34:50 +1000 Subject: [PATCH 01/19] fix(swift-ios): release accepted commands before optional refreshes --- apps/swift-ios/App/NativeFeatureClient.swift | 83 ++- .../NativeRetryIdentityTests.swift | 619 +++++++++++++++++- 2 files changed, 691 insertions(+), 11 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index a011101359bc..dd2a55d677bb 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -104,6 +104,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var pendingTurnSubmissions: [String: PendingTurnSubmission] = [:] private var projectSettingsWriteTask: Task? private var projectSettingsWriteGeneration: UInt64 = 0 + private struct AcceptedCommandRefresh: Sendable { + let id: UUID + let client: T3Client + let generation: Int + let task: Task + var pending = false + var needsDetail: Bool + } + private var acceptedCommandRefreshes: [String: AcceptedCommandRefresh] = [:] private var approvalRoutes: [String: PendingRequestRoute] = [:] private var inputRoutes: [String: PendingRequestRoute] = [:] private var relayDeviceSessionIDs: Set = [] @@ -223,6 +232,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detailCatchUpTask?.cancel() detailPublishTask?.cancel() projectFaviconRefreshTasks.values.forEach { $0.cancel() } + acceptedCommandRefreshes.values.forEach { $0.task.cancel() } continuation.finish() } @@ -470,6 +480,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func setEnvironmentEnabled(id: String, enabled: Bool) async throws { try await runtime.setEnabled(id: id, enabled: enabled) if !enabled { + cancelAcceptedCommandRefreshes(environmentID: id) environmentConnectionStates[id] = .disconnected environmentConnectionDetails[id] = nil environmentClients[id] = nil @@ -488,6 +499,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, if environment?.kind == .managedDPoP { try await runtime.revokeCredential(id: id) } + cancelAcceptedCommandRefreshes(environmentID: id) try await runtime.remove(id: id) saveGitHubRoutingGrants(gitHubRoutingGrants.filter { $0.environmentID != id }) if removesActiveEnvironment { @@ -1024,6 +1036,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func clearEnvironmentState(preserveEnvironmentSnapshots: Bool = false) { environmentGeneration &+= 1 + cancelAcceptedCommandRefreshes() resetDetailRefresh() resetDetailStream() archivedRefreshTask?.cancel() @@ -2012,6 +2025,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func loadThread(id: String, fresh: Bool) async throws -> FeatureThreadDetail { let route = try threadRoute(for: id) + if let previous = activeThreadID, previous != route.uiID { + cancelAcceptedCommandRefreshes(threadID: previous) + } let client = route.client let environment = client.environment let generation = environmentGeneration @@ -2154,6 +2170,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func releaseThread(id: String) { + cancelAcceptedCommandRefreshes(threadID: id) guard activeThreadID == id else { return } retainActiveThread() resetDetailRefresh() @@ -2273,11 +2290,62 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, if pendingTurnSubmissions[route.uiID]?.identity == pending.identity { pendingTurnSubmissions[route.uiID] = nil } - // Live sync reconciles these snapshots. Refreshes are opportunistic - // after the accepted command so transient reads cannot invite a - // duplicate user turn. - try? await refreshThread(id: route.uiID, client: client) - try? await refresh(client: client) + scheduleAcceptedCommandRefresh(threadID: route.uiID, client: client, generation: generation) + } + + /// Complete accepted commands without waiting for optional reads. A send + /// needs detail and shell recovery; Stop retains its shell-only refresh. + private func scheduleAcceptedCommandRefresh( + threadID: String, client: T3Client, generation: Int, refreshDetail: Bool = true + ) { + guard isKnownClient(client, environmentID: client.environment.id, generation: generation) else { + return + } + if let current = acceptedCommandRefreshes[threadID], + current.client === client, current.generation == generation { + acceptedCommandRefreshes[threadID]?.pending = true + acceptedCommandRefreshes[threadID]?.needsDetail = current.needsDetail || refreshDetail + return + } + cancelAcceptedCommandRefreshes(threadID: threadID) + let id = UUID() + let task = Task { [weak self] in + while self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true { + let needsDetail = self?.acceptedCommandRefreshes[threadID]?.needsDetail == true + self?.acceptedCommandRefreshes[threadID]?.needsDetail = false + self?.acceptedCommandRefreshes[threadID]?.pending = false + if needsDetail { + try? await self?.refreshThread(id: threadID, client: client) + } + guard self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true else { break } + try? await self?.refresh(client: client) + guard self?.acceptedCommandRefreshes[threadID]?.pending == true else { break } + } + if self?.acceptedCommandRefreshes[threadID]?.id == id { + self?.acceptedCommandRefreshes[threadID] = nil + } + } + acceptedCommandRefreshes[threadID] = AcceptedCommandRefresh( + id: id, client: client, generation: generation, task: task, needsDetail: refreshDetail + ) + } + + private func isCurrentAcceptedCommandRefresh(threadID: String, id: UUID) -> Bool { + guard !Task.isCancelled, let refresh = acceptedCommandRefreshes[threadID], refresh.id == id else { + return false + } + return isKnownClient( + refresh.client, environmentID: refresh.client.environment.id, generation: refresh.generation + ) + } + + private func cancelAcceptedCommandRefreshes(threadID: String? = nil, environmentID: String? = nil) { + for (key, refresh) in acceptedCommandRefreshes { + guard threadID.map({ $0 == key }) ?? true, + environmentID.map({ $0 == refresh.client.environment.id }) ?? true else { continue } + refresh.task.cancel() + acceptedCommandRefreshes[key] = nil + } } private func messageWasCommitted( @@ -2293,12 +2361,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func cancelTurn(threadID: String) async throws { let route = try threadRoute(for: threadID) + let generation = environmentGeneration let turnID = shellsByEnvironmentID[route.environmentID]?.threads .first(where: { $0.id == route.wireID })? .latestTurn? .turnId _ = try await route.client.interrupt(threadID: route.wireID, turnID: turnID) - try? await refresh(client: route.client) + scheduleAcceptedCommandRefresh( + threadID: route.uiID, client: route.client, generation: generation, refreshDetail: false + ) } func canRewindConversation(threadID: String, messageID: String) -> Bool { diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index 0ce3e6a1a4f7..268e07e5878a 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -1,4 +1,5 @@ import Foundation +import Observation import XCTest @testable import T3Code @@ -71,6 +72,203 @@ final class NativeRetryIdentityTests: XCTestCase { } } + + func testRejectedStopPropagatesRemoteError() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + await fixture.transport.holdFollowups() + let before = await fixture.transport.snapshotReadCount + await fixture.socket.rejectInterruptAcknowledgements() + do { + try await fixture.client.cancelTurn(threadID: fixture.threadID) + XCTFail("Rejected interrupt must remain a command error") + } catch let error as RPCError { + guard case .remote("Stop rejected by fixture") = error else { throw error } + } + // These counters cover the command-return boundary only; they are + // not a proof that no mistakenly queued future task could read later. + let after = await fixture.transport.snapshotReadCount + let pending = await fixture.transport.heldCount + XCTAssertEqual(after, before) + XCTAssertEqual(pending, 0) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["type"]?.stringValue, "thread.turn.interrupt") + } + + func testStopDuringHeldSendDetailPreservesDetailAndShellRecovery() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Send before Stop", selection: nil) + let firstRead = await reads.next() + let detail = try XCTUnwrap(firstRead) + XCTAssertTrue(detail.path.hasPrefix("/api/orchestration/threads/")) + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let count = await fixture.transport.heldCount + XCTAssertEqual(count, 1) + let detailStillPending = await fixture.transport.isPending(detail.id) + XCTAssertTrue(detailStillPending) + await fixture.transport.release(detail.id) + let nextRead = await reads.next() + let shell = try XCTUnwrap(nextRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + await fixture.transport.release(shell.id) + let trailingRead = await reads.next() + let trailing = try XCTUnwrap(trailingRead) + XCTAssertEqual(trailing.path, "/api/orchestration/shell", "Stop requests shell repair without discarding or repeating the send detail repair") + await fixture.client.disconnect() + let cancelled = await cancellations.next() + XCTAssertEqual(cancelled, trailing.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2) + } + + func testAcceptedStopCompletesBeforeOptionalShellReadFinishes() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let completed = expectation(description: "Accepted Stop completes without snapshot reads") + let stopping = Task { + await fixture.model.cancelTurn(threadID: fixture.threadID) + completed.fulfill() + } + let read = await reads.next() + XCTAssertEqual(read?.path, "/api/orchestration/shell") + await fulfillment(of: [completed], timeout: 2) + XCTAssertFalse(fixture.model.isPerformingAction) + + await fixture.transport.failFollowups() + await stopping.value + XCTAssertNil(fixture.model.errorMessage) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["type"]?.stringValue, "thread.turn.interrupt") + } + + func testSendDuringStopRefreshRetainsItsDetailRefresh() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let shellRead = await reads.next() + let shell = try XCTUnwrap(shellRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "After Stop", selection: nil) + let heldCount = await fixture.transport.heldCount + XCTAssertEqual(heldCount, 1) + await fixture.transport.release(shell.id) + let detailRead = await reads.next() + let detail = try XCTUnwrap(detailRead) + XCTAssertTrue(detail.path.hasPrefix("/api/orchestration/threads/")) + + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, detail.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2) + } + + func testDisconnectCancelsStopRefreshWithoutAnotherInterrupt() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let read = await reads.next() + let held = try XCTUnwrap(read) + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + let pending = await fixture.transport.heldCount + XCTAssertEqual(pending, 0) + } + + func testAcceptedSendCompletesOutboxBeforeOptionalReadsFinish() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let completed = expectation(description: "Accepted send completes without snapshot reads") + let sending = Task { + let sent = await fixture.model.sendMessage(threadID: fixture.threadID, text: "Accepted now", selection: nil) + completed.fulfill() + return sent + } + _ = await reads.next() + await fulfillment(of: [completed], timeout: 2) + let queued = try await fixture.outbox.submissions() + XCTAssertTrue(queued.isEmpty, "Acknowledgement must retire the outbox while optional reads are held.") + XCTAssertFalse(fixture.model.isPerformingAction) + + await fixture.transport.failFollowups() + let sent = await sending.value + XCTAssertTrue(sent, "Optional read failures cannot turn acceptance into a failed send.") + XCTAssertNil(fixture.model.errorMessage) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(fixture.model.details[fixture.threadID]?.messages.last?.state, .complete) + await fixture.client.disconnect() + } + + func testAcceptedSendRefreshesCoalesceAndCancelWhenThreadCloses() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "First", selection: nil) + let firstRead = await reads.next() + let first = try XCTUnwrap(firstRead) + XCTAssertTrue(first.path.hasPrefix("/api/orchestration/threads/")) + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Second", selection: nil) + let heldCount = await fixture.transport.heldCount + XCTAssertEqual(heldCount, 1, "A second acceptance must coalesce behind the current read.") + + await fixture.transport.release(first.id) + let shellRead = await reads.next() + let shell = try XCTUnwrap(shellRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + await fixture.transport.release(shell.id) + let trailingRead = await reads.next() + let trailing = try XCTUnwrap(trailingRead) + XCTAssertTrue(trailing.path.hasPrefix("/api/orchestration/threads/")) + fixture.client.releaseThread(id: fixture.threadID) + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, trailing.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2, "Read reconciliation must never redispatch a command.") + await fixture.transport.failFollowups() + await fixture.client.disconnect() + } + + func testDisconnectCancelsAcceptedSendRefreshWithoutAnotherCommand() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Accepted before disconnect", selection: nil) + let heldRead = await reads.next() + let held = try XCTUnwrap(heldRead) + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + let pending = await fixture.transport.heldCount + XCTAssertEqual(pending, 0) + } + func testSavedSettingsSurviveAConnectionRepublish() async throws { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("t3-native-settings-republish-\(UUID().uuidString)") @@ -100,14 +298,13 @@ final class NativeRetryIdentityTests: XCTestCase { let settingsStore = UserDefaults(suiteName: settingsSuite)! defer { settingsStore.removePersistentDomain(forName: settingsSuite) } let client = NativeFeatureClient(runtime: runtime, settingsStore: settingsStore) + var events = client.events().makeAsyncIterator() let initial = try await client.initialSnapshot() await connection.waitUntilConnected() var updated = initial.settings updated.textSize = FeatureTextSizeAdjustment(steps: 2) updated.codeSize = FeatureTextSizeAdjustment(steps: -1) try await client.saveSettings(updated) - var events = client.events().makeAsyncIterator() - await connection.failReceive() var receivedReconnect = false @@ -160,7 +357,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-concurrent-retry-\(UUID().uuidString)" )! ) - _ = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() await transport.rejectShellReads() @@ -226,7 +426,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-retry-\(UUID().uuidString)" )! let client = NativeFeatureClient(runtime: runtime, settingsStore: settings) - let initial = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + let initial = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } XCTAssertEqual(initial.threads.first?.runtimeMode, .approvalRequired) XCTAssertEqual(initial.threads.first?.interactionMode, .standard) await connection.waitUntilConnected() @@ -341,7 +544,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-partial-\(UUID().uuidString)" )! let client = NativeFeatureClient(runtime: runtime, settingsStore: settings) - _ = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() let identity = FeatureSubmissionIdentity( @@ -923,3 +1129,406 @@ private func retryDispatchCommand(from request: URLRequest) throws -> JSONValue } return try JSONDecoder.t3.decode(JSONValue.self, from: body) } + +@MainActor +private struct AcceptedSendFixture { + let modelTask: Task + let directory: URL + let threadID: String + let settingsName: String + let client: NativeFeatureClient + let model: FeatureRootModel + let outbox: FeatureOutboxStore + let transport: AcceptedSendHTTPTransport + let socket: AcceptedSendSocket + let runtime: EnvironmentRuntime + + static func make(includePeer: Bool = false) async throws -> Self { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("t3-accepted-send-\(UUID().uuidString)") + let environment = Environment( + id: "accepted-send", label: "Accepted send", + httpBaseURL: URL(string: "https://accepted-send.example")!, + webSocketBaseURL: URL(string: "wss://accepted-send.example")! + ) + let store = EnvironmentStore(fileURL: directory.appendingPathComponent("environments.json")) + let peer = Environment( + id: "accepted-peer", label: "Accepted peer", + httpBaseURL: URL(string: "https://accepted-peer.example")!, + webSocketBaseURL: URL(string: "wss://accepted-peer.example")! + ) + try await store.save(includePeer ? [environment, peer] : [environment]) + try await store.setActiveEnvironment(id: environment.id) + let transport = AcceptedSendHTTPTransport() + let socket = AcceptedSendSocket() + let runtime = EnvironmentRuntime( + environmentStore: store, + credentialStore: InMemoryCredentialStore(credentials: [ + environment.id: EnvironmentCredential(accessToken: "fixture-token"), + peer.id: EnvironmentCredential(accessToken: "fixture-peer-token"), + ]), + httpTransport: transport, + webSocketConnector: AcceptedSendConnector(socket: socket, peer: AcceptedSendSocket()) + ) + let settingsName = "t3-accepted-send-\(UUID().uuidString)" + let client = NativeFeatureClient( + runtime: runtime, settingsStore: UserDefaults(suiteName: settingsName)!, + fallbackPollingInitialDelay: .seconds(3600), aggregateRefreshInterval: .seconds(3600), + aggregateIdleRefreshInterval: .seconds(3600) + ) + let outbox = FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")) + let model = FeatureRootModel(client: client, outboxStore: outbox) + let modelTask = Task { await model.start() } + do { + try await AcceptedSendRootReadiness(model: model).wait() + let threadID = FeatureScopedID.thread(environmentID: environment.id, wireID: "thread-existing") + _ = await model.detail(for: threadID) + return Self(modelTask: modelTask, directory: directory, threadID: threadID, + settingsName: settingsName, client: client, model: model, + outbox: outbox, transport: transport, socket: socket, runtime: runtime) + } catch { + modelTask.cancel() + await client.disconnect() + await modelTask.value + UserDefaults.standard.removePersistentDomain(forName: settingsName) + try? FileManager.default.removeItem(at: directory) + throw error + } + } + + func cleanUp() async { + modelTask.cancel() + await client.disconnect() + await modelTask.value + UserDefaults.standard.removePersistentDomain(forName: settingsName) + try? FileManager.default.removeItem(at: directory) + } +} + +@MainActor +private final class AcceptedSendRootReadiness { + private let model: FeatureRootModel + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel) { self.model = model } + + func wait() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor in self.finish(CancellationError()) } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + !model.isLoading + && model.snapshot.projects.contains { $0.id == FeatureScopedID.project(environmentID: "accepted-send", wireID: "project-1") } + && model.snapshot.threads.contains { $0.id == FeatureScopedID.thread(environmentID: "accepted-send", wireID: "thread-existing") } + && model.snapshot.environments.first(where: { $0.id == "accepted-send" })?.connectionState == .connected + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { finish(nil) } + } + + private func finish(_ error: Error?) { + guard let pending = continuation else { return } + continuation = nil + if let error { pending.resume(throwing: error) } else { pending.resume() } + } +} + +@MainActor +private final class AcceptedSendDetailReceipt { + private let model: FeatureRootModel + private let threadID: String + private let expectedMessage: String + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel, threadID: String, expectedMessage: String) { + self.model = model + self.threadID = threadID + self.expectedMessage = expectedMessage + } + + func wait() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor in self.finish(CancellationError()) } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + model.details[threadID]?.messages.last?.text == expectedMessage + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { finish(nil) } + } + + private func finish(_ error: Error?) { + guard let pending = continuation else { return } + continuation = nil + if let error { pending.resume(throwing: error) } else { pending.resume() } + } +} + +private actor AcceptedSendHTTPTransport: HTTPTransport { + struct HeldRead: Sendable { + let id: UUID + let path: String + } + nonisolated let heldReads: AsyncStream + nonisolated let cancellations: AsyncStream + private let reads: AsyncStream.Continuation + private let cancelled: AsyncStream.Continuation + private(set) var snapshotReadCount = 0 + private var holdsShell = true + private var holds = false + private var fails = false + private var pending: [UUID: CheckedContinuation] = [:] + private var detailMessages: [UUID: String] = [:] + + init() { + (heldReads, reads) = AsyncStream.makeStream() + (cancellations, cancelled) = AsyncStream.makeStream() + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let path = request.url?.path ?? "" + if path == "/api/auth/websocket-ticket" { + return (Data(#"{"ticket":"fixture-ticket","expiresAt":"2026-09-07T12:05:00Z"}"#.utf8), retryHTTPResponse(request)) + } + guard path == "/api/orchestration/shell" || path.hasPrefix("/api/orchestration/threads/") else { + throw URLError(.unsupportedURL) + } + snapshotReadCount += 1 + if fails { throw URLError(.networkConnectionLost) } + var heldID: UUID? + if holds && (holdsShell || path != "/api/orchestration/shell") { + let id = UUID() + heldID = id + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + pending[id] = continuation + reads.yield(HeldRead(id: id, path: path)) + } + } onCancel: { + Task { await self.cancel(id) } + } + } + let data: Data + if path == "/api/orchestration/shell" { + data = try JSONEncoder.t3.encode(retryShellSnapshot()) + } else { + let snapshot = retryEmptyThreadDetail(id: request.url!.lastPathComponent) + if let heldID, let text = detailMessages.removeValue(forKey: heldID) { + var thread = try JSONValue.encode(snapshot.thread).decode([String: JSONValue].self) + thread["messages"] = .array([.object([ + "id": .string("replacement-generation-message"), "role": .string("assistant"), + "text": .string(text), "turnId": .null, "streaming": .bool(false), + "attachments": .array([]), "createdAt": .string("2026-07-30T12:00:00.000Z"), + "updatedAt": .string("2026-07-30T12:00:00.000Z"), + ])]) + data = try JSONEncoder.t3.encode(OrchestrationThreadDetailSnapshot( + snapshotSequence: 99, + thread: try JSONValue.object(thread).decode(OrchestrationThread.self) + )) + } else { data = try JSONEncoder.t3.encode(snapshot) } + } + return (data, retryHTTPResponse(request)) + } + + var heldCount: Int { pending.count } + + func isPending(_ id: UUID) -> Bool { pending[id] != nil } + + func holdFollowups(includeShell: Bool = true) { holds = true; holdsShell = includeShell } + + func release(_ id: UUID, detailMessage: String? = nil) { + guard let continuation = pending.removeValue(forKey: id) else { return } + if let detailMessage { detailMessages[id] = detailMessage } + continuation.resume() + } + + func failFollowups() { + fails = true + holds = false + let waiting = pending.values + pending.removeAll() + waiting.forEach { $0.resume(throwing: URLError(.networkConnectionLost)) } + } + + private func cancel(_ id: UUID) { + guard let continuation = pending.removeValue(forKey: id) else { return } + continuation.resume(throwing: CancellationError()) + cancelled.yield(id) + } +} + +private struct AcceptedSendConnector: WebSocketConnecting { + let socket: AcceptedSendSocket + let peer: AcceptedSendSocket + func connect(to url: URL) -> any WebSocketConnection { + url.host == "accepted-peer.example" ? peer : socket + } +} + +private actor AcceptedSendSocket: WebSocketConnection { + nonisolated let heldInterrupts: AsyncStream + private let interruptReceipts: AsyncStream.Continuation + private var holdsInterrupts = false + private var rejectsInterrupts = false + private var pendingInterruptResponses: [Data] = [] + + init() { (heldInterrupts, interruptReceipts) = AsyncStream.makeStream() } + func holdInterruptAcknowledgements() { holdsInterrupts = true } + func rejectInterruptAcknowledgements() { rejectsInterrupts = true } + func releaseInterruptAcknowledgements() { + holdsInterrupts = false + let responses = pendingInterruptResponses + pendingInterruptResponses.removeAll() + responses.forEach { enqueue($0) } + } + private(set) var commands: [JSONValue] = [] + private var queued: [Data] = [] + private var receiver: CheckedContinuation? + + func send(_ data: Data) throws { + let request = try JSONDecoder.t3.decode(JSONValue.self, from: data) + let tag = request["tag"]?.stringValue + if tag == RPCMethod.serverGetConfig.rawValue || tag == RPCMethod.subscribeServerConfig.rawValue, + let response = try retryConfigResponse(for: request) { + enqueue(response) + } else if tag == RPCMethod.dispatchCommand.rawValue, let payload = request["payload"] { + commands.append(payload) + let isInterrupt = payload["type"]?.stringValue == "thread.turn.interrupt" + let exit: JSONValue = isInterrupt && rejectsInterrupts + ? .object(["_tag": .string("Failure"), "cause": .array([ + .object(["_tag": .string("Fail"), "error": .object(["message": .string("Stop rejected by fixture")])]), + ])]) + : .object(["_tag": .string("Success"), "value": .object(["sequence": .number(2)])]) + let response = try JSONEncoder.t3.encode(JSONValue.object([ + "_tag": .string("Exit"), "requestId": request["id"]!, "exit": exit, + ])) + if isInterrupt && holdsInterrupts { + pendingInterruptResponses.append(response) + interruptReceipts.yield(()) + } else { enqueue(response) } + } + } + + func receive() async throws -> Data { + if !queued.isEmpty { return queued.removeFirst() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + receiver?.resume(throwing: CancellationError()) + receiver = nil + } + + private func enqueue(_ data: Data) { + if let receiver { self.receiver = nil; receiver.resume(returning: data) } + else { queued.append(data) } + } +} + +@MainActor +private final class AcceptedCommandSnapshotRecorder { + private(set) var history: [FeatureSnapshot] + private var task: Task? + private var waiters: [UUID: (@MainActor (FeatureSnapshot) -> Bool, CheckedContinuation)] = [:] + private var finished = false + + init(seed: FeatureSnapshot, events: AsyncStream) { + history = [seed] + task = Task { [weak self] in + for await event in events { + guard !Task.isCancelled else { break } + guard let self else { break } + switch event { + case let .snapshot(snapshot): self.record(snapshot) + case let .thread(thread): + guard var snapshot = self.history.last else { continue } + if let index = snapshot.threads.firstIndex(where: { $0.id == thread.id }) { + snapshot.threads[index] = thread + } else { snapshot.threads.append(thread) } + self.record(snapshot) + case let .threadRemoved(id): + guard var snapshot = self.history.last else { continue } + snapshot.threads.removeAll { $0.id == id } + self.record(snapshot) + default: break + } + } + self?.stop() + } + } + + func record(_ snapshot: FeatureSnapshot) { + history.append(snapshot) + let ready = waiters.filter { $0.value.0(snapshot) } + for (id, waiter) in ready { + waiters[id] = nil + waiter.1.resume(returning: snapshot) + } + } + + func wait(_ predicate: @escaping @MainActor (FeatureSnapshot) -> Bool) async throws -> FeatureSnapshot { + try Task.checkCancellation() + if let snapshot = history.last, predicate(snapshot) { return snapshot } + guard !finished else { throw CancellationError() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (predicate, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + } + } + + func stop() { + finished = true + task?.cancel() + task = nil + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.1.resume(throwing: CancellationError()) } + } + + deinit { task?.cancel() } +} + +private extension NativeFeatureClient { + /// Sends plain text through the full submission API with fixture defaults. + func sendMessage(threadID: String, text: String, selection: FeatureSelection?) async throws { + try await sendMessage( + threadID: threadID, text: text, selection: selection, runtimeMode: .fullAccess, + attachments: [], identity: FeatureSubmissionIdentity(threadID: threadID) + ) + } +} + +private extension FeatureRootModel { + func sendMessage(threadID: String, text: String, selection: FeatureSelection?) async -> Bool { + await sendMessage(FeatureMessageSubmission(threadID: threadID, text: text, selection: selection)) + } +} From 46657718b5fbe1e26436f0139c72a3d566c3f292 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:31:39 +1000 Subject: [PATCH 02/19] fix(swift-ios): cancel accepted-command refresh when deleting a thread A detached accepted-command refresh could finish after deleteThread and publish the deleted thread again. Cancel it once the server accepts the delete. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 2 ++ .../FeatureTests/NativeRetryIdentityTests.swift | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index dd2a55d677bb..7286588ab9ac 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -1997,6 +1997,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func deleteThread(id: String) async throws { let route = try threadRoute(for: id) _ = try await route.client.delete(threadID: route.wireID) + // A pending accepted-command refresh would publish the deleted thread again. + cancelAcceptedCommandRefreshes(threadID: route.uiID) archivedThreadsByEnvironmentID[route.environmentID]?.removeAll { $0.id == route.uiID } diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index 268e07e5878a..289e872db59b 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -251,6 +251,23 @@ final class NativeRetryIdentityTests: XCTestCase { await fixture.client.disconnect() } + func testDeletingThreadCancelsItsAcceptedSendRefresh() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Accepted before delete", selection: nil) + let heldRead = await reads.next() + let held = try XCTUnwrap(heldRead) + let deletion = Task { try await fixture.client.deleteThread(id: fixture.threadID) } + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id, "A deleted thread's pending refresh must not publish it again.") + await fixture.transport.failFollowups() + _ = try? await deletion.value + await fixture.client.disconnect() + } + func testDisconnectCancelsAcceptedSendRefreshWithoutAnotherCommand() async throws { let fixture = try await AcceptedSendFixture.make() addTeardownBlock { await fixture.cleanUp() } From 462ff3d9a3fd79923bb620b31fd5b9d7f0c8185c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:36:01 +1000 Subject: [PATCH 03/19] fix(swift-ios): keep superseded accepted-send refreshes from publishing older state A detail read that started before a newer send was accepted could publish after it and hide the newer message until the next read. Skip publishing a superseded read (its refresh loop reads again), and stop a cancelled shell refresh from writing a shell read before the change. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 14 ++++-- .../NativeRetryIdentityTests.swift | 49 +++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index 7286588ab9ac..c79b52e71e52 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -2317,7 +2317,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self?.acceptedCommandRefreshes[threadID]?.needsDetail = false self?.acceptedCommandRefreshes[threadID]?.pending = false if needsDetail { - try? await self?.refreshThread(id: threadID, client: client) + try? await self?.refreshThread(id: threadID, client: client) { [weak self] in + self?.acceptedCommandRefreshes[threadID]?.pending == true + } } guard self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true else { break } try? await self?.refresh(client: client) @@ -5119,7 +5121,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let environment = client.environment let generation = environmentGeneration let shell = try await client.shellSnapshot() - guard isKnownClient(client, environmentID: environment.id, generation: generation) else { + // A cancelled refresh (for example, of a thread deleted meanwhile) must not + // write a shell that was read before the change. + guard !Task.isCancelled, + isKnownClient(client, environmentID: environment.id, generation: generation) else { throw CancellationError() } guard shell.snapshotSequence @@ -5176,7 +5181,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func refreshThread( - id: String, client: T3Client, expectedStreamGeneration: Int? = nil + id: String, client: T3Client, expectedStreamGeneration: Int? = nil, + isSuperseded: (() -> Bool)? = nil ) async throws { let route = try threadRoute(for: id) guard route.client === client else { @@ -5198,6 +5204,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { throw CancellationError() } + // A newer accepted command makes this snapshot stale; its owner reads again. + if isSuperseded?() == true { return } if activeThreadID == route.uiID { if activeRawThread == nil, historyEpoch != threadHistoryEpoch { return diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index 289e872db59b..71c2e9e0d785 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -251,6 +251,33 @@ final class NativeRetryIdentityTests: XCTestCase { await fixture.client.disconnect() } + func testSupersededAcceptedSendRefreshDoesNotPublishItsOlderDetail() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups(includeShell: false) + let history = AcceptedSendDetailHistory(model: fixture.model, threadID: fixture.threadID) + history.start() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "First", selection: nil) + let firstRead = await reads.next() + let first = try XCTUnwrap(firstRead) + XCTAssertTrue(first.path.hasPrefix("/api/orchestration/threads/")) + // The second acceptance lands while the first read is still in flight. + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Second", selection: nil) + let stale = "Read started before the second send was accepted" + await fixture.transport.release(first.id, detailMessage: stale) + let followUpRead = await reads.next() + let followUp = try XCTUnwrap(followUpRead) + XCTAssertTrue(followUp.path.hasPrefix("/api/orchestration/threads/")) + let fresh = "Read after the second send was accepted" + await fixture.transport.release(followUp.id, detailMessage: fresh) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: fresh + ).wait() + XCTAssertFalse(history.seen.contains(stale), "A superseded read must not replace newer accepted state.") + await fixture.client.disconnect() + } + func testDeletingThreadCancelsItsAcceptedSendRefresh() async throws { let fixture = try await AcceptedSendFixture.make() addTeardownBlock { await fixture.cleanUp() } @@ -1261,6 +1288,28 @@ private final class AcceptedSendRootReadiness { } } +/// Records every last-message text the model shows for a thread. +@MainActor +private final class AcceptedSendDetailHistory { + private let model: FeatureRootModel + private let threadID: String + private(set) var seen: [String] = [] + + init(model: FeatureRootModel, threadID: String) { + self.model = model + self.threadID = threadID + } + + func start() { + let text = withObservationTracking { + model.details[threadID]?.messages.last?.text + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.start() } + } + if let text, seen.last != text { seen.append(text) } + } +} + @MainActor private final class AcceptedSendDetailReceipt { private let model: FeatureRootModel From c4e63af2e20cf081b98912bff40b61f8110d5131 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 19:07:56 +1000 Subject: [PATCH 04/19] fix(swift-ios): keep accepted messages visible until their detail arrives A Stop accepted during a send's detail read discarded that read without requesting another, and an older detail could replace the transcript before the accepted message reached it. Re-read detail after a discarded read, and keep delivered messages until a server transcript includes them. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 5 ++- .../Features/Root/FeatureRootModel.swift | 22 +++++++++- .../FeatureTests/FeatureRootModelTests.swift | 42 +++++++++++++++++++ .../NativeRetryIdentityTests.swift | 15 ++++++- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index c79b52e71e52..008893653031 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -2318,7 +2318,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self?.acceptedCommandRefreshes[threadID]?.pending = false if needsDetail { try? await self?.refreshThread(id: threadID, client: client) { [weak self] in - self?.acceptedCommandRefreshes[threadID]?.pending == true + guard self?.acceptedCommandRefreshes[threadID]?.pending == true else { return false } + // The discarded detail must be read again, even for a shell-only Stop. + self?.acceptedCommandRefreshes[threadID]?.needsDetail = true + return true } } guard self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true else { break } diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index cb91af596570..ce6322ccf5c6 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -90,6 +90,8 @@ public final class FeatureRootModel { draftStore: draftStore ) private var pendingSubmissionsByID: [String: FeatureQueuedSubmission] = [:] + /// Accepted messages not yet present in a server transcript, by message ID. + private var deliveredAwaitingDetail: [String: FeatureQueuedSubmission] = [:] private var activeSubmissionCounts: [String: Int] = [:] private var pendingThreadsByID: [String: FeatureThread] = [:] private var pendingSettlementMutations: [String: PendingSettlementMutation] = [:] @@ -1317,6 +1319,7 @@ public final class FeatureRootModel { } private func removeThread(id: String) { + deliveredAwaitingDetail = deliveredAwaitingDetail.filter { $0.value.threadID != id } guard let index = snapshot.threads.firstIndex(where: { $0.id == id }) else { return } let projectID = snapshot.threads[index].projectID snapshot.threads.remove(at: index) @@ -1738,12 +1741,26 @@ public final class FeatureRootModel { } private func addingPendingMessages(to incoming: FeatureThreadDetail) -> FeatureThreadDetail { + let existing = Set(incoming.messages.map(\.id)) + // A delivered message stays visible until a server transcript includes it. + // Its refresh no longer blocks the send, so an older read can arrive first. + for (id, submission) in deliveredAwaitingDetail + where submission.threadID == incoming.thread.id && existing.contains(id) { + deliveredAwaitingDetail.removeValue(forKey: id) + } let queued = pendingSubmissionsByID.values .filter { $0.threadID == incoming.thread.id } .sorted { $0.identity.createdAt < $1.identity.createdAt } - guard !queued.isEmpty else { return incoming } + let delivered = deliveredAwaitingDetail.values + .filter { $0.threadID == incoming.thread.id } + .sorted { $0.identity.createdAt < $1.identity.createdAt } + guard !queued.isEmpty || !delivered.isEmpty else { return incoming } var result = incoming - let existing = Set(result.messages.map(\.id)) + result.messages.append(contentsOf: delivered.lazy.map { submission in + var message = self.queuedMessage(for: submission) + message.state = .complete + return message + }) result.messages.append(contentsOf: queued.lazy .filter { !existing.contains($0.identity.messageID) } .map(queuedMessage(for:))) @@ -1823,6 +1840,7 @@ public final class FeatureRootModel { } pendingCompletionSubmissionIDs.remove(submission.id) pendingSubmissionsByID.removeValue(forKey: submission.id) + deliveredAwaitingDetail[submission.identity.messageID] = submission setAttachmentOutboxOwnership(false, for: submission) pendingThreadsByID.removeValue(forKey: submission.threadID) markQueuedMessageDelivered(submission) diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 241354476b03..70faa5a40774 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -1683,6 +1683,48 @@ struct FeatureRootModelTests { #expect(model.details[thread.id]?.messages.last?.state == .complete) } + @Test + func deliveredMessageSurvivesAnOlderDetailUntilTheServerIncludesIt() async throws { + let client = FeatureClientStub() + let thread = FeatureThread( + id: "thread-1", + projectID: "project-1", + environmentID: "environment-1", + title: "Thread" + ) + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", + name: "Studio", + endpoint: "https://studio.example", + isActive: true, + connectionState: .connected + ), + ], + threads: [thread] + ) + client.threadDetail = FeatureThreadDetail(thread: thread) + let model = testRootModel(client: client) + await model.reload() + _ = await model.detail(for: thread.id) + + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + // Its detail refresh no longer blocks the send, so an older read can land first. + _ = await model.detail(for: thread.id, force: true) + let delivered = try #require(model.details[thread.id]?.messages.last) + #expect(delivered.text == "ship it") + #expect(delivered.state == .complete) + + client.threadDetail = FeatureThreadDetail( + thread: thread, + messages: [FeatureMessage(id: delivered.id, role: .user, text: "ship it")] + ) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.filter { $0.id == delivered.id }.count == 1) + } + @Test func sendPreservesTheThreadAutomaticPermission() async { let client = FeatureClientStub() diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index 71c2e9e0d785..c3a5a4f146bc 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -118,10 +118,21 @@ final class NativeRetryIdentityTests: XCTestCase { await fixture.transport.release(shell.id) let trailingRead = await reads.next() let trailing = try XCTUnwrap(trailingRead) - XCTAssertEqual(trailing.path, "/api/orchestration/shell", "Stop requests shell repair without discarding or repeating the send detail repair") + XCTAssertTrue( + trailing.path.hasPrefix("/api/orchestration/threads/"), + "Stop superseded the in-flight send detail, so the send's repair reads detail again" + ) + let marker = "Send detail recovered after Stop" + await fixture.transport.release(trailing.id, detailMessage: marker) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: marker + ).wait() + let finalRead = await reads.next() + let final = try XCTUnwrap(finalRead) + XCTAssertEqual(final.path, "/api/orchestration/shell") await fixture.client.disconnect() let cancelled = await cancellations.next() - XCTAssertEqual(cancelled, trailing.id) + XCTAssertEqual(cancelled, final.id) let commands = await fixture.socket.commands XCTAssertEqual(commands.count, 2) } From 302ea89c8e51b689b2bc8d765568169ed26054db Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:16:06 +1000 Subject: [PATCH 05/19] fix(swift-ios): release retained messages once confirmed or cleared Do not retain a delivered message that a server transcript already confirmed, keep only its display copy, and drop retained messages when their thread details are removed or cleared. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Features/Root/FeatureRootModel.swift | 35 +++++++++----- .../FeatureTests/FeatureRootModelTests.swift | 47 +++++++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index ce6322ccf5c6..4580fd44f5fd 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -91,7 +91,9 @@ public final class FeatureRootModel { ) private var pendingSubmissionsByID: [String: FeatureQueuedSubmission] = [:] /// Accepted messages not yet present in a server transcript, by message ID. - private var deliveredAwaitingDetail: [String: FeatureQueuedSubmission] = [:] + private var deliveredAwaitingDetail: [String: (threadID: String, message: FeatureMessage)] = [:] + /// Message IDs in each thread's latest server transcript. + private var serverMessageIDs: [String: Set] = [:] private var activeSubmissionCounts: [String: Int] = [:] private var pendingThreadsByID: [String: FeatureThread] = [:] private var pendingSettlementMutations: [String: PendingSettlementMutation] = [:] @@ -1319,7 +1321,7 @@ public final class FeatureRootModel { } private func removeThread(id: String) { - deliveredAwaitingDetail = deliveredAwaitingDetail.filter { $0.value.threadID != id } + forgetDeliveredMessages(threadID: id) guard let index = snapshot.threads.firstIndex(where: { $0.id == id }) else { return } let projectID = snapshot.threads[index].projectID snapshot.threads.remove(at: index) @@ -1526,7 +1528,13 @@ public final class FeatureRootModel { return true } + private func forgetDeliveredMessages(threadID: String) { + deliveredAwaitingDetail = deliveredAwaitingDetail.filter { $0.value.threadID != threadID } + serverMessageIDs[threadID] = nil + } + private func removeDetail(id: String) { + forgetDeliveredMessages(threadID: id) if details.removeValue(forKey: id) != nil { detailRecency.removeAll { $0 == id } } @@ -1539,6 +1547,8 @@ public final class FeatureRootModel { } private func clearDetails() { + deliveredAwaitingDetail.removeAll() + serverMessageIDs.removeAll() detailLoadGeneration &+= 1 detailLoadRevisions.removeAll() storedDetailLoadRequestRevisions.removeAll() @@ -1742,10 +1752,11 @@ public final class FeatureRootModel { private func addingPendingMessages(to incoming: FeatureThreadDetail) -> FeatureThreadDetail { let existing = Set(incoming.messages.map(\.id)) + serverMessageIDs[incoming.thread.id] = existing // A delivered message stays visible until a server transcript includes it. // Its refresh no longer blocks the send, so an older read can arrive first. - for (id, submission) in deliveredAwaitingDetail - where submission.threadID == incoming.thread.id && existing.contains(id) { + for (id, delivered) in deliveredAwaitingDetail + where delivered.threadID == incoming.thread.id && existing.contains(id) { deliveredAwaitingDetail.removeValue(forKey: id) } let queued = pendingSubmissionsByID.values @@ -1753,14 +1764,11 @@ public final class FeatureRootModel { .sorted { $0.identity.createdAt < $1.identity.createdAt } let delivered = deliveredAwaitingDetail.values .filter { $0.threadID == incoming.thread.id } - .sorted { $0.identity.createdAt < $1.identity.createdAt } + .map(\.message) + .sorted { $0.createdAt < $1.createdAt } guard !queued.isEmpty || !delivered.isEmpty else { return incoming } var result = incoming - result.messages.append(contentsOf: delivered.lazy.map { submission in - var message = self.queuedMessage(for: submission) - message.state = .complete - return message - }) + result.messages.append(contentsOf: delivered) result.messages.append(contentsOf: queued.lazy .filter { !existing.contains($0.identity.messageID) } .map(queuedMessage(for:))) @@ -1840,7 +1848,12 @@ public final class FeatureRootModel { } pendingCompletionSubmissionIDs.remove(submission.id) pendingSubmissionsByID.removeValue(forKey: submission.id) - deliveredAwaitingDetail[submission.identity.messageID] = submission + // A transcript that already includes it confirmed delivery; do not retain it. + if serverMessageIDs[submission.threadID]?.contains(submission.identity.messageID) != true { + var message = queuedMessage(for: submission) + message.state = .complete + deliveredAwaitingDetail[submission.identity.messageID] = (submission.threadID, message) + } setAttachmentOutboxOwnership(false, for: submission) pendingThreadsByID.removeValue(forKey: submission.threadID) markQueuedMessageDelivered(submission) diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 70faa5a40774..30fc011a17bc 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -1725,6 +1725,53 @@ struct FeatureRootModelTests { #expect(model.details[thread.id]?.messages.filter { $0.id == delivered.id }.count == 1) } + private func deliveredMessageModel() async -> (FeatureClientStub, FeatureRootModel, FeatureThread) { + let client = FeatureClientStub() + let thread = FeatureThread( + id: "thread-1", projectID: "project-1", environmentID: "environment-1", title: "Thread" + ) + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", name: "Studio", endpoint: "https://studio.example", + isActive: true, connectionState: .connected + ), + ], + threads: [thread] + ) + client.threadDetail = FeatureThreadDetail(thread: thread) + let model = testRootModel(client: client) + await model.reload() + _ = await model.detail(for: thread.id) + return (client, model, thread) + } + + @Test + func confirmedDeliveredMessageDoesNotReturnAfterARewind() async throws { + let (client, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + client.threadDetail = FeatureThreadDetail( + thread: thread, messages: [FeatureMessage(id: id, role: .user, text: "ship it")] + ) + _ = await model.detail(for: thread.id, force: true) + // A rewind removes the message from the server transcript. + client.threadDetail = FeatureThreadDetail(thread: thread) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) + } + + @Test + func clearedDetailsDropRetainedDeliveredMessages() async throws { + let (_, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + await model.reloadAfterConnection() + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) + } + @Test func sendPreservesTheThreadAutomaticPermission() async { let client = FeatureClientStub() From b8228675df9308a3a458497aee9c53c5e152bbf8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 06:59:04 +1000 Subject: [PATCH 06/19] fix(swift-ios): keep accepted messages across eviction and local details Evicting a cached thread detail forgot its accepted messages, so a stale reopen hid them. A restored outbox or a new task's local detail was also recorded as the server transcript, so its accepted prompt was treated as confirmed and disappeared on the next older read. Only server transcripts now confirm delivery, and eviction keeps retained messages. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Features/Root/FeatureRootModel.swift | 44 +++++++++++------ .../FeatureTests/FeatureRootModelTests.swift | 49 +++++++++++++++++++ 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index 4580fd44f5fd..567d59e88467 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -1454,15 +1454,18 @@ public final class FeatureRootModel { } } + /// `isServerTranscript` is false for details built locally, such as a + /// restored outbox or a pending creation; they cannot confirm delivery. private func store( _ incoming: FeatureThreadDetail, - invalidatesInFlightLoad: Bool = true + invalidatesInFlightLoad: Bool = true, + isServerTranscript: Bool = true ) { var incoming = retainingLocalAttachmentPreviews(in: incoming) incoming.thread = retainingPendingSettlement(in: incoming.thread) let id = incoming.thread.id acknowledgeDeliveredMessages(incoming.messages) - let prepared = addingPendingMessages(to: incoming) + let prepared = addingPendingMessages(to: incoming, isServerTranscript: isServerTranscript) let next = details[id].map { current in FeatureThreadDetail( thread: prepared.thread, @@ -1489,7 +1492,7 @@ public final class FeatureRootModel { incoming.thread = retainingPendingSettlement(in: incoming.thread) let id = incoming.thread.id acknowledgeDeliveredMessages(incoming.messages) - let next = addingPendingMessages(to: incoming) + let next = addingPendingMessages(to: incoming, isServerTranscript: true) details[id] = next markDetailRecentlyUsed(id) bumpDetailLoadRevision(id: id) @@ -1533,8 +1536,13 @@ public final class FeatureRootModel { serverMessageIDs[threadID] = nil } - private func removeDetail(id: String) { - forgetDeliveredMessages(threadID: id) + private func removeDetail(id: String, forgettingDeliveredMessages: Bool = true) { + if forgettingDeliveredMessages { + forgetDeliveredMessages(threadID: id) + } else { + // The next server transcript records these again; keep the cache bounded. + serverMessageIDs[id] = nil + } if details.removeValue(forKey: id) != nil { detailRecency.removeAll { $0 == id } } @@ -1584,7 +1592,8 @@ public final class FeatureRootModel { while details.count > Self.maximumRetainedThreadDetails, let candidate = detailRecency.first(where: { !protected.contains($0) }) { detailRecency.removeAll { $0 == candidate } - removeDetail(id: candidate) + // Eviction only drops the cache; accepted messages must survive a reopen. + removeDetail(id: candidate, forgettingDeliveredMessages: false) } } @@ -1627,7 +1636,7 @@ public final class FeatureRootModel { if snapshot.threads.contains(where: { $0.id == submission.threadID }) { pendingSubmissionsByID[submission.id] = submission if let detail = details[submission.threadID] { - store(addingPendingMessages(to: detail)) + store(detail, isServerTranscript: false) } continue } @@ -1658,7 +1667,7 @@ public final class FeatureRootModel { } pendingSubmissionsByID[submission.id] = submission if let detail = details[submission.threadID] { - store(addingPendingMessages(to: detail)) + store(detail, isServerTranscript: false) } } } @@ -1721,7 +1730,7 @@ public final class FeatureRootModel { store(FeatureThreadDetail( thread: thread, messages: [queuedMessage(for: submission)] - )) + ), isServerTranscript: false) } private func provider(id: String?, environmentID: String) -> FeatureProvider? { @@ -1750,14 +1759,19 @@ public final class FeatureRootModel { ) } - private func addingPendingMessages(to incoming: FeatureThreadDetail) -> FeatureThreadDetail { + private func addingPendingMessages( + to incoming: FeatureThreadDetail, + isServerTranscript: Bool + ) -> FeatureThreadDetail { let existing = Set(incoming.messages.map(\.id)) - serverMessageIDs[incoming.thread.id] = existing // A delivered message stays visible until a server transcript includes it. // Its refresh no longer blocks the send, so an older read can arrive first. - for (id, delivered) in deliveredAwaitingDetail - where delivered.threadID == incoming.thread.id && existing.contains(id) { - deliveredAwaitingDetail.removeValue(forKey: id) + if isServerTranscript { + serverMessageIDs[incoming.thread.id] = existing + for (id, delivered) in deliveredAwaitingDetail + where delivered.threadID == incoming.thread.id && existing.contains(id) { + deliveredAwaitingDetail.removeValue(forKey: id) + } } let queued = pendingSubmissionsByID.values .filter { $0.threadID == incoming.thread.id } @@ -1768,7 +1782,7 @@ public final class FeatureRootModel { .sorted { $0.createdAt < $1.createdAt } guard !queued.isEmpty || !delivered.isEmpty else { return incoming } var result = incoming - result.messages.append(contentsOf: delivered) + result.messages.append(contentsOf: delivered.filter { !existing.contains($0.id) }) result.messages.append(contentsOf: queued.lazy .filter { !existing.contains($0.identity.messageID) } .map(queuedMessage(for:))) diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 30fc011a17bc..89c1dde8bda5 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -1772,6 +1772,55 @@ struct FeatureRootModelTests { #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) } + @Test + func evictedDetailKeepsItsDeliveredMessageOnReopen() async throws { + let (client, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + client.loadThreadHandler = { id in + FeatureThreadDetail(thread: FeatureThread( + id: id, projectID: "project-1", environmentID: "environment-1", title: id + )) + } + // Opening and closing other threads evicts this detail from the bounded cache. + for index in 0..<6 { _ = await model.detail(for: "other-\(index)") } + model.releaseThread("other-5") + #expect(model.details[thread.id] == nil) + // The server transcript has not caught up yet. + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) + } + + @Test + func startedTaskKeepsItsPromptUntilTheServerIncludesIt() async throws { + let client = FeatureClientStub() + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", name: "Studio", endpoint: "https://studio.example", + isActive: true, connectionState: .connected + ), + ], + projects: [.init(id: "project-1", environmentID: "environment-1", name: "Native", path: "/native")] + ) + let model = testRootModel(client: client) + await model.reload() + // The server creates the thread under the ID the app queued it with. + client.beforeStartTask = { @MainActor in + if let pending = model.snapshot.threads.first { client.createdThread = pending } + } + let thread = try #require(await model.startTask(NewTaskRequest( + projectID: "project-1", prompt: "Ship it", selection: nil, + runtimeMode: .fullAccess, interactionMode: .standard + ))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + // An older read that does not include the accepted prompt yet. + client.threadDetail = FeatureThreadDetail(thread: thread) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) + } + @Test func sendPreservesTheThreadAutomaticPermission() async { let client = FeatureClientStub() From 93fe31400265ecfb2440cbc06d0489b271de4210 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:26:44 +1000 Subject: [PATCH 07/19] fix(swift-ios): keep local message copies in send order A newer send can be delivered while an older one waits to retry. Delivered copies were placed before every queued copy, which reversed them. Merge both by send time. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Features/Root/FeatureRootModel.swift | 15 ++++++--------- .../FeatureTests/FeatureRootModelTests.swift | 13 +++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index 567d59e88467..1ff9a34c2b96 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -1773,19 +1773,16 @@ public final class FeatureRootModel { deliveredAwaitingDetail.removeValue(forKey: id) } } - let queued = pendingSubmissionsByID.values - .filter { $0.threadID == incoming.thread.id } - .sorted { $0.identity.createdAt < $1.identity.createdAt } let delivered = deliveredAwaitingDetail.values - .filter { $0.threadID == incoming.thread.id } + .filter { $0.threadID == incoming.thread.id && !existing.contains($0.message.id) } .map(\.message) - .sorted { $0.createdAt < $1.createdAt } + let queued = pendingSubmissionsByID.values + .filter { $0.threadID == incoming.thread.id && !existing.contains($0.identity.messageID) } + .map(queuedMessage(for:)) guard !queued.isEmpty || !delivered.isEmpty else { return incoming } var result = incoming - result.messages.append(contentsOf: delivered.filter { !existing.contains($0.id) }) - result.messages.append(contentsOf: queued.lazy - .filter { !existing.contains($0.identity.messageID) } - .map(queuedMessage(for:))) + // A newer send can be delivered while an older one waits to retry; keep send order. + result.messages.append(contentsOf: (delivered + queued).sorted { $0.createdAt < $1.createdAt }) return result } diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 89c1dde8bda5..66fe2aa236a4 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -1791,6 +1791,19 @@ struct FeatureRootModelTests { #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) } + @Test + func localCopiesKeepSendOrderWhenANewerSendIsDeliveredFirst() async throws { + let (client, model, thread) = await deliveredMessageModel() + client.sendMessageError = URLError(.notConnectedToInternet) + _ = await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "first", selection: nil)) + client.sendMessageError = nil + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "second", selection: nil))) + // An older read that includes neither message. + _ = await model.detail(for: thread.id, force: true) + let texts = model.details[thread.id]?.messages.map(\.text) ?? [] + #expect(texts.filter { $0 == "first" || $0 == "second" } == ["first", "second"]) + } + @Test func startedTaskKeepsItsPromptUntilTheServerIncludesIt() async throws { let client = FeatureClientStub() From 392394d4a7c21c987b2c9fabe69607e3ed9c2fbe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:38:04 +1000 Subject: [PATCH 08/19] fix(swift-ios): recover missing completed thread replies --- apps/swift-ios/App/NativeFeatureClient.swift | 95 +++- .../NativeThreadCatchUpTests.swift | 458 +++++++++++++++++- 2 files changed, 522 insertions(+), 31 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index 008893653031..ff29349a68fa 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -55,6 +55,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private let aggregateRefreshSleep: @Sendable (Duration) async throws -> Void private let environmentShellTimeoutInterval: TimeInterval private let threadSnapshotTimeoutInterval: TimeInterval + private let detailPublicationSleep: @Sendable () async throws -> Void private let catchUpDelay: @Sendable () async throws -> Void private let threadRetryDelay: @Sendable (Int) async throws -> Void private let aggregateEnvironmentLoader: @Sendable (EnvironmentRuntime) async throws -> [Environment] @@ -172,6 +173,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, }, environmentShellTimeoutInterval: TimeInterval = 6, threadSnapshotTimeoutInterval: TimeInterval = 8, + detailPublicationSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(80)) + }, catchUpDelay: @escaping @Sendable () async throws -> Void = { try await Task.sleep(for: .seconds(2)) }, @@ -212,6 +216,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.aggregateRefreshSleep = aggregateRefreshSleep self.environmentShellTimeoutInterval = environmentShellTimeoutInterval self.threadSnapshotTimeoutInterval = threadSnapshotTimeoutInterval + self.detailPublicationSleep = detailPublicationSleep self.catchUpDelay = catchUpDelay self.threadRetryDelay = threadRetryDelay self.aggregateEnvironmentLoader = aggregateEnvironmentLoader @@ -2069,9 +2074,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, activeThreadSequence = cached.sequence activeThreadPage = cached.page detail.page = cached.page + if latestDetails[route.uiID]?.page != detail.page { + publish(detail, threadID: route.uiID, renderCacheIsSource: true) + } markThreadCacheRecentlyUsed(route.uiID) startDetailStream(route, warmConnectionID: warmConnectionID) - return detail + if let shell = shellsByEnvironmentID[environment.id] { + synchronizeActiveDetail(with: shell, environment: environment) + } + return latestDetails[route.uiID] ?? detail } continuation.yield(.threadSync(id: route.uiID, state: .catchingUp)) let snapshot: OrchestrationThreadDetailSnapshot @@ -4763,8 +4774,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pendingDetailRenderMutations.formUnion(mutation) guard detailPublishTask == nil else { return } let streamGeneration = detailStreamGeneration + let sleep = detailPublicationSleep detailPublishTask = Task { [weak self] in - try? await Task.sleep(for: .milliseconds(80)) + do { try await sleep() } catch { return } guard let self else { return } guard !Task.isCancelled, self.detailStreamGeneration == streamGeneration, @@ -5325,10 +5337,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard activeThreadEnvironmentID == environment.id, let threadID = activeThreadID, let wireID = threadWireIDs[threadID], - let shellThread = shell.threads.first(where: { $0.id == wireID }), - var detail = latestDetails[threadID] else { + let shellThread = shell.threads.first(where: { $0.id == wireID }) else { return } + repairMissingCompletedTurn(shellThread, sequence: shell.snapshotSequence, threadID: threadID) + guard var detail = latestDetails[threadID] else { return } let backgroundLiveness = shellThread.backgroundLiveness let backgroundWorkIsActive = backgroundLiveness == .working @@ -5339,22 +5352,26 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detail.thread.supportsTitleRegeneration = capabilities?.threadTitleRegeneration detail.thread.supportsPullRequestLinking = capabilities?.threadPullRequestLinking detail.thread.supportsMultiplePullRequests = capabilities?.threadPullRequests - let sessionIsLive = shellThread.session?.status == "starting" - || shellThread.session?.status == "running" - detail.thread.state = Self.resolveThreadState( - latestTurn: shellThread.latestTurn, - session: shellThread.session, - hasApprovals: !detail.approvals.isEmpty, - hasUserInput: !detail.userInputs.isEmpty, - backgroundLiveness: backgroundLiveness - ) - detail.thread.workingStartedAt = workingStartedAt( - latestTurn: shellThread.latestTurn, - session: shellThread.session, - backgroundWorkIsActive: backgroundWorkIsActive, - fallbackUpdatedAt: shellThread.updatedAt - ) - if shell.snapshotSequence >= (activeThreadSequence ?? .min) { + let shellMetadataIsCurrent = shell.snapshotSequence >= (activeThreadSequence ?? .min) + let latestTurn = shellMetadataIsCurrent ? shellThread.latestTurn : activeRawThread?.latestTurn + let session = shellMetadataIsCurrent ? shellThread.session : activeRawThread?.session + let sessionIsLive = session?.status == "starting" || session?.status == "running" + if shellMetadataIsCurrent || activeRawThread != nil { + detail.thread.state = Self.resolveThreadState( + latestTurn: latestTurn, + session: session, + hasApprovals: !detail.approvals.isEmpty, + hasUserInput: !detail.userInputs.isEmpty, + backgroundLiveness: backgroundLiveness + ) + detail.thread.workingStartedAt = workingStartedAt( + latestTurn: latestTurn, + session: session, + backgroundWorkIsActive: backgroundWorkIsActive, + fallbackUpdatedAt: shellThread.updatedAt + ) + } + if shellMetadataIsCurrent { applyShellMetadataAuthority(from: shellThread, to: &detail.thread) if let compaction = detailRenderCaches[threadID]?.compaction { detail.isCompacting = compaction.isActive( @@ -5372,6 +5389,44 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, publish(detail, threadID: threadID, renderCacheIsSource: true) } + private func repairMissingCompletedTurn( + _ shellThread: OrchestrationThreadShell, sequence: Int, threadID: String + ) { + guard let rawThread = activeRawThread, + sequence >= (activeThreadSequence ?? .min), + let completed = shellThread.latestTurn, completed.state == "completed", + let route = try? threadRoute(for: threadID) else { return } + if let messageID = completed.assistantMessageId { + if let message = rawThread.messages.first(where: { $0.id == messageID }), + !message.streaming { + flushDetailPublish(route) + return + } + } else if rawThread.latestTurn?.turnId == completed.turnId, + rawThread.latestTurn?.state == completed.state, + rawThread.latestTurn?.completedAt == completed.completedAt, + rawThread.messages.contains(where: { + $0.role == "assistant" && $0.turnId == completed.turnId && !$0.streaming + }), + !rawThread.messages.contains(where: { + $0.role == "assistant" && $0.turnId == completed.turnId && $0.streaming + }) { + flushDetailPublish(route) + return + } + + // Equal-cursor completion may flush final content, but only a newer + // shell can prove that detail is missing and require HTTP repair. + guard sequence > (activeThreadSequence ?? .min) else { return } + + // Keep rendered partial text while requiring a snapshot that includes + // the shell's completion. Later detail events advance this same floor. + flushDetailPublish(route) + activeRawThread = nil + activeThreadSequence = sequence + scheduleDetailRefresh(threadID: threadID, client: route.client, force: true) + } + /// Thread-only shell changes stay granular so Home does not replace and /// diff the aggregate snapshot for every active turn update. Structural /// changes retain the canonical snapshot event as a safe fallback. diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index e97c5d3f5718..267a00c1a3e3 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -187,6 +187,286 @@ final class NativeThreadCatchUpTests: XCTestCase { await fixture.client.disconnect() } + func testLegacyBufferedFinalPrecedesShellDone() async throws { + for shellSequence in [3, 10] { + let clock = CatchUpPublicationClock() + let fixture = try await CatchUpFixture.make( + completionMarker: nil, detailPublicationSleep: { try await clock.wait() } + ) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + var entries = clock.entries.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.sendBurst(count: 1, includeMarker: false) + guard await entries.next(isolation: #isolation) != nil else { throw CancellationError() } + try await shell.completeShell(sequence: shellSequence, assistantMessageID: "burst-message") + var reachedDone = false + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + guard value.thread.id == fixture.firstID else { continue } + if value.thread.state == .completed { + XCTAssertEqual(value.messages.last?.text, "x") + reachedDone = true + } + default: continue + } + if reachedDone { break } + } + try Task.checkCancellation() + XCTAssertTrue(reachedDone) + } catch { + await clock.release() + await fixture.client.disconnect() + throw error + } + await clock.release() + await fixture.client.disconnect() + } + } + + func testDelayedCompletedShellPreservesNewerRunningDetailAndBackgroundLiveness() async throws { + for background: String? in [nil, "working"] { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + try await shell.completeShell(sequence: 10, assistantMessageID: nil, title: "Shell 10") + while let event = await events.next(isolation: #isolation) { + if case let .snapshot(snapshot) = event, + snapshot.threads.contains(where: { $0.id == fixture.firstID && $0.title == "Shell 10" }) { break } + } + try Task.checkCancellation() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.runningTurnSnapshot(sequence: 12) + var latest: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): latest = value + case .threadSync(fixture.firstID, .live): break + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(latest?.thread.state, .working) + let startedAt = try XCTUnwrap(latest?.thread.workingStartedAt) + try await shell.completeShell( + sequence: 11, assistantMessageID: nil, backgroundLiveness: background, title: "Shell 11" + ) + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + latest = value + XCTAssertEqual(value.thread.state, .working) + XCTAssertEqual(value.thread.workingStartedAt, startedAt) + default: break + } + if case let .snapshot(snapshot) = event, + snapshot.threads.contains(where: { $0.id == fixture.firstID && $0.title == "Shell 11" }) { break } + } + try Task.checkCancellation() + // The published shell title proves that the delayed shell was applied + // before this detail marker. + try await detail.synchronize() + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + latest = value + XCTAssertEqual(value.thread.state, .working) + XCTAssertEqual(value.thread.workingStartedAt, startedAt) + case .threadSync(fixture.firstID, .live): break + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + try Task.checkCancellation() + XCTAssertEqual(latest?.thread.state, .working) + XCTAssertEqual(latest?.backgroundWorkIsActive, background == "working") + } catch { + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + } + + func testShellCompletionRepairsMissingOrStreamingFinalWithoutClosingDetailStream() async throws { + for mode in ["missing", "streaming", "no-message-id", "no-message-id-streaming", "no-message-id-completed"] { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + if mode == "streaming" || mode == "no-message-id-streaming" { + await fixture.http.setCompletionResponse(text: "Partial", sequence: 2, streaming: true) + } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + if mode == "no-message-id-streaming" || mode == "no-message-id-completed" { + try await detail.completeTurnWithoutMessageID(sequence: 3) + } + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + + await fixture.http.setCompletionResponse(text: "Final assistant response", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Completion starts a required snapshot: \(mode)") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + try await shell.completeShell(sequence: 10, assistantMessageID: mode.hasPrefix("no-message-id") ? nil : "answer-0") + // Failure watchdog only: successful progress is signaled by the held HTTP read. + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let read = await waiting.value else { + await fixture.client.disconnect() + continue + } + read.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final assistant response"], mode) + + // The same subscription remains usable after HTTP repair. + try await detail.sendMessage(text: "Next response", sequence: 11) + try await detail.synchronize() + let next = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(next, ["Final assistant response", "Next response"], mode) + try await shell.completeShell( + sequence: 30, assistantMessageID: mode.hasPrefix("no-message-id") ? nil : "answer-0", + activeOrderKey: "already-complete" + ) + while let event = await events.next(isolation: #isolation) { + if case .threadSync(fixture.firstID, .catchingUp) = event { + XCTFail("Already-complete detail must not start another repair") + } + if case let .detail(value) = event, value.thread.activeOrderKey == "already-complete" { break } + if case let .detailDelta(value, _) = event, value.thread.activeOrderKey == "already-complete" { break } + } + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 2, mode) + await fixture.client.disconnect() + } + } + + func testShellCompletionCoalescesWhileRequiredSnapshotTracksNewerDetailEvents() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setCompletionResponse(text: "Final", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Completion starts required read") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + try await shell.completeShell(sequence: 10, assistantMessageID: "answer-0") + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let first = await waiting.value else { + await fixture.client.disconnect() + return + } + await nextCatchUp(&events, threadID: fixture.firstID) + + // An unrelated shell cursor advance must not invalidate the same completion twice. + try await shell.completeShell(sequence: 100, assistantMessageID: "answer-0", activeOrderKey: "completion-seen-again") + while let event = await events.next(isolation: #isolation) { + if case let .detail(value) = event, value.thread.activeOrderKey == "completion-seen-again" { break } + if case let .detailDelta(value, _) = event, value.thread.activeOrderKey == "completion-seen-again" { break } + } + await fixture.http.setCompletionResponse(text: "Final with newer detail", sequence: 20) + try await detail.sendMessage(text: "Final with newer detail", sequence: 20) + await nextCatchUp(&events, threadID: fixture.firstID) + var reads = fixture.http.heldRequests.makeAsyncIterator() + first.succeed() + let replacement = try await nextHeldRead(&reads) + replacement.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final with newer detail"]) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 3, "Initial read plus one stale repair and one cursor replacement") + await fixture.client.disconnect() + } + + func testWarmCachedOpenRepairsAlreadyCompletedShellWithoutAnotherShellEvent() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let first = try await nextThreadRequest(&requests) + try await first.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + fixture.client.releaseThread(id: fixture.firstID) + try await shell.completeShell(sequence: 10, assistantMessageID: "answer-0", activeOrderKey: "cached-completion") + while let event = await events.next(isolation: #isolation) { + if case let .thread(value) = event, value.activeOrderKey == "cached-completion" { break } + if case let .snapshot(value) = event, + value.threads.contains(where: { $0.activeOrderKey == "cached-completion" }) { break } + } + await fixture.http.setCompletionResponse(text: "Final while closed", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Cached completion starts required read on reopen") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + _ = try await fixture.client.loadThread(id: fixture.firstID) + let resumed = try await nextThreadRequest(&requests) + try await resumed.synchronize() + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let read = await waiting.value else { + await fixture.client.disconnect() + return + } + // Ignore the warm-cache immediate live receipt preceding the repair. + while let event = await events.next(isolation: #isolation) { + if case .threadSync(fixture.firstID, .catchingUp) = event { break } + } + read.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final while closed"]) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 2) + await fixture.client.disconnect() + } + + private func nextShellRequest( + _ iterator: inout AsyncStream.Iterator + ) async throws -> CatchUpRequest { + while let request = await iterator.next(isolation: #isolation) { + if request.tag == RPCMethod.subscribeShell.rawValue { return request } + } + throw CancellationError() + } + func testRequestSnapshotsKeepTerminalRequestsClosedAndOtherFailuresRetryable() async throws { var activities: [OrchestrationActivity] = [] for kind in ["approval", "user-input"] { @@ -444,6 +724,34 @@ final class NativeThreadCatchUpTests: XCTestCase { await fixture.client.disconnect() } + func testWarmReopenClearsAnInterruptedOlderPageLoadingState() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + await fixture.http.setPage(.init(beforeCursor: "older", hasMore: true, snapshotSequence: 2)) + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + var reads = fixture.http.heldRequests.makeAsyncIterator() + let older = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let held = try await nextHeldRead(&reads) + fixture.client.releaseThread(id: fixture.firstID) + let restored = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(restored.page?.isLoading, false) + XCTAssertEqual(restored.page?.hasMore, true) + held.succeed() + _ = try await older.value + await fixture.http.holdThreadReads(false) + await fixture.http.setPage(.init(beforeCursor: nil, hasMore: false, snapshotSequence: 2)) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 3, "Reopened history must allow another older-page request") + await fixture.client.disconnect() + } + func testWarmNavigationResumesAfterAppliedMessagesWithoutAnotherHTTPRead() async throws { let fixture = try await CatchUpFixture.make() defer { fixture.cleanUp() } @@ -1116,8 +1424,11 @@ private struct CatchUpFixture { var secondID: String { FeatureScopedID.thread(environmentID: "one", wireID: "second") } static func make( - completionMarker: Bool = true, + completionMarker: Bool? = true, activities: [OrchestrationActivity] = [], + detailPublicationSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(80)) + }, threadRetryDelay: @escaping @Sendable (Int) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(250)) } @@ -1145,6 +1456,7 @@ private struct CatchUpFixture { runtime: runtime, settingsStore: UserDefaults(suiteName: UUID().uuidString)!, fallbackPollingInitialDelay: .seconds(3_600), aggregateRefreshInterval: .seconds(3_600), + detailPublicationSleep: detailPublicationSleep, catchUpDelay: { try await delay.wait() }, threadRetryDelay: threadRetryDelay ) @@ -1159,7 +1471,8 @@ private actor CatchUpHTTPTransport: HTTPTransport { private(set) var threadRequests: [URLRequest] = [] private var messages: [OrchestrationMessage] = [] private var activities: [OrchestrationActivity] = [] - private var page: OrchestrationThreadDetailPage? + private var completionResponse = false + private var responsePage: OrchestrationThreadDetailPage? private var sequence = 2 private var holdsThreadReads = false private let heldReadContinuation: AsyncStream.Continuation @@ -1183,11 +1496,21 @@ private actor CatchUpHTTPTransport: HTTPTransport { self.sequence = sequence } + func setCompletionResponse(text: String, sequence: Int, streaming: Bool = false) { + messages = [OrchestrationMessage( + id: "answer-0", role: "assistant", text: text, attachments: [], turnId: "turn-1", + streaming: streaming, createdAt: "2026-09-02T12:00:00Z", updatedAt: "2026-09-02T12:00:00Z" + )] + self.sequence = sequence + completionResponse = !streaming + } + + func setPage(_ page: OrchestrationThreadDetailPage) { responsePage = page } + func holdThreadReads(_ hold: Bool) { holdsThreadReads = hold } func setActivities(_ activities: [OrchestrationActivity]) { self.activities = activities } - func setPage(_ page: OrchestrationThreadDetailPage) { self.page = page } func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { let value: JSONValue @@ -1212,8 +1535,13 @@ private actor CatchUpHTTPTransport: HTTPTransport { ) var thread = snapshot.thread thread.activities = activities + if completionResponse { + var object = try JSONValue.encode(thread).decode([String: JSONValue].self) + object["latestTurn"] = catchUpCompletedTurn(assistantMessageID: "answer-0") + thread = try JSONValue.object(object).decode(OrchestrationThread.self) + } value = try .encode(OrchestrationThreadDetailSnapshot( - snapshotSequence: snapshot.snapshotSequence, thread: thread, page: page ?? snapshot.page + snapshotSequence: snapshot.snapshotSequence, thread: thread, page: responsePage ?? snapshot.page )) } let response = (try JSONEncoder.t3.encode(value), HTTPURLResponse( @@ -1258,7 +1586,7 @@ private func catchUpMessage( private struct CatchUpConnector: WebSocketConnecting { let requests: AsyncStream.Continuation - let completionMarker: Bool + let completionMarker: Bool? func connect(to url: URL) async throws -> any WebSocketConnection { CatchUpSocket(requests: requests, completionMarker: completionMarker) } @@ -1288,6 +1616,35 @@ private struct CatchUpRequest: Sendable { ]) } + func completeShell(sequence: Int, assistantMessageID: String?, activeOrderKey: String? = nil, backgroundLiveness: String? = nil, title: String = "First") async throws { + let shell = multiEnvironmentShell(projectID: "project", threadID: "first", title: title) + var thread = try JSONValue.encode(shell.threads[0]).decode([String: JSONValue].self) + thread["latestTurn"] = catchUpCompletedTurn(assistantMessageID: assistantMessageID) + thread["activeOrderKey"] = activeOrderKey.map(JSONValue.string) + thread["backgroundLiveness"] = backgroundLiveness.map(JSONValue.string) + let snapshot = OrchestrationShellSnapshot( + snapshotSequence: sequence, projects: shell.projects, + threads: [try JSONValue.object(thread).decode(OrchestrationThreadShell.self)], updatedAt: shell.updatedAt + ) + try await socket.chunk(id: id, values: [.object([ + "kind": .string("snapshot"), "snapshot": try .encode(snapshot), + ])]) + } + + func completeTurnWithoutMessageID(sequence: Int) async throws { + try await socket.chunk(id: id, values: [.object([ + "kind": .string("event"), "event": .object([ + "type": .string("thread.turn-diff-completed"), "sequence": .number(Double(sequence)), + "occurredAt": .string("2026-09-02T12:01:00Z"), "payload": .object([ + "threadId": payload["threadId"]!, "turnId": .string("turn-1"), + "checkpointTurnCount": .number(1), "checkpointRef": .string("refs/t3/checkpoints/turn-1"), + "status": .string("ready"), "files": .array([]), + "completedAt": .string("2026-09-02T12:01:00Z"), "assistantMessageId": .null, + ]), + ]), + ])]) + } + func terminate(_ failure: CatchUpStreamFailure) async throws { switch failure { case .malformed: @@ -1342,6 +1699,45 @@ private struct CatchUpRequest: Sendable { ])]) } + func runningTurnSnapshot(sequence: Int) async throws { + let snapshot = multiEnvironmentDetail( + projectID: "project", threadID: payload["threadId"]!.stringValue!, snapshotSequence: sequence + ) + var thread = try JSONValue.encode(snapshot.thread).decode([String: JSONValue].self) + thread["latestTurn"] = .object([ + "turnId": .string("turn-2"), "state": .string("running"), + "requestedAt": .string("2026-09-02T12:02:00Z"), + "startedAt": .string("2026-09-02T12:02:00Z"), + "completedAt": .null, "assistantMessageId": .null, + ]) + thread["session"] = catchUpRunningSession(threadID: payload["threadId"]!.stringValue!, turnID: "turn-2") + let updated = OrchestrationThreadDetailSnapshot( + snapshotSequence: sequence, thread: try JSONValue.object(thread).decode(OrchestrationThread.self) + ) + try await socket.chunk(id: id, values: [ + .object(["kind": .string("snapshot"), "snapshot": try .encode(updated)]), + ]) + } + + func sendBurst(count: Int, includeMarker: Bool) async throws { + var values: [JSONValue] = (1...count).map { index in + .object([ + "kind": .string("event"), "event": .object([ + "type": .string("thread.message-sent"), "sequence": .number(Double(index + 2)), + "occurredAt": .string("2026-09-02T12:00:00Z"), "payload": .object([ + "threadId": payload["threadId"]!, "messageId": .string("burst-message"), + "role": .string("assistant"), "text": .string(index < count ? "x" : String(repeating: "x", count: count)), + "streaming": .bool(index < count), + "createdAt": .string("2026-09-02T12:00:00Z"), + "updatedAt": .string("2026-09-02T12:00:00Z"), + ]), + ]), + ]) + } + if includeMarker { values.append(.object(["kind": .string("synchronized")])) } + try await socket.chunk(id: id, values: values) + } + func sendMessage(text: String, sequence: Int) async throws { try await socket.chunk(id: id, values: [.object([ "kind": .string("event"), "event": .object([ @@ -1359,13 +1755,13 @@ private struct CatchUpRequest: Sendable { private actor CatchUpSocket: WebSocketConnection { let requests: AsyncStream.Continuation - let completionMarker: Bool + let completionMarker: Bool? private(set) var assetRequestCount = 0 private var pending: [Data] = [] private var receiver: CheckedContinuation? private var closed = false - init(requests: AsyncStream.Continuation, completionMarker: Bool) { + init(requests: AsyncStream.Continuation, completionMarker: Bool?) { self.requests = requests self.completionMarker = completionMarker } @@ -1379,11 +1775,12 @@ private actor CatchUpSocket: WebSocketConnection { guard let tag = request["tag"]?.stringValue, case let .number(id) = request["id"] else { return } if tag == RPCMethod.assetsCreateURL.rawValue { assetRequestCount += 1 } if tag == RPCMethod.subscribeServerConfig.rawValue { + var config: [String: JSONValue] = [ + "providers": .array([]), "threadSnapshotPagination": .bool(true), + ] + if let completionMarker { config["threadResumeCompletionMarker"] = .bool(completionMarker) } try chunk(id: Int(id), values: [.object([ - "type": .string("snapshot"), "config": .object([ - "providers": .array([]), "threadSnapshotPagination": .bool(true), - "threadResumeCompletionMarker": .bool(completionMarker), - ]), + "type": .string("snapshot"), "config": .object(config), ])]) } requests.yield(.init(tag: tag, id: Int(id), payload: request["payload"]!, socket: self)) @@ -1506,3 +1903,42 @@ private actor CatchUpDelay { waiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) } } + +/// The session, not `latestTurn`, marks a thread as working. +private func catchUpRunningSession(threadID: String, turnID: String) -> JSONValue { + .object([ + "threadId": .string(threadID), "status": .string("running"), + "providerName": .string("codex"), "providerInstanceId": .null, + "runtimeMode": .string("full-access"), "activeTurnId": .string(turnID), + "lastError": .null, "updatedAt": .string("2026-09-02T12:02:00Z"), + ]) +} + +private func catchUpCompletedTurn(assistantMessageID: String?) -> JSONValue { + .object([ + "turnId": .string("turn-1"), "state": .string("completed"), + "requestedAt": .string("2026-09-02T12:00:00Z"), + "startedAt": .string("2026-09-02T12:00:00Z"), + "completedAt": .string("2026-09-02T12:01:00Z"), + "assistantMessageId": assistantMessageID.map(JSONValue.string) ?? .null, + ]) +} + +private final class CatchUpPublicationClock: Sendable { + private let gate = CatchUpDelay() + private let receipt: AsyncStream.Continuation + let entries: AsyncStream + + init() { + let pair = AsyncStream.makeStream() + entries = pair.stream + receipt = pair.continuation + } + + func wait() async throws { + receipt.yield(()) + try await gate.wait() + } + + func release() async { await gate.release() } +} From d21341c87a81987f658b652756a88814d8d99ad0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:42:56 +1000 Subject: [PATCH 09/19] fix(swift-ios): hydrate live environments independently --- apps/swift-ios/App/NativeFeatureClient.swift | 1171 ++++++++--- apps/swift-ios/Core/T3Client.swift | 23 + .../Features/Root/FeatureRootModel.swift | 1 + .../Features/Shared/FeatureClient.swift | 3 + .../FeatureTests/FeatureRootModelTests.swift | 5 +- .../NativeMultiEnvironmentTests.swift | 1741 +++++++++++++++-- .../NativeRetryIdentityTests.swift | 33 +- .../NativeThreadCatchUpTests.swift | 36 + .../T3ConnectNativeCapabilityTests.swift | 4 +- 9 files changed, 2553 insertions(+), 464 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index ff29349a68fa..b1af76250544 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -53,6 +53,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private let aggregateIdleRefreshInterval: Duration private let aggregateFailureRefreshInterval: Duration private let aggregateRefreshSleep: @Sendable (Duration) async throws -> Void + private let aggregateRefreshReceipt: @MainActor @Sendable (NativePassiveShellReceipt) -> Void + private let aggregateStreamRetrySleep: @Sendable (String, Duration) async throws -> Void + private let aggregatePublishSleep: @Sendable () async throws -> Void + private let aggregatePeerRefreshSleep: @Sendable (String, Duration) async throws -> Void private let environmentShellTimeoutInterval: TimeInterval private let threadSnapshotTimeoutInterval: TimeInterval private let detailPublicationSleep: @Sendable () async throws -> Void @@ -125,11 +129,26 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var terminalSnapshots: [TerminalKey: FeatureTerminalSnapshot] = [:] // Keep versions unique when a terminal cache is evicted or an environment reconnects. private var terminalLifecycleVersion = 0 + private var foregroundBootstrapID = UUID() + private var activeHydrationTask: Task? + private var activeHydrationID: UUID? + private var activeHydrationPending = false + private var activeHTTPAuthorityRevision = 0 + private var shellConnectionIDsByEnvironmentID: [String: UUID] = [:] + private var activeShellConnectionID: UUID? + private var activeShellEpochHasSnapshot = false + private var activeHasHydrated = false + private var activeStreamIsAuthoritative = false private var pollingTask: Task? private var fallbackPollingTask: Task? private var configurationTask: Task? private var aggregateRefreshTask: Task? private var aggregateRefreshID: UUID? + private var aggregatePublishTask: Task? + private var isForeground = true + private(set) var aggregateRefreshWorkers: [ + String: (environment: Environment, task: Task) + ] = [:] private var shellPublishTask: Task? private var archivedRefreshTask: Task? private var detailRefreshTask: Task? @@ -145,7 +164,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var detailStreamGeneration = 0 private var pendingDetailRenderMutations = NativeDetailRenderMutations() private var environmentGeneration = 0 - private var lastShellEventAt: Date? private var activeRawThread: OrchestrationThread? private var activeThreadSequence: Int? private var activeThreadPage: FeatureThreadPage? @@ -171,6 +189,16 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, aggregateRefreshSleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }, + aggregatePeerRefreshSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(250)) + }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, environmentShellTimeoutInterval: TimeInterval = 6, threadSnapshotTimeoutInterval: TimeInterval = 8, detailPublicationSleep: @escaping @Sendable () async throws -> Void = { @@ -213,6 +241,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.aggregateRefreshInterval = aggregateRefreshInterval self.aggregateIdleRefreshInterval = aggregateIdleRefreshInterval self.aggregateFailureRefreshInterval = aggregateFailureRefreshInterval + self.aggregateRefreshReceipt = aggregateRefreshReceipt + self.aggregateStreamRetrySleep = aggregateStreamRetrySleep + self.aggregatePublishSleep = aggregatePublishSleep + self.aggregatePeerRefreshSleep = aggregatePeerRefreshSleep self.aggregateRefreshSleep = aggregateRefreshSleep self.environmentShellTimeoutInterval = environmentShellTimeoutInterval self.threadSnapshotTimeoutInterval = threadSnapshotTimeoutInterval @@ -226,10 +258,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } deinit { + activeHydrationTask?.cancel() pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() aggregateRefreshTask?.cancel() + aggregatePublishTask?.cancel() + aggregateRefreshWorkers.values.forEach { $0.task.cancel() } shellPublishTask?.cancel() archivedRefreshTask?.cancel() detailRefreshTask?.cancel() @@ -242,46 +277,172 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func initialSnapshot() async throws -> FeatureSnapshot { + beginForegroundBootstrap() + let bootstrapID = foregroundBootstrapID let environments = try await runtime.environments() + guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } guard let activeClient = try await runtime.activeClient() else { - await clearActiveEnvironment() + await clearActiveEnvironment(disconnectClient: false) let snapshot = disconnectedSnapshot(environments: environments) latestSnapshot = snapshot return snapshot } - // The runtime actor can change its active selection at any suspension - // point. Derive both values from one client so the snapshot cannot pair - // one environment with another environment's connection. + guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } let environment = activeClient.environment - await adoptEnvironment(environment, client: activeClient) - let generation = environmentGeneration - let loads = await loadEnvironmentShells(environments.filter(\.isEnabled)) - guard isCurrentSession(client: activeClient, generation: generation) else { + guard bootstrapID == foregroundBootstrapID, + isCurrentSession(client: activeClient, generation: environmentGeneration) else { throw CancellationError() } - reconcileEnvironmentLoads(loads, savedEnvironments: environments) - latestShell = shellsByEnvironmentID[environment.id] - startPolling(activeClient) - let activeLoad = loads.first { $0.environment.id == environment.id } - let activeIsReachable = activeLoad?.shell != nil - if activeIsReachable { - scheduleArchivedRefresh(client: activeClient, environment: environment) - } - if activeLoad?.credentialRejected == true { - markActiveEnvironmentNeedsPairing(detail: activeLoad?.failureDetail) + reconcileEnvironmentLoads([], savedEnvironments: environments) + for saved in environments where saved.isEnabled { + environmentConnectionStates[saved.id] = .reconnecting + environmentConnectionDetails[saved.id] = nil } + latestShell = shellsByEnvironmentID[environment.id] let snapshot = makeSnapshot( - environments: environments, - activeEnvironment: environment, - connectionState: environmentConnectionStates[environment.id] ?? .disconnected, - connectionDetail: environmentConnectionDetails[environment.id] + environments: environments, activeEnvironment: environment, + connectionState: .reconnecting ) latestSnapshot = snapshot + // No suspension after assigning the seed: the root installs it before + // it consumes buffered hydration events through its single iterator. + if isForeground { + startPolling(activeClient) + requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + startAggregateRefresh(activeClient, refreshImmediately: true) + } return snapshot } + private func beginForegroundBootstrap() { + foregroundBootstrapID = UUID() + activeHydrationTask?.cancel() + activeHydrationTask = nil + activeHydrationID = nil + activeHydrationPending = false + activeHTTPAuthorityRevision &+= 1 + activeHasHydrated = false + activeStreamIsAuthoritative = false + pollingTask?.cancel() + fallbackPollingTask?.cancel() + configurationTask?.cancel() + archivedRefreshTask?.cancel() + archivedRefreshTask = nil + shellPublishTask?.cancel() + shellPublishTask = nil + cancelAggregateRefresh() + } + + private func isCurrentForeground(_ client: T3Client, generation: Int, bootstrapID: UUID) -> Bool { + !Task.isCancelled && isForeground && foregroundBootstrapID == bootstrapID + && isCurrentSession(client: client, generation: generation) + } + + /// Bootstrap, fallback polling, and unknown stream events share one HTTP + /// request owner. An invalidated request drains before its pending repair. + private func requestActiveShellHydration(_ activeClient: T3Client, bootstrapID: UUID, force: Bool = false) { + let generation = environmentGeneration + guard isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } + guard activeHydrationTask == nil else { + if force { activeHydrationPending = true } + return + } + let requestID = UUID() + let revision = activeHTTPAuthorityRevision + let runtime = runtime + let timeout = environmentShellTimeoutInterval + activeHydrationID = requestID + activeHydrationPending = false + activeHydrationTask = Task { [weak self] in + let shell: OrchestrationShellSnapshot? + var hydrationError: (any Error)? + do { shell = try await activeClient.shellSnapshot(timeoutInterval: timeout) } + catch { shell = nil; hydrationError = error } + let environments = try? await runtime.environments() + guard let self else { return } + defer { self.aggregateRefreshReceipt(.httpFinished(environmentID: activeClient.environment.id)) } + guard self.activeHydrationID == requestID, + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) + else { return } + self.activeHydrationTask = nil + self.activeHydrationID = nil + guard let environments, environments.contains(activeClient.environment), + activeClient.environment.isEnabled else { return } + if revision == self.activeHTTPAuthorityRevision { + if let error = hydrationError, error.isRejectedAuthorization { + self.markActiveEnvironmentNeedsPairing(detail: error.localizedDescription) + return + } + if let shell { + self.activeHydrationPending = false + self.acceptActiveShell(shell, client: activeClient, environments: environments, + socketAuthoritative: false, connectHeader: false) + } else { + self.emitConnection( + self.activeHasHydrated ? .reconnecting : .disconnected, + detail: "Server unreachable. Retrying automatically." + ) + } + } + if self.activeHydrationPending && !self.activeStreamIsAuthoritative { + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + } + } + } + + private func acceptActiveShell( + _ candidate: OrchestrationShellSnapshot, client: T3Client, + environments: [Environment], socketAuthoritative: Bool, connectHeader: Bool + ) { + let shell: OrchestrationShellSnapshot + if activeShellEpochHasSnapshot, let cached = latestShell, + cached.snapshotSequence > candidate.snapshotSequence { shell = cached } + else { shell = candidate } + let firstHydration = !activeHasHydrated + activeShellEpochHasSnapshot = true + activeHasHydrated = true + if socketAuthoritative { + shellConnectionIDsByEnvironmentID[client.environment.id] = activeShellConnectionID + activeStreamIsAuthoritative = true + activeHTTPAuthorityRevision &+= 1 + activeHydrationPending = false + } + let removed = Set(shellsByEnvironmentID[client.environment.id]?.threads.map(\.id) ?? []) + .subtracting(shell.threads.map(\.id)) + for id in removed { + clearRemovedThreadDetail(FeatureScopedID.thread(environmentID: client.environment.id, wireID: id)) + } + latestShell = shell + shellsByEnvironmentID[client.environment.id] = shell + environmentConnectionStates[client.environment.id] = .connected + environmentConnectionDetails[client.environment.id] = nil + rebuildEntityIndexes(environments) + synchronizeActiveDetail(with: shell, environment: client.environment) + guard let activeEnvironment else { return } + publish(makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: connectHeader ? .connected : (latestSnapshot?.connection.state ?? .reconnecting), + connectionDetail: connectHeader ? nil : latestSnapshot?.connection.detail + )) + aggregateRefreshReceipt(.shellApplied(environmentID: client.environment.id, sequence: shell.snapshotSequence)) + if firstHydration { scheduleArchivedRefresh(client: client, environment: client.environment) } + } + + func suspendForBackground() { + isForeground = false + beginForegroundBootstrap() + } + func resumeAfterBackground(reconnect: Bool) async { + isForeground = true + if client == nil { + if let snapshot = try? await initialSnapshot(), isForeground { + continuation.yield(.snapshot(snapshot)) + } + return + } + if let client { startAggregateRefresh(client) } let sessionGeneration = environmentGeneration let selectedRoute = activeThreadID.flatMap { try? threadRoute(for: $0) } detailWasSynchronized = false @@ -306,7 +467,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } guard sessionGeneration == environmentGeneration else { return } - if let client { startPolling(client) } + if let client { + startPolling(client) + requestActiveShellHydration(client, bootstrapID: foregroundBootstrapID) + } if let selectedRoute, activeThreadID == selectedRoute.uiID, isKnownClient(selectedRoute.client, environmentID: selectedRoute.environmentID, generation: sessionGeneration) { resetDetailRefresh() @@ -356,6 +520,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pairedClient = try await runtime.pair(url: endpoint, clientLabel: "T3 Code Swift") } await adoptEnvironment(pairedClient.environment, client: pairedClient) + startAggregateRefresh(pairedClient) startPolling(pairedClient) } @@ -432,6 +597,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) publish(snapshot) } + startAggregateRefresh(managedClient) startPolling(managedClient) } @@ -483,6 +649,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func setEnvironmentEnabled(id: String, enabled: Bool) async throws { + if !enabled { + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() + } try await runtime.setEnabled(id: id, enabled: enabled) if !enabled { cancelAcceptedCommandRefreshes(environmentID: id) @@ -499,6 +669,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func removeEnvironment(id: String) async throws { + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() let removesActiveEnvironment = activeEnvironment?.id == id let environment = try await runtime.environments().first { $0.id == id } if environment?.kind == .managedDPoP { @@ -988,18 +1160,22 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, _ environment: Environment, client newClient: T3Client ) async { + cancelAggregateRefresh() if activeEnvironment?.id == environment.id, client === newClient { activeEnvironment = environment environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - startAggregateRefresh(newClient) return } - let previousClient = client + let bootstrapID = foregroundBootstrapID + let generation = environmentGeneration + let adoptedConnectionID = await newClient.currentConnectionID() + let selectedEnvironment = try? await runtime.activeEnvironment() + guard bootstrapID == foregroundBootstrapID, generation == environmentGeneration, + selectedEnvironment == environment else { return } pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() - aggregateRefreshTask?.cancel() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil @@ -1007,23 +1183,27 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, aggregateRefreshTask = nil aggregateRefreshID = nil archivedRefreshTask = nil + // A passive snapshot is authoritative for adoption only while its + // exact socket is still current. A replacement socket starts a new epoch. + activeShellConnectionID = adoptedConnectionID + activeShellEpochHasSnapshot = adoptedConnectionID != nil + && shellConnectionIDsByEnvironmentID[environment.id] == adoptedConnectionID clearEnvironmentState(preserveEnvironmentSnapshots: true) activeEnvironment = environment client = newClient environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - if let previousClient, previousClient !== newClient { - await previousClient.disconnect() - } - startAggregateRefresh(newClient) + // Runtime owns shared transports. The former inbox client may now + // serve a passive shell or a selected detail, so adoption does not close it. } private func clearActiveEnvironment(disconnectClient: Bool = true) async { + beginForegroundBootstrap() let previousClient = client pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() - aggregateRefreshTask?.cancel() + cancelAggregateRefresh() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil @@ -1040,6 +1220,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func clearEnvironmentState(preserveEnvironmentSnapshots: Bool = false) { + activeHydrationTask?.cancel() + activeHydrationTask = nil + activeHydrationID = nil + activeHydrationPending = false + activeHTTPAuthorityRevision &+= 1 + activeHasHydrated = false + activeStreamIsAuthoritative = false environmentGeneration &+= 1 cancelAcceptedCommandRefreshes() resetDetailRefresh() @@ -1049,10 +1236,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, shellPublishTask?.cancel() shellPublishTask = nil latestShell = nil - lastShellEventAt = nil latestServerConfig = nil if !preserveEnvironmentSnapshots { environmentClients.removeAll() + shellConnectionIDsByEnvironmentID.removeAll() shellsByEnvironmentID.removeAll() shellProjectionCache.removeAll() indexedShellMembership = nil @@ -3848,64 +4035,67 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, fallbackPollingTask?.cancel() configurationTask?.cancel() let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + let streamRetrySleep = aggregateStreamRetrySleep pollingTask = Task { [weak self] in while !Task.isCancelled, - self?.isCurrentSession(client: activeClient, generation: generation) == true + self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true { do { await activeClient.connect() guard - self?.isCurrentSession( - client: activeClient, - generation: generation - ) == true + self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true else { return } - let sequence = self?.latestShell?.snapshotSequence - let events = await activeClient.shellEventBatches(after: sequence, reconnect: false) + let subscription = try await activeClient.shellEventBatchesOnCurrentConnection() + guard self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true + else { return } + if let previous = self?.activeShellConnectionID, previous != subscription.connectionID { + self?.activeShellEpochHasSnapshot = false + self?.activeHTTPAuthorityRevision &+= 1 + self?.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + } + self?.activeShellConnectionID = subscription.connectionID + let events = subscription.events // Re-bind self per event instead of holding it strongly across // the indefinite stream, so the client can deinit mid-stream. - for try await batch in events { + shellEvents: for try await batch in events { guard !Task.isCancelled, let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { break } - self.lastShellEventAt = .now - self.emitConnection(.connected) + let environments = try await self.runtime.environments() + let connectionID = await activeClient.currentConnectionID() + guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), + connectionID == subscription.connectionID, + environments.contains(activeClient.environment) else { break } var deltas: [ShellStreamItem] = [] for item in batch { - switch item { - case let .snapshot(shell): - await self.consume(deltas: deltas, client: activeClient, generation: generation) - deltas.removeAll(keepingCapacity: true) - await self.consume( - shell: shell, - client: activeClient, - generation: generation, - refreshActiveThread: true - ) - case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: - deltas.append(item) - case .refreshRequired: - await self.consume(deltas: deltas, client: activeClient, generation: generation) - deltas.removeAll(keepingCapacity: true) - if let shell = try? await activeClient.shellSnapshot() { - await self.consume( - shell: shell, - client: activeClient, - generation: generation, - refreshActiveThread: true - ) - } - case .synchronized: - break + switch item { + case let .snapshot(shell): + await self.consume(deltas: deltas, client: activeClient, generation: generation) + deltas.removeAll(keepingCapacity: true) + self.acceptActiveShell(shell, client: activeClient, environments: environments, + socketAuthoritative: true, connectHeader: true) + case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: + guard self.activeStreamIsAuthoritative else { + self.activeHTTPAuthorityRevision &+= 1 + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + break shellEvents } + deltas.append(item) + case .refreshRequired: + self.activeStreamIsAuthoritative = false + self.activeHTTPAuthorityRevision &+= 1 + self.emitConnection(.reconnecting, detail: "Refreshing environment data.") + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + break shellEvents + case .synchronized: + break + } } await self.consume(deltas: deltas, client: activeClient, generation: generation) } @@ -3924,16 +4114,21 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard !Task.isCancelled, let self, - self.isCurrentSession(client: activeClient, generation: generation) + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } - self.lastShellEventAt = nil + self.shellConnectionIDsByEnvironmentID[activeClient.environment.id] = nil + if self.activeStreamIsAuthoritative { + self.activeHTTPAuthorityRevision &+= 1 + } + self.activeStreamIsAuthoritative = false + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) self.emitConnection( .reconnecting, detail: "Live updates paused. Refreshing over HTTP." ) - do { try await Task.sleep(for: .milliseconds(250)) } catch { return } + do { try await streamRetrySleep(activeClient.environment.id, .seconds(20)) } catch { return } } } let fallbackPollingInitialDelay = fallbackPollingInitialDelay @@ -3946,53 +4141,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } while !Task.isCancelled { guard let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } - let socketIsSynchronized = - await activeClient.liveConnectionActive() - && self.lastShellEventAt != nil - if !socketIsSynchronized { - self.emitConnection( - .reconnecting, - detail: "Live updates reconnecting. Refreshing over HTTP." - ) - do { - let shell = try await activeClient.shellSnapshot() - guard !Task.isCancelled, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { - return - } - await self.consumeFallbackShell( - shell: shell, - client: activeClient, - generation: generation - ) - } catch is CancellationError { - return - } catch { - guard !Task.isCancelled, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { - return - } - if error.isRejectedAuthorization { - self.markActiveEnvironmentNeedsPairing(detail: error.localizedDescription) - return - } - self.emitConnection( - .reconnecting, - detail: "Server unreachable. Retrying automatically." - ) - } + if !self.activeStreamIsAuthoritative { + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) } do { try await Task.sleep(for: fallbackPollingInterval) @@ -4006,10 +4159,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, for try await event in await activeClient.serverConfigEvents() { guard !Task.isCancelled, let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { break } switch event { @@ -4071,11 +4221,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, case .unrelated: continue } - if let shell = self.latestShell { - await self.emitSnapshot( - shell, client: activeClient, expectedGeneration: generation - ) - } + let environments = try await self.runtime.environments() + guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), + environments.contains(activeClient.environment), let activeEnvironment = self.activeEnvironment + else { return } + self.publish(self.makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: self.latestSnapshot?.connection.state ?? .reconnecting, + connectionDetail: self.latestSnapshot?.connection.detail + )) } } catch is CancellationError { return @@ -4089,124 +4243,455 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, /// Non-active environments do not hold WebSocket subscriptions. A quiet /// HTTP refresh keeps their home rows and reachability useful without /// multiplying live streams or creating a high-frequency battery cost. - private func startAggregateRefresh(_ activeClient: T3Client) { + private func cancelAggregateRefresh() { aggregateRefreshTask?.cancel() - let generation = environmentGeneration - let refreshID = UUID() - let fastInterval = aggregateRefreshInterval - let idleInterval = aggregateIdleRefreshInterval - let failureInterval = aggregateFailureRefreshInterval - let sleep = aggregateRefreshSleep - let loadEnvironments = aggregateEnvironmentLoader - aggregateRefreshID = refreshID - aggregateRefreshTask = Task { [weak self] in - var nextInterval = fastInterval - var failureBackoffs: [String: Duration] = [:] - while !Task.isCancelled { - let elapsedInterval = nextInterval + aggregatePublishTask?.cancel() + aggregateRefreshWorkers.values.forEach { $0.task.cancel() } + aggregateRefreshWorkers.removeAll() + aggregatePublishTask = nil + aggregateRefreshID = nil + } + + private func startAggregateRefresh(_ activeClient: T3Client, refreshImmediately: Bool = false) { + cancelAggregateRefresh() + guard isForeground else { return } + let context = AggregateRefreshContext(owner: self, activeClient: activeClient, refreshImmediately: refreshImmediately) + aggregateRefreshID = context.refreshID + aggregateRefreshTask = Task { + defer { context.cancelWorkers() } + while context.isCurrent { + var interval = context.fastInterval do { - try await sleep(nextInterval) - } catch { + let environments = try await context.loadEnvironments(context.runtime) + guard context.isCurrent else { return } + interval = context.reconcileWorkers(environments) + } catch is CancellationError where Task.isCancelled { return + } catch { + interval = context.failureInterval } - guard let self, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ), - let activeEnvironment = self.activeEnvironment else { - return + do { try await context.sleep(interval) } catch { return } + } + } + } + + /// A worker retains runtime inputs, never its UI owner across a suspension. + /// This lets deinit cancel the exact outstanding shell/catalogue tasks. + @MainActor + private final class AggregateRefreshContext { + weak var owner: NativeFeatureClient? + let runtime: EnvironmentRuntime + let activeClient: T3Client + let generation: Int + let refreshID = UUID() + let refreshImmediately: Bool + let fastInterval: Duration + let idleInterval: Duration + let failureInterval: Duration + let shellTimeout: TimeInterval + let sleep: @Sendable (Duration) async throws -> Void + let peerSleep: @Sendable (String, Duration) async throws -> Void + let loadEnvironments: @Sendable (EnvironmentRuntime) async throws -> [Environment] + let streamRetrySleep: @Sendable (String, Duration) async throws -> Void + let publishSleep: @Sendable () async throws -> Void + let refreshReceipt: @MainActor @Sendable (NativePassiveShellReceipt) -> Void + var savedEnvironments: [Environment] = [] + var dirtyEnvironmentIDs = Set() + + init(owner: NativeFeatureClient, activeClient: T3Client, refreshImmediately: Bool) { + self.owner = owner + self.refreshImmediately = refreshImmediately + runtime = owner.runtime + self.activeClient = activeClient + generation = owner.environmentGeneration + fastInterval = owner.aggregateRefreshInterval + idleInterval = owner.aggregateIdleRefreshInterval + failureInterval = owner.aggregateFailureRefreshInterval + shellTimeout = owner.environmentShellTimeoutInterval + sleep = owner.aggregateRefreshSleep + peerSleep = owner.aggregatePeerRefreshSleep + loadEnvironments = owner.aggregateEnvironmentLoader + streamRetrySleep = owner.aggregateStreamRetrySleep + publishSleep = owner.aggregatePublishSleep + refreshReceipt = owner.aggregateRefreshReceipt + } + + var isCurrent: Bool { + !Task.isCancelled && owner?.isForeground == true && owner?.aggregateRefreshID == refreshID + && owner?.isCurrentSession(client: activeClient, generation: generation) == true + } + + func cancelWorkers() { + guard let owner, owner.aggregateRefreshID == refreshID else { return } + owner.aggregateRefreshWorkers.values.forEach { $0.task.cancel() } + owner.aggregateRefreshWorkers.removeAll() + owner.aggregatePublishTask?.cancel() + owner.aggregatePublishTask = nil + } + + func owns(_ environment: Environment) -> Bool { + isCurrent && owner?.aggregateRefreshWorkers[environment.id]?.environment == environment + && owner?.activeEnvironment?.id != environment.id + } + + func reconcileWorkers(_ environments: [Environment]) -> Duration { + guard let owner, isCurrent else { return idleInterval } + savedEnvironments = environments + let passive = environments.filter { $0.isEnabled && $0.id != owner.activeEnvironment?.id } + let current = Dictionary(uniqueKeysWithValues: passive.map { ($0.id, $0) }) + for (id, worker) in owner.aggregateRefreshWorkers where current[id] != worker.environment { + worker.task.cancel() + owner.aggregateRefreshWorkers[id] = nil + } + // Prune once per topology check, not once per shell result. + owner.reconcileEnvironmentLoads([], savedEnvironments: environments) + if owner.latestSnapshot?.environments != environments.map({ + owner.mapEnvironment($0, activeID: owner.activeEnvironment?.id) + }) { + owner.publishAggregateSnapshot(environments) + } + for environment in passive where owner.aggregateRefreshWorkers[environment.id] == nil { + let state = PassiveShellState() + state.needsHTTP = refreshImmediately || owner.shellsByEnvironmentID[environment.id] == nil + let task = Task { + await withTaskGroup(of: Void.self) { group in + group.addTask { await self.refreshShell(environment, state: state) } + group.addTask { await self.followShell(environment, state: state) } + group.addTask { await self.refreshCatalogue(environment) } + } } - let environments: [Environment] + owner.aggregateRefreshWorkers[environment.id] = (environment, task) + } + if let dirtyID = dirtyEnvironmentIDs.first { schedulePublication(dirtyID) } + return passive.isEmpty ? idleInterval : fastInterval + } + + /// Revalidate membership after suspension, including endpoint edits. + /// Persistence errors retry; removed or disabled peers end their work. + func currentEnvironments(for environment: Environment) async throws -> [Environment]? { + guard isCurrent else { return nil } + let environments = try await runtime.environments() + guard isCurrent, environments.contains(environment), environment.isEnabled, + owner?.activeEnvironment?.id != environment.id else { return nil } + return environments + } + + func refreshShell(_ environment: Environment, state: PassiveShellState) async { + var interval = fastInterval + while owns(environment) { do { - environments = try await loadEnvironments(self.runtime) + if state.isLive { await state.waitForRepair() } + if !state.needsHTTP { + let delay = interval + await withTaskGroup(of: Void.self) { group in + group.addTask { try? await self.peerSleep(environment.id, delay) } + group.addTask { await state.waitForRepair() } + _ = await group.next() + group.cancelAll() + } + } + guard owns(environment) else { return } + state.needsHTTP = false + guard !state.isLive else { continue } + guard try await currentEnvironments(for: environment) != nil else { return } + let client = await runtime.client(for: environment) + let epoch = state.epoch + let authority = state.authorityRevision + let connectionID = await client.currentConnectionID() + guard owns(environment), epoch == state.epoch, + authority == state.authorityRevision else { continue } + let shell: OrchestrationShellSnapshot? + var hydrationError: (any Error)? + do { shell = try await client.shellSnapshot(timeoutInterval: shellTimeout) } + catch { shell = nil; hydrationError = error } + defer { refreshReceipt(.httpFinished(environmentID: environment.id)) } + let currentConnection = await client.currentConnectionID() + guard let environments = try await currentEnvironments(for: environment) else { return } + guard owns(environment), epoch == state.epoch, + authority == state.authorityRevision, connectionID == currentConnection, + !state.isLive else { continue } + if let error = hydrationError, error.isRejectedAuthorization { + owner?.applyEnvironmentLoad(EnvironmentShellLoad( + environment: environment, client: client, shell: nil, config: nil, + credentialRejected: true, failureDetail: error.localizedDescription + )) + owner?.publishAggregateSnapshot(environments) + interval = .seconds(24 * 60 * 60) + continue + } + // A validated HTTP snapshot remains in this socket's cache + // epoch without claiming that the live stream is complete. + if shell != nil { owner?.shellConnectionIDsByEnvironmentID[environment.id] = connectionID } + // The first authoritative response in this subscription epoch + // may reset a cache left over from a restarted server. + interval = applyShell( + shell, client: client, environment: environment, saved: environments, + replacingEpoch: state.cacheEpoch != epoch + ) + if shell != nil { state.cacheEpoch = epoch } } catch is CancellationError where Task.isCancelled { return } catch { - // Persistence can be briefly unavailable while another - // actor atomically replaces the environment document. - // Back off while keeping the loop alive for recovery. - nextInterval = failureInterval - continue + interval = failureInterval } - guard !Task.isCancelled, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + } + } + + func followShell(_ environment: Environment, state: PassiveShellState) async { + while owns(environment) { + do { + guard try await currentEnvironments(for: environment) != nil else { return } + let client = await runtime.client(for: environment) + let subscription = try await client.shellEventsOnCurrentConnection() + guard try await currentEnvironments(for: environment) != nil, + owns(environment) else { return } + if state.connectionID != subscription.connectionID { + let needsRepair = state.connectionID != nil + || owner?.shellsByEnvironmentID[environment.id] == nil + state.connectionID = subscription.connectionID + state.epoch = UUID() + // A fresh bootstrap already read this peer. Its first + // socket attempt need not duplicate that HTTP request. + if needsRepair { state.requestRepair() } + } + let epoch = state.epoch + state.isLive = false + for try await item in subscription.events { + let connectionID = await client.currentConnectionID() + guard owns(environment), epoch == state.epoch, + connectionID == subscription.connectionID else { break } + guard applyStream(item, client: client, environment: environment, state: state) + else { break } + } + } catch is CancellationError where Task.isCancelled { return + } catch { + // HTTP recovery belongs to this peer's single fallback loop. } - let passiveEnvironments = environments.filter { - $0.isEnabled && $0.id != activeEnvironment.id - } - guard !passiveEnvironments.isEmpty else { - nextInterval = idleInterval - continue - } - let passiveIDs = Set(passiveEnvironments.map(\.id)) - failureBackoffs = failureBackoffs.reduce(into: [:]) { result, entry in - guard passiveIDs.contains(entry.key) else { return } - result[entry.key] = max(.zero, entry.value - elapsedInterval) + guard owns(environment) else { return } + state.isLive = false + // A read begun before the stream lost completeness cannot + // repair that gap, even if the socket identity is unchanged. + state.authorityRevision &+= 1 + markStreamPaused(environment) + state.requestRepair() + do { try await streamRetrySleep(environment.id, failureInterval) } catch { return } + } + } + + func markStreamPaused(_ environment: Environment) { + guard let owner, owns(environment) else { return } + owner.shellConnectionIDsByEnvironmentID[environment.id] = nil + owner.environmentConnectionStates[environment.id] = .reconnecting + owner.environmentConnectionDetails[environment.id] = "Live updates paused. Refreshing over HTTP." + schedulePublication(environment.id) + } + + func applyStream( + _ item: ShellStreamItem, client: T3Client, environment: Environment, state: PassiveShellState + ) -> Bool { + guard let owner, owns(environment) else { return false } + let current = owner.shellsByEnvironmentID[environment.id] + let shell: OrchestrationShellSnapshot + switch item { + case let .snapshot(snapshot): + if state.cacheEpoch == state.epoch, let current, + current.snapshotSequence > snapshot.snapshotSequence { + shell = current + } else { + shell = snapshot } - let refreshableEnvironments = passiveEnvironments.filter { - failureBackoffs[$0.id, default: .zero] <= .zero + let remainingIDs = Set(shell.threads.map(\.id)) + for removed in current?.threads ?? [] where !remainingIDs.contains(removed.id) { + owner.clearRemovedThreadDetail(FeatureScopedID.thread( + environmentID: environment.id, wireID: removed.id + )) } - guard !refreshableEnvironments.isEmpty else { - nextInterval = fastInterval - continue + owner.shellConnectionIDsByEnvironmentID[environment.id] = state.connectionID + state.authorityRevision &+= 1 + state.cacheEpoch = state.epoch + state.isLive = true + state.needsHTTP = false + case .synchronized: + // A completion marker cannot make an arbitrary cached shell authoritative. + return state.isLive + case .refreshRequired: + return false + case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: + guard state.isLive, let current else { return false } + var projects = current.projects + var threads = current.threads + let sequence: Int + switch item { + case let .projectUpserted(nextSequence, project): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + if let index = projects.firstIndex(where: { $0.id == project.id }) { + projects[index] = project + } else { projects.append(project) } + case let .projectRemoved(nextSequence, id): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + projects.removeAll { $0.id == id } + case let .threadUpserted(nextSequence, thread): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + owner.archivedThreadsByEnvironmentID[environment.id]?.removeAll { + ($0.wireID ?? $0.id) == thread.id + } + if let index = threads.firstIndex(where: { $0.id == thread.id }) { + threads[index] = thread + } else { threads.append(thread) } + case let .threadRemoved(nextSequence, id): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + threads.removeAll { $0.id == id } + owner.clearRemovedThreadDetail(FeatureScopedID.thread( + environmentID: environment.id, wireID: id + )) + default: + return false } - let loads = await self.loadEnvironmentShells(refreshableEnvironments) - guard !Task.isCancelled, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + shell = OrchestrationShellSnapshot( + snapshotSequence: sequence, projects: projects, threads: threads, updatedAt: current.updatedAt + ) + } + let membershipChanged = current?.projects.map(\.id) != shell.projects.map(\.id) + || current?.threads.map(\.id) != shell.threads.map(\.id) + owner.shellsByEnvironmentID[environment.id] = shell + owner.environmentClients[environment.id] = client + owner.environmentConnectionStates[environment.id] = .connected + owner.environmentConnectionDetails[environment.id] = nil + if membershipChanged { owner.rebuildEntityIndexes(savedEnvironments) } + owner.synchronizeActiveDetail(with: shell, environment: environment) + refreshReceipt(.shellApplied(environmentID: environment.id, sequence: shell.snapshotSequence)) + schedulePublication(environment.id) + return true + } + + func schedulePublication(_ environmentID: String) { + guard let owner, isCurrent else { return } + dirtyEnvironmentIDs.insert(environmentID) + guard owner.aggregatePublishTask == nil else { return } + owner.aggregatePublishTask = Task { + do { try await self.publishSleep() } catch { return } + let environments: [Environment] + do { environments = try await self.runtime.environments() } + catch { + if self.isCurrent { self.owner?.aggregatePublishTask = nil } return } - let shellsChanged = loads.contains { load in - guard let shell = load.shell else { return false } - return shell != self.shellsByEnvironmentID[load.environment.id] - } - let hasActiveWork = loads.contains { load in - load.shell.map(Self.shellNeedsFrequentAggregateRefresh) == true + guard let owner = self.owner, self.isCurrent else { return } + self.savedEnvironments = environments + owner.reconcileEnvironmentLoads([], savedEnvironments: environments) + let dirty = self.dirtyEnvironmentIDs.filter { + owner.aggregateRefreshWorkers[$0] != nil } - for load in loads { - if load.credentialRejected { - // Only re-pairing changes this. Re-pairing replaces - // the runtime client and restarts this loop. - failureBackoffs[load.environment.id] = .seconds(24 * 60 * 60) - } else if load.shell == nil { - failureBackoffs[load.environment.id] = failureInterval - } else { - failureBackoffs[load.environment.id] = nil + self.dirtyEnvironmentIDs.removeAll() + owner.aggregatePublishTask = nil + guard !dirty.isEmpty else { return } + owner.publishAggregateSnapshot(self.savedEnvironments, changedEnvironmentIDs: dirty) + } + } + + func applyShell( + _ shell: OrchestrationShellSnapshot?, client: T3Client, + environment: Environment, saved: [Environment], replacingEpoch: Bool = false + ) -> Duration { + guard let owner, isCurrent else { return failureInterval } + let previous = owner.shellsByEnvironmentID[environment.id] + let changed = shell.map { $0 != previous } ?? false + let membershipChanged = shell.map { + $0.projects.map(\.id) != previous?.projects.map(\.id) + || $0.threads.map(\.id) != previous?.threads.map(\.id) + } ?? false + if replacingEpoch, let shell { owner.shellsByEnvironmentID[environment.id] = shell } + owner.applyEnvironmentLoad(EnvironmentShellLoad( + environment: environment, client: client, shell: shell, config: nil + )) + if membershipChanged { owner.rebuildEntityIndexes(saved) } + owner.publishAggregateSnapshot(saved, changedEnvironmentIDs: [environment.id]) + guard let shell else { return failureInterval } + return changed || NativeFeatureClient.shellNeedsFrequentAggregateRefresh(shell) + ? fastInterval : idleInterval + } + + func refreshCatalogue(_ environment: Environment) async { + while isCurrent && owner?.serverConfigsByEnvironmentID[environment.id] == nil { + do { + guard try await currentEnvironments(for: environment) != nil else { return } + // Never disconnect the shared client if this peer becomes selected. + let probe = await runtime.ephemeralClient(for: environment) + let config = try? await probe.serverConfig() + await probe.disconnect() + guard let environments = try await currentEnvironments(for: environment) else { return } + if let config { + applyCatalogue(config, environment: environment, saved: environments) + return } + } catch is CancellationError where Task.isCancelled { + return + } catch { + // Retry a transient environment-document read just like a failed probe. } - self.reconcileEnvironmentLoads(loads, savedEnvironments: environments) - let currentConnection = self.latestSnapshot?.connection - ?? FeatureConnection( - state: .disconnected, - environmentName: activeEnvironment.label, - endpoint: activeEnvironment.httpBaseURL.absoluteString - ) - let snapshot = self.makeSnapshot( - environments: environments, - activeEnvironment: activeEnvironment, - connectionState: currentConnection.state, - connectionDetail: currentConnection.detail - ) - self.publish(snapshot) - if shellsChanged || hasActiveWork { - nextInterval = fastInterval - } else { - nextInterval = idleInterval + do { try await Task.sleep(for: failureInterval) } catch { return } + } + } + + func applyCatalogue(_ config: ServerConfigSnapshot, environment: Environment, saved: [Environment]) { + guard let owner, isCurrent else { return } + owner.setServerConfig(config, environmentID: environment.id) + owner.publishAggregateSnapshot(saved, changedEnvironmentIDs: [environment.id]) + } + } + + @MainActor + private final class PassiveShellState { + var epoch = UUID() + var connectionID: UUID? + var cacheEpoch: UUID? + var authorityRevision = 0 + var isLive = false + var needsHTTP = false + private var waiter: (id: UUID, continuation: CheckedContinuation)? + + func requestRepair() { + needsHTTP = true + waiter?.continuation.resume() + waiter = nil + } + + func waitForRepair() async { + guard !needsHTTP else { return } + let id = UUID() + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if Task.isCancelled || needsHTTP { continuation.resume() } + else { waiter = (id, continuation) } + } + } onCancel: { + Task { @MainActor [weak self] in + guard self?.waiter?.id == id else { return } + self?.waiter?.continuation.resume() + self?.waiter = nil } } } } + private func publishAggregateSnapshot( + _ environments: [Environment], changedEnvironmentIDs: Set? = nil + ) { + guard let activeEnvironment else { return } + let connection = latestSnapshot?.connection + publish(makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: connection?.state ?? .disconnected, + connectionDetail: connection?.detail, + changedEnvironmentIDs: changedEnvironmentIDs + )) + } + nonisolated private static func shellNeedsFrequentAggregateRefresh( _ shell: OrchestrationShellSnapshot ) -> Bool { @@ -4243,35 +4728,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } - /// HTTP fallback refreshes data while preserving the socket's reconnecting - /// state. The generation travels through the awaited snapshot publish so a - /// task from a previous environment session cannot publish late results. - private func consumeFallbackShell( - shell: OrchestrationShellSnapshot, - client: T3Client, - generation: Int - ) async { - guard isCurrentSession(client: client, generation: generation), - shell.snapshotSequence >= (latestShell?.snapshotSequence ?? .min) else { - return - } - shellPublishTask?.cancel() - shellPublishTask = nil - latestShell = shell - shellsByEnvironmentID[client.environment.id] = shell - await emitSnapshot( - shell, - client: client, - expectedGeneration: generation, - markSourceConnected: false - ) - guard isCurrentSession(client: client, generation: generation), - let threadID = activeThreadID else { - return - } - scheduleDetailRefresh(threadID: threadID, client: client) - } - private func consume(deltas: [ShellStreamItem], client: T3Client, generation: Int) async { guard !deltas.isEmpty, !Task.isCancelled, isCurrentSession(client: client, generation: generation) else { return } @@ -4333,20 +4789,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, changedThreadIDs.insert(uiThreadID) shouldRefreshArchived = true threads.removeAll { $0.id == threadID } - latestDetails[uiThreadID] = nil - detailRenderCaches[uiThreadID] = nil - detailCacheRecency.removeAll { $0 == uiThreadID } - if activeThreadID == uiThreadID { - resetDetailRefresh() - resetDetailStream() - activeThreadID = nil - activeThreadEnvironmentID = nil - activeRawThread = nil - activeThreadSequence = nil - activeThreadPage = nil - threadHistoryEpoch &+= 1 - pendingOlderThreadPage = nil - } + clearRemovedThreadDetail(uiThreadID) case .snapshot, .synchronized, .refreshRequired: continue } @@ -4372,6 +4815,23 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } + private func clearRemovedThreadDetail(_ uiThreadID: String) { + latestDetails[uiThreadID] = nil + detailRenderCaches[uiThreadID] = nil + detailCacheRecency.removeAll { $0 == uiThreadID } + if activeThreadID == uiThreadID { + resetDetailRefresh() + resetDetailStream() + activeThreadID = nil + activeThreadEnvironmentID = nil + activeRawThread = nil + activeThreadSequence = nil + activeThreadPage = nil + threadHistoryEpoch &+= 1 + pendingOlderThreadPage = nil + } + } + /// Shell streams can emit many metadata updates during one provider turn. /// Home only needs the newest row state, so publish at most four times per /// second while the selected transcript continues on its dedicated stream. @@ -4939,6 +5399,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) { let savedIDs = Set(savedEnvironments.map(\.id)) environmentClients = environmentClients.filter { savedIDs.contains($0.key) } + shellConnectionIDsByEnvironmentID = shellConnectionIDsByEnvironmentID.filter { savedIDs.contains($0.key) } shellsByEnvironmentID = shellsByEnvironmentID.filter { savedIDs.contains($0.key) } shellProjectionCache = shellProjectionCache.filter { savedIDs.contains($0.key) } serverConfigsByEnvironmentID = serverConfigsByEnvironmentID.filter { @@ -4960,31 +5421,33 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, savedIDs.contains($0.key) } - for load in loads { - environmentClients[load.environment.id] = load.client - if let config = load.config { - setServerConfig(config, environmentID: load.environment.id) - if load.environment.id == activeEnvironment?.id { - latestServerConfig = config - } + for load in loads { applyEnvironmentLoad(load) } + rebuildEntityIndexes(savedEnvironments) + } + + private func applyEnvironmentLoad(_ load: EnvironmentShellLoad) { + environmentClients[load.environment.id] = load.client + if let config = load.config { + setServerConfig(config, environmentID: load.environment.id) + if load.environment.id == activeEnvironment?.id { + latestServerConfig = config } - if let shell = load.shell { - if shell.snapshotSequence - >= (shellsByEnvironmentID[load.environment.id]?.snapshotSequence ?? .min) { - shellsByEnvironmentID[load.environment.id] = shell - } - environmentConnectionStates[load.environment.id] = .connected - environmentConnectionDetails[load.environment.id] = nil - } else if load.credentialRejected && load.environment.kind != .managedDPoP { - environmentConnectionStates[load.environment.id] = .needsPairing - environmentConnectionDetails[load.environment.id] = Self.needsPairingDetail - } else { - environmentConnectionStates[load.environment.id] = .disconnected - environmentConnectionDetails[load.environment.id] = - load.failureDetail ?? "That server is currently unreachable." + } + if let shell = load.shell { + if shell.snapshotSequence + >= (shellsByEnvironmentID[load.environment.id]?.snapshotSequence ?? .min) { + shellsByEnvironmentID[load.environment.id] = shell } + environmentConnectionStates[load.environment.id] = .connected + environmentConnectionDetails[load.environment.id] = nil + } else if load.credentialRejected && load.environment.kind != .managedDPoP { + environmentConnectionStates[load.environment.id] = .needsPairing + environmentConnectionDetails[load.environment.id] = Self.needsPairingDetail + } else { + environmentConnectionStates[load.environment.id] = .disconnected + environmentConnectionDetails[load.environment.id] = + load.failureDetail ?? "That server is currently unreachable." } - rebuildEntityIndexes(savedEnvironments) } static let needsPairingDetail = "This computer no longer accepts the saved pairing. Pair again." @@ -4999,7 +5462,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pollingTask = nil fallbackPollingTask = nil configurationTask = nil - lastShellEventAt = nil if activeEnvironment?.kind == .managedDPoP { emitConnection(.disconnected, detail: detail) } else { @@ -5132,45 +5594,60 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, indexedProvisionalRoutes = provisionalThreadRoutes } + /// A missing expectation means an ordinary publication. A present value + /// retains even an absent source cache or socket as an exact expectation. + private struct RefreshPublicationGuard { + let bootstrapID: UUID + let sourceSequence: Int? + let connectionID: UUID? + let activeAuthorityRevision: Int? + } + private func refresh(client: T3Client, includeArchived: Bool = false) async throws { let environment = client.environment let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + let sourceSequence = shellsByEnvironmentID[environment.id]?.snapshotSequence + let authorityRevision = activeHTTPAuthorityRevision + // Archive RPC can establish the first socket. Bind the subsequent shell + // read to that socket instead of treating its creation as replacement. + let archivedShell = includeArchived ? try? await client.archivedShellSnapshot() : nil + let connectionID = await client.currentConnectionID() + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID else { throw CancellationError() } let shell = try await client.shellSnapshot() - // A cancelled refresh (for example, of a thread deleted meanwhile) must not - // write a shell that was read before the change. - guard !Task.isCancelled, + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, isKnownClient(client, environmentID: environment.id, generation: generation) else { throw CancellationError() } - guard shell.snapshotSequence - >= (shellsByEnvironmentID[environment.id]?.snapshotSequence ?? .min) else { - return - } - shellsByEnvironmentID[environment.id] = shell - if activeEnvironment?.id == environment.id { - latestShell = shell - } - if includeArchived, - let archivedShell = try? await client.archivedShellSnapshot(), - isKnownClient(client, environmentID: environment.id, generation: generation) { - archivedThreadsByEnvironmentID[environment.id] = archivedShell.threads.map { - mapThread($0, environment: environment) - } - archivedShellThreadsByEnvironmentID[environment.id] = Dictionary( - uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } - ) + let currentConnectionID = await client.currentConnectionID() + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, + isKnownClient(client, environmentID: environment.id, generation: generation), + connectionID == currentConnectionID, + sourceSequence == shellsByEnvironmentID[environment.id]?.snapshotSequence, + activeEnvironment?.id != environment.id || authorityRevision == activeHTTPAuthorityRevision else { + throw CancellationError() } - await emitSnapshot(shell, client: client, expectedGeneration: generation) + guard shell.snapshotSequence + >= (shellsByEnvironmentID[environment.id]?.snapshotSequence ?? .min) else { return } + await emitSnapshot( + shell, client: client, expectedGeneration: generation, + refreshGuard: RefreshPublicationGuard( + bootstrapID: bootstrapID, sourceSequence: sourceSequence, connectionID: connectionID, + activeAuthorityRevision: activeEnvironment?.id == environment.id ? authorityRevision : nil + ), archivedShell: archivedShell + ) } private func scheduleArchivedRefresh(client: T3Client, environment: Environment) { archivedRefreshTask?.cancel() let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID archivedRefreshTask = Task { [weak self] in - guard let self, - let archivedShell = try? await client.archivedShellSnapshot(), + guard let self else { return } + defer { self.aggregateRefreshReceipt(.archiveFinished(environmentID: environment.id)) } + guard let archivedShell = try? await client.archivedShellSnapshot(), !Task.isCancelled, - self.isCurrentSession(client: client, generation: generation) else { + self.isCurrentForeground(client, generation: generation, bootstrapID: bootstrapID) else { return } self.archivedThreadsByEnvironmentID[environment.id] = archivedShell.threads.map { @@ -5180,7 +5657,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } ) if let shell = self.latestShell { - await self.emitSnapshot(shell, client: client, expectedGeneration: generation) + await self.emitSnapshot(shell, client: client, expectedGeneration: generation, markSourceConnected: false) } } } @@ -5276,16 +5753,32 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, _ shell: OrchestrationShellSnapshot, client sourceClient: T3Client, expectedGeneration: Int, + refreshGuard: RefreshPublicationGuard? = nil, + archivedShell: OrchestrationShellSnapshot? = nil, markSourceConnected: Bool = true ) async { let sourceEnvironment = sourceClient.environment guard !Task.isCancelled, + refreshGuard.map({ + $0.bootstrapID == foregroundBootstrapID + && $0.sourceSequence == shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence + && ($0.activeAuthorityRevision.map { $0 == activeHTTPAuthorityRevision } ?? true) + }) ?? true, let environment = activeEnvironment, isKnownClient( sourceClient, environmentID: sourceEnvironment.id, generation: expectedGeneration ) else { return } let environments = (try? await runtime.environments()) ?? [environment] + // The metadata await can outlive the socket too. Read its identity last, + // then validate all expectations together before any cache mutation. + let currentConnectionID = refreshGuard == nil ? nil : await sourceClient.currentConnectionID() guard !Task.isCancelled, + refreshGuard.map({ $0.connectionID == currentConnectionID }) ?? true, + refreshGuard.map({ + $0.bootstrapID == foregroundBootstrapID + && $0.sourceSequence == shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence + && ($0.activeAuthorityRevision.map { $0 == activeHTTPAuthorityRevision } ?? true) + }) ?? true, isKnownClient( sourceClient, environmentID: sourceEnvironment.id, generation: expectedGeneration ), @@ -5295,6 +5788,17 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, >= (shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence ?? .min) else { return } + if let archivedShell { + archivedThreadsByEnvironmentID[sourceEnvironment.id] = archivedShell.threads.map { + mapThread($0, environment: sourceEnvironment) + } + archivedShellThreadsByEnvironmentID[sourceEnvironment.id] = Dictionary( + uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } + ) + } + if let refreshGuard { + shellConnectionIDsByEnvironmentID[sourceEnvironment.id] = refreshGuard.connectionID + } shellsByEnvironmentID[sourceEnvironment.id] = shell if markSourceConnected { environmentConnectionStates[sourceEnvironment.id] = .connected @@ -5302,6 +5806,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } if sourceEnvironment.id == environment.id { latestShell = shell + if markSourceConnected { + activeHasHydrated = true + activeShellEpochHasSnapshot = true + activeHTTPAuthorityRevision &+= 1 + } } rebuildEntityIndexes(environments) synchronizeActiveDetail( @@ -5311,8 +5820,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let connectionState: FeatureConnection.State let connectionDetail: String? if sourceEnvironment.id == environment.id, markSourceConnected { - connectionState = .connected - connectionDetail = nil + connectionState = activeStreamIsAuthoritative ? .connected : .reconnecting + connectionDetail = activeStreamIsAuthoritative ? nil + : latestSnapshot?.connection.detail ?? "Live updates reconnecting. Refreshing over HTTP." } else { connectionState = latestSnapshot?.connection.state ?? environmentConnectionStates[environment.id] @@ -5649,14 +6159,29 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environments: [Environment], activeEnvironment: Environment, connectionState: FeatureConnection.State, - connectionDetail: String? = nil + connectionDetail: String? = nil, + changedEnvironmentIDs: Set? = nil ) -> FeatureSnapshot { let enabledEnvironments = environments.filter(\.isEnabled) let enabledIDs = Set(enabledEnvironments.map(\.id)) shellProjectionCache = shellProjectionCache.filter { enabledIDs.contains($0.key) } var threads: [FeatureThread] = [] var projects: [FeatureProject] = [] + let previousThreads = changedEnvironmentIDs == nil ? [:] + : Dictionary(grouping: latestSnapshot?.threads ?? [], by: \.environmentID) + let previousProjects = changedEnvironmentIDs == nil ? [:] + : Dictionary(grouping: latestSnapshot?.projects ?? [], by: \.environmentID) + let previousEnvironments = Dictionary(uniqueKeysWithValues: + (latestSnapshot?.environments ?? []).map { ($0.id, $0) } + ) for environment in enabledEnvironments { + if let changedEnvironmentIDs, !changedEnvironmentIDs.contains(environment.id), + previousEnvironments[environment.id] + == mapEnvironment(environment, activeID: activeEnvironment.id) { + threads.append(contentsOf: previousThreads[environment.id] ?? []) + projects.append(contentsOf: previousProjects[environment.id] ?? []) + continue + } // Take ownership while updating so the cache does not copy its // retained arrays when one row changes. var projection = shellProjectionCache.removeValue(forKey: environment.id) @@ -5736,6 +6261,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let providersByEnvironment = enabledEnvironments.reduce( into: [String: [FeatureProvider]]() ) { catalogues, environment in + if let changedEnvironmentIDs, !changedEnvironmentIDs.contains(environment.id), + let previous = latestSnapshot?.providersByEnvironment?[environment.id] { + catalogues[environment.id] = previous + return + } guard let shell = shellsByEnvironmentID[environment.id] else { return } catalogues[environment.id] = mapProviders( environmentID: environment.id, @@ -6835,6 +7365,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, providerCatalogCache[environmentID] = nil } serverConfigsByEnvironmentID[environmentID] = config + aggregateRefreshReceipt(.configurationApplied(environmentID: environmentID)) } private func mapProviders( @@ -8628,3 +9159,11 @@ private enum NativeFeatureClientError: LocalizedError { } } } + +/// Deterministic receipts at the passive transport/publication boundary. +enum NativePassiveShellReceipt: Sendable { + case shellApplied(environmentID: String, sequence: Int) + case httpFinished(environmentID: String) + case configurationApplied(environmentID: String) + case archiveFinished(environmentID: String) +} diff --git a/apps/swift-ios/Core/T3Client.swift b/apps/swift-ios/Core/T3Client.swift index 0c8659ac4e58..902c5fa081e5 100644 --- a/apps/swift-ios/Core/T3Client.swift +++ b/apps/swift-ios/Core/T3Client.swift @@ -638,6 +638,29 @@ public actor T3Client { ) } + /// Passive shells start with a full snapshot on each socket subscription. + /// Registering its identity atomically prevents late socket results from + /// acquiring authority over a replacement connection. + public func shellEventsOnCurrentConnection() async throws -> ( + events: AsyncThrowingStream, connectionID: UUID + ) { + try await rpc.subscribeOnCurrentConnection( + RPCMethod.subscribeShell.rawValue, + payload: .object(["requestCompletionMarker": .bool(true)]), + as: ShellStreamItem.self + ) + } + + public func shellEventBatchesOnCurrentConnection() async throws -> ( + events: AsyncThrowingStream<[ShellStreamItem], Error>, connectionID: UUID + ) { + try await rpc.subscribeBatchesOnCurrentConnection( + RPCMethod.subscribeShell.rawValue, + payload: .object(["requestCompletionMarker": .bool(true)]), + as: ShellStreamItem.self + ) + } + public func threadEventBatches( threadID: String, after sequence: Int? = nil, diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index 1ff9a34c2b96..c9fb47fc99d0 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -153,6 +153,7 @@ public final class FeatureRootModel { func applicationDidEnterBackground(at date: Date = .now) { backgroundedAt = date + client.suspendForBackground() } func applicationDidBecomeActive(at date: Date = .now) async { diff --git a/apps/swift-ios/Features/Shared/FeatureClient.swift b/apps/swift-ios/Features/Shared/FeatureClient.swift index 5b791cca92cf..8a99278d612a 100644 --- a/apps/swift-ios/Features/Shared/FeatureClient.swift +++ b/apps/swift-ios/Features/Shared/FeatureClient.swift @@ -10,6 +10,7 @@ public protocol FeatureClient: AnyObject { func backgroundSnapshot() async throws -> FeatureSnapshot func events() -> AsyncStream func resumeAfterBackground(reconnect: Bool) async + func suspendForBackground() func preuploadAttachment( _ attachment: FeatureUploadAttachment, @@ -314,6 +315,8 @@ public extension FeatureClient { func resumeAfterBackground(reconnect: Bool) async {} + func suspendForBackground() {} + func preuploadAttachment( _ attachment: FeatureUploadAttachment, environmentID: String diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 66fe2aa236a4..7c3aacb4ec9f 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -287,6 +287,7 @@ struct FeatureRootModelTests { await model.applicationDidBecomeActive(at: start.addingTimeInterval(10)) await model.applicationDidBecomeActive(at: start.addingTimeInterval(11)) #expect(client.foregroundReconnects == [false, true]) + #expect(client.backgroundSuspends == 2) } @Test @@ -3924,6 +3925,8 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { _ = try await rewindHandler(threadID, messageID) } var foregroundReconnects: [Bool] = [] + var backgroundSuspends = 0 + func suspendForBackground() { backgroundSuspends += 1 } func resumeAfterBackground(reconnect: Bool) async { foregroundReconnects.append(reconnect) } private let eventStream: AsyncStream private let eventContinuation: AsyncStream.Continuation @@ -4125,7 +4128,7 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { selection: FeatureSelection?, runtimeMode: FeatureRuntimeMode, attachments _: [FeatureUploadAttachment], - identity _: FeatureSubmissionIdentity, + identity: FeatureSubmissionIdentity, context: OrchestrationMessageContext? = nil ) async throws { sentRuntimeModes.append(runtimeMode) diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index cc188815bb14..7b48469a2790 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -1,4 +1,6 @@ import Foundation +import Darwin +import Observation import Testing import XCTest @testable import T3Code @@ -24,7 +26,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { projectID: "project-shared", threadID: "thread-shared", title: "Remote files", workspaceRoot: #"C:\work\project"#, worktreePath: #"D:\worktrees\feature"# ), host: "two.example") - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let root = try await fixture.client.listFiles(threadID: thread.id, path: nil) @@ -56,7 +58,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { rpcConnectionWaitTimeout: .seconds(2) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "one" }) let root = try await fixture.client.listFiles(threadID: thread.id, path: nil) @@ -87,17 +89,19 @@ final class NativeMultiEnvironmentTests: XCTestCase { ])]] ) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) XCTAssertEqual(project.defaultSelection?.modelID, environmentDefault.model) _ = try await fixture.client.createThread(projectID: project.id, title: "Task", selection: nil) - let creates = await fixture.transport.dispatchRecords().filter { $0.command["type"] == .string("thread.create") } - XCTAssertEqual(creates.last?.command["modelSelection"], try JSONValue.encode(environmentDefault)) + // The configured peer keeps its socket, so the command crosses it rather than HTTP. + let creates = await server.dispatchedCommands(host: "two.example").filter { $0["type"] == .string("thread.create") } + XCTAssertEqual(creates.last?["modelSelection"], try JSONValue.encode(environmentDefault)) await fixture.client.disconnect() } @@ -117,12 +121,13 @@ final class NativeMultiEnvironmentTests: XCTestCase { ]] ) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) XCTAssertEqual(project.defaultSelection?.modelID, environmentDefault.model) XCTAssertEqual(project.supportsProjectSettingsOverrides, true) @@ -139,7 +144,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .responseStreamingMode, value: .string("turn") )) - let updated = try await fixture.client.initialSnapshot() + let updated = try await fixture.hydratedSnapshot() let updatedProject = try XCTUnwrap(updated.projects.first { $0.id == project.id }) XCTAssertEqual(updatedProject.defaultSelection?.modelID, projectDefault.model) XCTAssertEqual(updatedProject.defaultWorkspaceMode, .worktree) @@ -162,12 +167,12 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .defaultModelSelection, value: .null )) - let noDefault = try await fixture.client.initialSnapshot() + let noDefault = try await fixture.hydratedSnapshot() XCTAssertNil(noDefault.projects.first { $0.id == project.id }?.defaultSelection) try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .defaultModelSelection, value: nil )) - let inherited = try await fixture.client.initialSnapshot() + let inherited = try await fixture.hydratedSnapshot() XCTAssertEqual(inherited.projects.first { $0.id == project.id }?.defaultSelection?.modelID, environmentDefault.model) await fixture.client.disconnect() @@ -194,7 +199,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { snapshotSequence: source.snapshotSequence, projects: [wireProject], threads: source.threads, updatedAt: source.updatedAt ), host: "two.example") - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) let preferences = try await fixture.client.projectPreferences(projectID: project.id) XCTAssertEqual(preferences.environment.defaultModelSelection?.model, "environment-model") @@ -212,7 +217,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) do { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( @@ -325,7 +330,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(Set(snapshot.projects.map(\.environmentID)), ["one", "two"]) XCTAssertEqual(Set(snapshot.threads.compactMap(\.wireID)), ["thread-one", "thread-two"]) @@ -383,19 +388,19 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveProviderRefreshKeepsActiveThreadsAndAcceptsTheirNextSequence() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let providers = try await fixture.client.refreshProviders(environmentID: "two") XCTAssertEqual(providers.map(\.id), ["codex-two.example"]) - let refreshed = try await fixture.client.initialSnapshot() + let refreshed = try await fixture.hydratedSnapshot() XCTAssertEqual( refreshed.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -421,7 +426,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let updated = try await fixture.client.initialSnapshot() + let updated = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(updated.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID)), ["thread-one", "thread-new"] @@ -435,20 +440,20 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveEnvironmentSettingsDoNotReplaceActiveThreads() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() try await fixture.client.updateServerPreferences( environmentID: "two", change: .environmentIcon("mac-mini") ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -465,6 +470,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testMachineModelDefaultsRefreshCachedProjectsWithoutChangingOtherEnvironments() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), @@ -479,7 +485,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { snapshotSequence: source.snapshotSequence, projects: [project], threads: source.threads, updatedAt: source.updatedAt ), host: "two.example") - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() XCTAssertNil(initial.projects.first { $0.environmentID == "two" }?.defaultSelection) let localDefault = initial.projects.first { $0.environmentID == "one" }?.defaultSelection @@ -488,13 +494,13 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateServerPreferences(environmentID: "two", change: .sharedPreferences(.object([ "defaultModelSelection": try JSONValue.encode(selection), ]))) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.projects.first { $0.environmentID == "two" }?.defaultSelection?.modelID, model) XCTAssertEqual(snapshot.projects.first { $0.environmentID == "one" }?.defaultSelection, localDefault) } await fixture.transport.setShell(source, host: "two.example") - let overridden = try await fixture.client.initialSnapshot() + let overridden = try await fixture.hydratedSnapshot() XCTAssertEqual(overridden.projects.first { $0.environmentID == "two" }?.defaultSelection?.modelID, "gpt-5.6-sol") await fixture.client.disconnect() } @@ -502,20 +508,20 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testSharedSettingsFanOutDoesNotReplaceActiveThreads() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() try await fixture.client.updateServerPreferences( environmentID: "one", change: .defaultThreadEnvMode(.worktree) ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -532,13 +538,14 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testRestartPreferenceOnlyReachesComputersThatSupportIt() async throws { let server = MultiEnvironmentConfigurationServer(restartSupportHosts: ["one.example"]) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.preferencesByEnvironment?["one"]?.continueThreadsAfterServerUpdate, false) XCTAssertNil(snapshot.preferencesByEnvironment?["two"]?.continueThreadsAfterServerUpdate) @@ -583,7 +590,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap( snapshot.threads.first(where: { $0.wireID == "thread-one" }) ) @@ -618,7 +625,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let detail = try await fixture.client.loadThread(id: thread.id) @@ -650,7 +657,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let detail = try await fixture.client.loadThread(id: thread.id) @@ -696,7 +703,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) XCTAssertTrue(thread.isRegeneratingTitle) @@ -724,7 +731,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture(repositoryIdentity: identity) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let groups = DailyUXCreationContext.projectGroups(in: snapshot) XCTAssertEqual(Set(snapshot.projects.compactMap(\.repositoryIdentity?.canonicalKey)), [ @@ -739,15 +746,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await fixture.transport.setReachable(false, host: "two.example") - let passiveFailure = try await fixture.client.initialSnapshot() + let passiveFailure = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(passiveFailure.threads.compactMap(\.wireID)), ["thread-one", "thread-two"] ) - XCTAssertEqual(passiveFailure.connection.state, .connected) + XCTAssertEqual(passiveFailure.environments.first { $0.id == "one" }?.connectionState, .connected) XCTAssertEqual( passiveFailure.environments.first(where: { $0.id == "two" })?.connectionState, .disconnected @@ -756,7 +763,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { await fixture.transport.setReachable(false, host: "one.example") await fixture.transport.setReachable(true, host: "two.example") - let activeFailure = try await fixture.client.initialSnapshot() + let activeFailure = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(activeFailure.threads.compactMap(\.wireID)), ["thread-one", "thread-two"] @@ -773,7 +780,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testCachedShellRowsApplySettlementAndRemoveDeletedRoutes() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() let original = try XCTUnwrap(initial.threads.first { $0.environmentID == "two" }) let updated = multiEnvironmentShell( projectID: "project-two", threadID: "thread-two", title: "Remote work", @@ -782,7 +789,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { settledOverride: "settled", settledAt: "2026-07-31T12:01:00.000Z" ) await fixture.transport.setShell(updated, host: "two.example") - let refreshed = try await fixture.client.initialSnapshot() + let refreshed = try await fixture.hydratedSnapshot() let settled = try XCTUnwrap(refreshed.threads.first { $0.id == original.id }) XCTAssertEqual(settled.updatedAt, original.updatedAt) XCTAssertTrue(settled.isSettled) @@ -796,7 +803,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "two.example" ) - let removed = try await fixture.client.initialSnapshot() + let removed = try await fixture.hydratedSnapshot() XCTAssertFalse(removed.threads.contains { $0.id == original.id }) XCTAssertEqual(removed.projects.first { $0.environmentID == "two" }?.threadCount, 0) do { @@ -811,7 +818,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testOlderHTTPSnapshotCannotReplaceNewerEnvironmentState() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let newer = multiEnvironmentShell( projectID: "project-one", @@ -827,7 +834,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let older = multiEnvironmentShell( projectID: "project-one", @@ -844,7 +851,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.first(where: { $0.environmentID == "one" })?.title, @@ -856,7 +863,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testThreadCreationCannotReplaceNewerEnvironmentStateWithAnOlderShell() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let newer = multiEnvironmentShell( projectID: "project-one", @@ -872,7 +879,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - let current = try await fixture.client.initialSnapshot() + let current = try await fixture.hydratedSnapshot() let project = try XCTUnwrap( current.projects.first(where: { $0.environmentID == "one" }) ) @@ -897,7 +904,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { title: "Another task", selection: nil ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.first(where: { $0.wireID == "thread-one" })?.title, @@ -969,7 +976,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await loader.waitForCallCount(2) let retryCallCount = await loader.callCount @@ -990,7 +997,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { _ = try await fixture.client.initialSnapshot() await loader.waitForCallCount(1) - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await loader.waitForFirstLoadCancellation() await loader.waitForCallCount(2) @@ -1003,7 +1010,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture(duplicateIDs: true) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.projects.count, 2) XCTAssertEqual(snapshot.threads.count, 2) XCTAssertEqual(Set(snapshot.projects.map(\.id)).count, 2) @@ -1026,7 +1033,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let remoteProject = try XCTUnwrap( snapshot.projects.first(where: { $0.environmentID == "two" }) ) @@ -1059,7 +1066,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveCreateRecoversACommittedThreadAfterItsReplyIsLost() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap( snapshot.projects.first(where: { $0.environmentID == "two" }) ) @@ -1084,32 +1091,17 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testUnarchiveImmediatelyRestoresLiveThreadWhenRefreshIsUnavailable() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap( initial.threads.first(where: { $0.environmentID == "one" }) ) - let events = fixture.client.events() - var iterator = events.makeAsyncIterator() + let recorder = try XCTUnwrap(fixture.recorder) await fixture.transport.setShellReadsEnabled(false, host: "one.example") - try await fixture.client.setThreadArchived(id: thread.id, archived: true) - while let event = await iterator.next() { - if case let .thread(candidate) = event, - candidate.id == thread.id, - candidate.isArchived { - break - } - } + _ = try await recorder.wait { $0.threads.contains { $0.id == thread.id && $0.isArchived } } try await fixture.client.setThreadArchived(id: thread.id, archived: false) - var restored: FeatureThread? - while let event = await iterator.next() { - if case let .thread(candidate) = event, - candidate.id == thread.id, - !candidate.isArchived { - restored = candidate - break - } - } + let restoredSnapshot = try await recorder.wait { $0.threads.contains { $0.id == thread.id && !$0.isArchived } } + let restored = restoredSnapshot.threads.first { $0.id == thread.id } XCTAssertEqual(restored?.id, thread.id) XCTAssertEqual(restored?.isArchived, false) @@ -1122,7 +1114,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { fallbackPollingInterval: .seconds(2) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let current = multiEnvironmentShell( projectID: "project-one", threadID: "thread-one", @@ -1148,23 +1140,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - let events = fixture.client.events() - var iterator = events.makeAsyncIterator() - var refreshed: FeatureSnapshot? - while let event = await iterator.next() { - if case let .snapshot(snapshot) = event, - snapshot.projects.contains(where: { $0.wireID == addedProject.id }) { - refreshed = snapshot - break - } - } - - XCTAssertEqual(refreshed?.connection.state, .reconnecting) + let recorder = try XCTUnwrap(fixture.recorder) + let refreshed = try await recorder.wait { $0.projects.contains { $0.wireID == addedProject.id } } + XCTAssertEqual(refreshed.connection.state, .reconnecting) await fixture.client.disconnect() } fileprivate static func makeFixture( duplicateIDs: Bool = false, + requireConfiguration: Bool = false, passiveSequence: Int = 1, includeThirdEnvironment: Bool = false, repositoryIdentity: RepositoryIdentity? = nil, @@ -1179,6 +1163,16 @@ final class NativeMultiEnvironmentTests: XCTestCase { aggregateRefreshSleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }, + aggregatePeerRefreshSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(250)) + }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, aggregateEnvironmentLoader: @escaping @Sendable (EnvironmentRuntime) async throws -> [Environment] = { try await $0.environments() } @@ -1273,9 +1267,11 @@ final class NativeMultiEnvironmentTests: XCTestCase { let settings = UserDefaults( suiteName: "t3-native-multi-\(UUID().uuidString)" )! + let receipts = PassiveLiveReceipts() return MultiEnvironmentFixture( directory: directory, transport: transport, + runtime: runtime, client: NativeFeatureClient( runtime: runtime, settingsStore: settings, @@ -1285,8 +1281,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { aggregateIdleRefreshInterval: aggregateIdleRefreshInterval, aggregateFailureRefreshInterval: aggregateFailureRefreshInterval, aggregateRefreshSleep: aggregateRefreshSleep, + aggregatePeerRefreshSleep: aggregatePeerRefreshSleep, + aggregateStreamRetrySleep: aggregateStreamRetrySleep, + aggregatePublishSleep: aggregatePublishSleep, + aggregateRefreshReceipt: { receipt in + receipts.record(receipt) + aggregateRefreshReceipt(receipt) + }, aggregateEnvironmentLoader: aggregateEnvironmentLoader - ) + ), receipts: receipts, requireConfiguration: requireConfiguration ) } } @@ -1369,11 +1372,13 @@ struct NativeGitHubRoutingTests { private func configuredFixture(_ server: MultiEnvironmentConfigurationServer, includeThird: Bool = false) async throws -> MultiEnvironmentFixture { let fixture = try await NativeMultiEnvironmentTests.makeFixture( + requireConfiguration: true, includeThirdEnvironment: includeThird, pullRequestsAvailable: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(2) ) - _ = try await fixture.client.initialSnapshot() + // Peers hydrate independently; routing reads each peer's configuration. + _ = try await fixture.hydratedSnapshot() for id in includeThird ? ["one", "two", "three"] : ["one", "two"] { try await fixture.client.setGitHubRoutingPermission(environmentID: id, permission: .readWrite) } @@ -1391,19 +1396,17 @@ struct NativePassiveThreadRefreshTests { func passiveThreadEventsArriveWithinFiveSecondsAndStayFastAfterChanges() async throws { let refreshSleep = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { _, interval in + try await refreshSleep.sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() - let thread = try #require( - initial.threads.first(where: { $0.environmentID == "two" }) - ) + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") let updatedTitle = "Passive work updated automatically" let eventProbe = ThreadTitleEventProbe( events: fixture.client.events(), - threadID: thread.id, + threadID: threadID, title: updatedTitle ) eventProbe.start() @@ -1433,8 +1436,8 @@ struct NativePassiveThreadRefreshTests { func passiveRefreshUsesTenSecondsWhenWorkIsUnchanged() async throws { let refreshSleep = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { _, interval in + try await refreshSleep.sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } @@ -1448,52 +1451,159 @@ struct NativePassiveThreadRefreshTests { await fixture.client.disconnect() } - @Test("A failed passive environment backs off without slowing an active peer") + @Test("A failed passive environment backs off without slowing an active peer", .timeLimit(.minutes(1))) func failedPassiveEnvironmentBacksOffWithoutSlowingActivePeer() async throws { - let refreshSleep = ControllableAggregateRefreshSleep() + let healthyClock = ControllableAggregateRefreshSleep() + let failedClock = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( includeThirdEnvironment: true, - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? healthyClock : failedClock).sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } _ = try await fixture.client.initialSnapshot() - await fixture.transport.setShell( - multiEnvironmentShell( - projectID: "project-two", - threadID: "thread-two", - title: "Remote work", - providerID: "claudeAgent", - modelID: "claude-opus-4-1", - backgroundLiveness: .working - ), - host: "two.example" - ) + _ = await healthyClock.waitUntilRequested(count: 1) + _ = await failedClock.waitUntilRequested(count: 1) await fixture.transport.setReachable(false, host: "three.example") + await failedClock.resume() + let failureCadence = await failedClock.waitUntilRequested(count: 2) + #expect(failureCadence == .seconds(20)) + for count in 2...4 { + await healthyClock.resume() + _ = await healthyClock.waitUntilRequested(count: count) + let failedReads = await fixture.transport.shellReadCount(host: "three.example") + #expect(failedReads == 2) + } + await failedClock.resume() + _ = await failedClock.waitUntilRequested(count: 3) + let retriedReads = await fixture.transport.shellReadCount(host: "three.example") + #expect(retriedReads == 3) + await fixture.client.disconnect() + } - let firstCadence = await refreshSleep.waitUntilRequested(count: 1) - #expect(firstCadence == .seconds(5)) - await refreshSleep.resume() - let secondCadence = await refreshSleep.waitUntilRequested(count: 2) - #expect(secondCadence == .seconds(5)) - let initialFailedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(initialFailedReadCount == 2) - - for requestCount in 2...4 { - await refreshSleep.resume() - let cadence = await refreshSleep.waitUntilRequested(count: requestCount + 1) - #expect(cadence == .seconds(5)) - let failedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(failedReadCount == 2) + @Test("Healthy rows publish twice while a peer shell and optional catalogue remain held", .timeLimit(.minutes(1))) + func healthyRowsPublishTwiceWhilePeerAndCatalogueAreHeld() async throws { + let healthyClock = ControllableAggregateRefreshSleep() + let slowClock = ControllableAggregateRefreshSleep() + let connector = GatedPassiveCatalogueConnector() + let topologyGate = PassiveRequestGate() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + webSocketConnector: connector, + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? healthyClock : slowClock).sleep(for: interval) + }, + aggregateEnvironmentLoader: { runtime in + await topologyGate.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let healthyID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + let heldShell = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "three.example", gate: heldShell) + await connector.holdPassiveConnections() + await topologyGate.release() + await connector.gate.waitUntilEntered() + _ = await healthyClock.waitUntilRequested(count: 1) + await heldShell.waitUntilEntered() + for update in 1...2 { + let title = "Healthy update \(update)" + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: healthyID, title: title) + probe.start() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: title, + snapshotSequence: update + 1 + ), host: "two.example") + await healthyClock.resume() + await probe.waitUntilObserved() + #expect(probe.didObserveTitle()) + _ = await healthyClock.waitUntilRequested(count: update + 1) + #expect(await heldShell.isHeld) + #expect(await connector.gate.isHeld) } + await fixture.client.disconnect() + await heldShell.release() + await connector.release() + } + @Test("A cancelled refresh generation cannot overwrite a restarted peer", .timeLimit(.minutes(1))) + func cancelledGenerationCannotOverwriteRestartedPeer() async throws { + let clock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + aggregatePeerRefreshSleep: { _, interval in try await clock.sleep(for: interval) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + _ = await clock.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let gate = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Stale generation", + snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: gate) + await clock.resume() + await gate.waitUntilEntered() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Current generation", + snapshotSequence: 2 + ), host: "two.example") + _ = try await fixture.client.initialSnapshot() + #expect(oldWorker.isCancelled) + await gate.release() + await oldWorker.value + // A stale 999 would survive this read of sequence 2 if it reached the cache. + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Current generation") + await fixture.client.disconnect() + } - await refreshSleep.resume() - _ = await refreshSleep.waitUntilRequested(count: 6) - let retriedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(retriedReadCount == 3) + @Test("Removal cancels a held peer and its late response never returns a row", .timeLimit(.minutes(1))) + func removedPeerCannotPublishLateResponse() async throws { + let removedClock = ControllableAggregateRefreshSleep() + let healthyClock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? removedClock : healthyClock).sleep(for: interval) + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let healthyID = FeatureScopedID.thread(environmentID: "three", wireID: "thread-three") + let probe = ThreadTitleEventProbe( + events: fixture.client.events(), threadID: healthyID, title: "Removal verified" + ) + probe.start() + _ = await removedClock.waitUntilRequested(count: 1) + _ = await healthyClock.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let gate = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Ghost removed row", + snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: gate) + await removedClock.resume() + await gate.waitUntilEntered() + try await fixture.client.removeEnvironment(id: "two") + #expect(oldWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + await gate.release() + await oldWorker.value + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-three", threadID: "thread-three", title: "Removal verified", + snapshotSequence: 2 + ), host: "three.example") + await healthyClock.resume() + await probe.waitUntilObserved() + #expect(!probe.sawThreadTitle("Ghost removed row")) + #expect(!probe.lastEnvironmentIDs.contains("two")) await fixture.client.disconnect() } + } private actor FailOnceAggregateEnvironmentLoader { @@ -1678,6 +1788,8 @@ private final class ThreadTitleEventProbe { private let threadID: String private let title: String private var observed = false + private var observedTitles = Set() + private(set) var lastEnvironmentIDs = Set() private var observedWaiters: [CheckedContinuation] = [] private var task: Task? @@ -1693,8 +1805,11 @@ private final class ThreadTitleEventProbe { for await event in events { switch event { case let .thread(thread): + observedTitles.insert(thread.title) observed = thread.id == threadID && thread.title == title case let .snapshot(snapshot): + observedTitles.formUnion(snapshot.threads.map(\.title)) + lastEnvironmentIDs = Set(snapshot.environments.map(\.id)) observed = snapshot.threads.contains { $0.id == self.threadID && $0.title == self.title } @@ -1710,6 +1825,8 @@ private final class ThreadTitleEventProbe { } } + func sawThreadTitle(_ title: String) -> Bool { observedTitles.contains(title) } + func didObserveTitle() -> Bool { observed } @@ -1727,6 +1844,7 @@ private final class ThreadTitleEventProbe { } private actor ControllableAggregateRefreshSleep { + var requestCount: Int { requestedCadences.count } private var requestedCadences: [Duration] = [] private var requestWaiters: [( count: Int, @@ -1768,10 +1886,47 @@ private actor ControllableAggregateRefreshSleep { } } -private struct MultiEnvironmentFixture { +@MainActor +private final class MultiEnvironmentFixture { let directory: URL let transport: MultiEnvironmentHTTPTransport + let runtime: EnvironmentRuntime let client: NativeFeatureClient + let receipts: PassiveLiveReceipts + let requireConfiguration: Bool + private(set) var recorder: BootstrapSnapshotRecorder? + + init(directory: URL, transport: MultiEnvironmentHTTPTransport, runtime: EnvironmentRuntime, + client: NativeFeatureClient, receipts: PassiveLiveReceipts, requireConfiguration: Bool) { + self.directory = directory + self.transport = transport + self.runtime = runtime + self.client = client + self.receipts = receipts + self.requireConfiguration = requireConfiguration + } + + func hydratedSnapshot() async throws -> FeatureSnapshot { + let ids = try await runtime.environments().filter(\.isEnabled).map(\.id) + let targets = Dictionary(uniqueKeysWithValues: ids.map { ($0, receipts.httpCount($0) + 1) }) + let seed = try await client.initialSnapshot() + if let recorder { recorder.record(seed) } + else { recorder = BootstrapSnapshotRecorder(seed: seed, events: client.events()) } + for id in ids { + try await receipts.waitForHTTP(id, count: targets[id]!) + if requireConfiguration { try await receipts.waitForConfiguration(id) } + } + let needsPublishedConfiguration = requireConfiguration + return try await recorder!.wait { snapshot in + ids.allSatisfy { id in + // The configuration receipt precedes publication. Wait for + // the same event consumer to observe settings in its snapshot. + if needsPublishedConfiguration && snapshot.preferencesByEnvironment?[id] == nil { return false } + guard let state = snapshot.environments.first(where: { $0.id == id })?.connectionState else { return false } + return state == .connected || state == .disconnected + } + } + } } private actor MultiEnvironmentHTTPTransport: HTTPTransport { @@ -1786,6 +1941,8 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { private var diffRequests: [(host: String, input: JSONValue)] = [] func pullRequestDiffRequests() -> [(host: String, input: JSONValue)] { diffRequests } + private var nextShellGates: [String: PassiveRequestGate] = [:] + private var nextDispatchGates: [String: PassiveRequestGate] = [:] init(shells: [String: OrchestrationShellSnapshot]) { self.shells = shells @@ -1837,11 +1994,30 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { hostsDroppingNextCreateReply.insert(host) } - func data(for request: URLRequest) throws -> (Data, HTTPURLResponse) { + func holdNextDispatch(host: String, gate: PassiveRequestGate) { nextDispatchGates[host] = gate } + + func holdNextShell(host: String, gate: PassiveRequestGate) { + nextShellGates[host] = gate + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { let host = request.url?.host ?? "" let path = request.url?.path ?? "" + if path == "/.well-known/t3/environment" { + let id = host == "one.example" ? "one" : "two" + return (try JSONEncoder.t3.encode(multiEnvironmentDescriptor( + environmentID: id, label: id == "one" ? "Left Book" : "Steam Box", pullRequestsAvailable: true + )), multiEnvironmentResponse(request)) + } + if path == "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/oauth/token" { + return (Data(#"{"access_token":"paired-token","issued_token_type":"urn:ietf:params:oauth:token-type:access_token","token_type":"Bearer","expires_in":3600,"scope":"read write"}"#.utf8), multiEnvironmentResponse(request)) + } if path == "/api/orchestration/shell" { shellReadCounts[host, default: 0] += 1 + if let gate = nextShellGates.removeValue(forKey: host), let data = shellData[host] { + await gate.enter() + return (data, multiEnvironmentResponse(request)) + } } guard reachableHosts.contains(host) else { throw URLError(.cannotConnectToHost) @@ -1876,6 +2052,7 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { dispatched.append( MultiEnvironmentDispatchRecord(host: host, command: command) ) + if let gate = nextDispatchGates.removeValue(forKey: host) { await gate.enter() } if command["type"]?.stringValue == "thread.create", hostsDroppingNextCreateReply.remove(host) != nil, let projectID = command["projectId"]?.stringValue, @@ -1935,6 +2112,7 @@ private actor MultiEnvironmentConfigurationServer { private let routingIdentityHosts: Set private let failPullRequestWrites: Bool private var prRequests: [(host: String, method: String, input: JSONValue)] = [] + private var dispatches: [(host: String, command: JSONValue)] = [] init( restartSupportHosts: Set = [], @@ -1962,6 +2140,7 @@ private actor MultiEnvironmentConfigurationServer { func settings(host: String) -> [String: JSONValue] { settingsByHost[host] ?? [:] } func fileRequests() -> [(host: String, input: JSONValue)] { directoryRequests } func pullRequestRequests() -> [(host: String, method: String, input: JSONValue)] { prRequests } + func dispatchedCommands(host: String) -> [JSONValue] { dispatches.filter { $0.host == host }.map(\.command) } func response(to request: JSONValue, host: String) throws -> JSONValue? { guard let tag = request["tag"]?.stringValue, @@ -2031,6 +2210,9 @@ private actor MultiEnvironmentConfigurationServer { } settingsByHost[host] = updated value = .object(settingsByHost[host] ?? [:]) + case RPCMethod.dispatchCommand.rawValue: + dispatches.append((host, request["payload"] ?? .object([:]))) + value = try JSONValue.encode(DispatchResult(sequence: 2)) case RPCMethod.getArchivedShellSnapshot.rawValue: value = try JSONValue.encode(OrchestrationShellSnapshot( snapshotSequence: 0, projects: [], threads: [], updatedAt: "2026-09-04T12:00:00.000Z" @@ -2333,3 +2515,1306 @@ private func multiEnvironmentResponse(_ request: URLRequest) -> HTTPURLResponse headerFields: ["Content-Type": "application/json"] )! } + +/// Deliberately ignores cancellation to model a transport completing an old read. +private actor PassiveRequestGate { + private var cancellableEntryWaiters: [UUID: CheckedContinuation] = [:] + private var entered = false + private var released = false + private var entryWaiters: [CheckedContinuation] = [] + private var releaseWaiters: [CheckedContinuation] = [] + var isHeld: Bool { entered && !released } + + func enter() async { + entered = true + let pending = cancellableEntryWaiters.values + cancellableEntryWaiters.removeAll() + pending.forEach { $0.resume() } + entryWaiters.forEach { $0.resume() } + entryWaiters.removeAll() + guard !released else { return } + await withCheckedContinuation { releaseWaiters.append($0) } + } + + func waitUntilEnteredCancellable() async throws { + try Task.checkCancellation() + guard !entered else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { cancellableEntryWaiters[id] = $0 } + } onCancel: { + Task { await self.cancelEntryWaiter(id) } + } + } + + private func cancelEntryWaiter(_ id: UUID) { + cancellableEntryWaiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } + + func waitUntilEntered() async { + guard !entered else { return } + await withCheckedContinuation { entryWaiters.append($0) } + } + + func release() { + released = true + releaseWaiters.forEach { $0.resume() } + releaseWaiters.removeAll() + } +} + +private actor GatedPassiveCatalogueConnector: WebSocketConnecting { + private var shouldHold = false + let gate = PassiveRequestGate() + + func holdPassiveConnections() { shouldHold = true } + + func connect(to url: URL) async throws -> any WebSocketConnection { + if shouldHold && url.host == "two.example" { + return GatedPassiveCatalogueConnection(gate: gate) + } + throw URLError(.cannotConnectToHost) + } + + func release() async { await gate.release() } +} + +private actor GatedPassiveCatalogueConnection: WebSocketConnection { + let gate: PassiveRequestGate + private var receiver: CheckedContinuation? + private var closed = false + + init(gate: PassiveRequestGate) { self.gate = gate } + + func send(_ data: Data) async throws { + let request = try JSONDecoder.t3.decode(JSONValue.self, from: data) + if request["tag"]?.stringValue != nil { await gate.enter() } + } + + func receive() async throws -> Data { + if closed { throw CancellationError() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + closed = true + receiver?.resume(throwing: CancellationError()) + receiver = nil + } +} + +@Suite("Native passive live shells") +@MainActor +struct NativePassiveLiveShellTests { + @Test("Live peer bursts publish together without polling", .timeLimit(.minutes(1))) + func livePeerBurstsPublishTogetherWithoutPolling() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let topologyClock = ControllableAggregateRefreshSleep() + let clocks = PassiveClockBank() + let publication = ControllableAggregateRefreshSleep() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregateRefreshSleep: { try await topologyClock.sleep(for: $0) }, + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: { try await publication.sleep(for: .zero) }, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threeID = FeatureScopedID.thread(environmentID: "three", wireID: "thread-three") + let heldTwo = PassiveRequestGate() + let heldThree = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "two.example", gate: heldTwo) + await fixture.transport.holdNextShell(host: "three.example", gate: heldThree) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await server.waitForSubscriptions(host: "three.example", count: 1) + await heldTwo.waitUntilEntered() + await heldThree.waitUntilEntered() + let shellTwo = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Live base", snapshotSequence: 10) + let shellThree = multiEnvironmentShell(projectID: "project-three", threadID: "thread-three", title: "Third base", snapshotSequence: 20) + try await server.snapshot(shellTwo, host: "two.example") + try await server.snapshot(shellThree, host: "three.example") + try await receipts.waitForShells("two", count: 1) + try await receipts.waitForShells("three", count: 1) + let finalTwo = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Burst final", snapshotSequence: 12) + let finalThree = multiEnvironmentShell(projectID: "project-three", threadID: "thread-three", title: "Third final", snapshotSequence: 21) + try await server.upsert(finalTwo.threads[0], sequence: 11, host: "two.example") + try await server.upsert(finalTwo.threads[0], sequence: 12, host: "two.example") + try await server.upsert(finalThree.threads[0], sequence: 21, host: "three.example") + try await receipts.waitForShells("two", count: 3) + try await receipts.waitForShells("three", count: 2) + _ = await publication.waitUntilRequested(count: 1) + #expect(await publication.requestCount == 1) + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: threeID, title: "Third final") + probe.start() + await publication.resume() + await probe.waitUntilObserved() + #expect(probe.sawThreadTitle("Burst final")) + #expect(await heldTwo.isHeld) + #expect(await heldThree.isHeld) + await heldTwo.release() + await heldThree.release() + try await receipts.waitForHTTP("two", count: 1) + try await receipts.waitForHTTP("three", count: 1) + let reads = await fixture.transport.shellReadCount(host: "two.example") + #expect(reads == 1) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + await fixture.client.disconnect() + } + + @Test("A replacement socket accepts a low sequence and rejects old HTTP", .timeLimit(.minutes(1))) + func replacementSocketRejectsOldHTTP() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let clocks = PassiveClockBank() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let oldHTTP = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Old HTTP", snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: oldHTTP) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await oldHTTP.waitUntilEntered() + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Old socket", snapshotSequence: 100 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + await server.closeLatest(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 2) + let fresh = multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Replacement socket", snapshotSequence: 2 + ) + await fixture.transport.setShell(fresh, host: "two.example") + try await server.snapshot(fresh, host: "two.example") + try await receipts.waitForShells("two", count: 2) + await oldHTTP.release() + try await receipts.waitForHTTP("two", count: 1) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Replacement socket") + #expect(receipts.sequences["two"] == [100, 2]) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + await fixture.client.disconnect() + } + + @Test("Unknown live events trigger one immediate HTTP repair and recover", .timeLimit(.minutes(1))) + func unknownEventRepairsWithoutPollingDelay() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + await server.waitForSubscriptions(host: "two.example", count: 1) + // Drain the initial repair before making the stream authoritative. + try await receipts.waitForHTTP("two", count: 1) + let live = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Live", snapshotSequence: 10) + try await server.snapshot(live, host: "two.example") + try await receipts.waitForShells("two", count: 1) + let initialReadCount = await fixture.transport.shellReadCount(host: "two.example") + let initialReceiptCount = receipts.httpCount("two") + let held = PassiveRequestGate() + let repaired = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "HTTP repaired", snapshotSequence: 11) + await fixture.transport.setShell(repaired, host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: held) + try await server.emit([.object(["kind": .string("future-shell-event")])], host: "two.example") + await held.waitUntilEntered() + await server.waitForSubscriptions(host: "two.example", count: 2) + let readCount = await fixture.transport.shellReadCount(host: "two.example") + #expect(readCount == initialReadCount + 1) + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: threadID, title: "HTTP repaired") + probe.start() + await held.release() + try await receipts.waitForHTTP("two", count: initialReceiptCount + 1) + await probe.waitUntilObserved() + try await server.snapshot(repaired, host: "two.example") + try await receipts.waitForShells("two", count: 2) + #expect(await fixture.transport.shellReadCount(host: "two.example") == initialReadCount + 1) + await fixture.client.disconnect() + } + + @Test("Background cancels passive work and foreground replaces subscriptions", .timeLimit(.minutes(1))) + func backgroundAndRemovalCancelOwnedSubscriptions() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let publication = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: { try await publication.sleep(for: .zero) }, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Cached live peer", snapshotSequence: 3 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + _ = await publication.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + fixture.client.suspendForBackground() + await oldWorker.value + await server.waitForInterrupts(host: "two.example", count: 1) + #expect(fixture.client.aggregateRefreshWorkers.isEmpty) + #expect(oldWorker.isCancelled) + await fixture.client.resumeAfterBackground(reconnect: false) + await server.waitForSubscriptions(host: "two.example", count: 2) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + let nextWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + try await fixture.client.removeEnvironment(id: "two") + await nextWorker.value + await server.waitForInterrupts(host: "two.example", count: 2) + #expect(nextWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(!snapshot.environments.contains { $0.id == "two" }) + await fixture.client.disconnect() + } + @Test("Repeated snapshot then stream end waits twenty seconds between subscriptions", .timeLimit(.minutes(1))) + func repeatedStreamEndsBackOffWhileHTTPRepairs() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, interval in try await retries.sleep(for: interval) }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + for attempt in 1...3 { + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Snapshot \(attempt)", snapshotSequence: 10 + attempt + ), host: "two.example") + try await receipts.waitForShells("two", count: attempt) + try await server.finish(host: "two.example") + let cadence = await retries.waitUntilRequested(count: attempt) + #expect(cadence == .seconds(20)) + try await receipts.waitForHTTP("two", count: attempt + 1) + #expect(await server.subscriptionCount(host: "two.example") == attempt) + if attempt < 3 { + await retries.resume() + await server.waitForSubscriptions(host: "two.example", count: attempt + 1) + } + } + await fixture.client.disconnect() + } + + @Test("Selecting a passive environment transfers shell ownership", .timeLimit(.minutes(1))) + func activeSwitchTransfersShellOwnership() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Selected peer", snapshotSequence: 10 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + // A same-socket HTTP refresh must retain the epoch association even + // when its complete snapshot is identical to the last socket snapshot. + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Selected peer", snapshotSequence: 10 + ), host: "two.example") + let readsBeforeRename = await fixture.transport.shellReadCount(host: "two.example") + try await fixture.client.renameThread( + id: FeatureScopedID.thread(environmentID: "two", wireID: "thread-two"), title: "Selected peer" + ) + #expect(await fixture.transport.shellReadCount(host: "two.example") == readsBeforeRename + 1) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Remote work", snapshotSequence: 1 + ), host: "two.example") + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let adoptedHTTPCount = receipts.httpCount("two") + 1 + _ = try await fixture.runtime.activate(id: "two") + let selected = try await fixture.client.initialSnapshot() + await oldWorker.value + await server.waitForSubscriptions(host: "two.example", count: 2) + try await receipts.waitForHTTP("two", count: adoptedHTTPCount) + #expect(oldWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + #expect(selected.environments.first { $0.isActive }?.id == "two") + #expect(selected.threads.first { $0.environmentID == "two" }?.title == "Selected peer") + await server.waitForSubscriptions(host: "one.example", count: 2) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "New passive peer", snapshotSequence: 2 + ), host: "one.example") + try await receipts.waitForShells("one", count: 2) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Selected peer") + #expect(snapshot.threads.first { $0.environmentID == "one" }?.title == "New passive peer") + + // Retaining the adopted socket's sequence must not pin a genuinely + // replacement socket to that old high watermark after a server restart. + let restarted = multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Restarted active peer", snapshotSequence: 2 + ) + await fixture.transport.setShell(restarted, host: "two.example") + await server.closeLatest(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 3) + try await server.snapshot(restarted, host: "two.example") + let recovered = try await recorder.wait { + $0.connection.state == .connected + && $0.threads.first { $0.environmentID == "two" }?.title == "Restarted active peer" + } + #expect(recovered.threads.first { $0.environmentID == "two" }?.title == "Restarted active peer") + await fixture.client.disconnect() + } + + @Test("HTTP begun before an unknown same-socket event cannot repair the gap", .timeLimit(.minutes(1))) + func sameSocketGapRejectsPreGapHTTP() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, interval in try await retries.sleep(for: interval) }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let oldRead = PassiveRequestGate() + let freshRead = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Pre-gap stale HTTP", snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: oldRead) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await oldRead.waitUntilEntered() + try await server.emit([.object(["kind": .string("unknown-before-baseline")])], host: "two.example") + _ = await retries.waitUntilRequested(count: 1) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Post-gap authoritative HTTP", snapshotSequence: 2 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: freshRead) + await oldRead.release() + try await receipts.waitForHTTP("two", count: 1) + await freshRead.waitUntilEntered() + #expect(await fixture.transport.shellReadCount(host: "two.example") == 2) + // If the old 999 was installed, this independent sequence-2 read + // cannot undo it and exposes the poisoned cache deterministically. + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Post-gap authoritative HTTP") + await freshRead.release() + try await receipts.waitForHTTP("two", count: 2) + await fixture.client.disconnect() + } + + @Test("An authoritative peer snapshot removes its selected detail subscription", .timeLimit(.minutes(1))) + func fullSnapshotRemovalStopsSelectedPeerDetail() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + // HTTP completion can mean a rejected read after the socket epoch + // changed. Establish an applied baseline before using its detail route. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Before removal", snapshotSequence: 9 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + _ = try await fixture.client.loadThread(id: threadID) + await server.waitForSubscriptions(host: "two.example#detail", count: 1) + let baseline = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Removed", snapshotSequence: 10) + try await server.snapshot(OrchestrationShellSnapshot( + snapshotSequence: 10, projects: baseline.projects, threads: [], updatedAt: baseline.updatedAt + ), host: "two.example") + try await receipts.waitForShells("two", count: 2) + await server.waitForInterrupts(host: "two.example#detail", count: 1) + #expect(await server.subscriptionCount(host: "one.example") == 1) + await fixture.client.disconnect() + } + +} + +@MainActor +private final class PassiveLiveReceipts { + private(set) var sequences: [String: [Int]] = [:] + private var counts: [String: Int] = [:] + private var waiters: [UUID: (String, Int, CheckedContinuation)] = [:] + + func record(_ receipt: NativePassiveShellReceipt) { + let key: String + switch receipt { + case let .shellApplied(id, sequence): + sequences[id, default: []].append(sequence) + key = "shell:" + id + case let .httpFinished(id): key = "http:" + id + case let .configurationApplied(id): key = "config:" + id + case let .archiveFinished(id): key = "archive:" + id + } + counts[key, default: 0] += 1 + let ready = waiters.filter { counts[$0.value.0, default: 0] >= $0.value.1 } + for (id, waiter) in ready { + waiters[id] = nil + waiter.2.resume() + } + } + + func httpCount(_ id: String) -> Int { counts["http:" + id, default: 0] } + func waitForShells(_ id: String, count: Int) async throws { try await wait("shell:" + id, count: count) } + func waitForHTTP(_ id: String, count: Int) async throws { try await wait("http:" + id, count: count) } + func waitForConfiguration(_ id: String) async throws { try await wait("config:" + id, count: 1) } + func waitForArchive(_ id: String) async throws { try await wait("archive:" + id, count: 1) } + + private func wait(_ key: String, count: Int) async throws { + try Task.checkCancellation() + guard counts[key, default: 0] < count else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (key, count, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.2.resume(throwing: CancellationError()) + } + } + } +} + +private actor PassiveClockBank { + private var clocks: [String: ControllableAggregateRefreshSleep] = [:] + + private func clock(_ id: String) -> ControllableAggregateRefreshSleep { + if let clock = clocks[id] { return clock } + let clock = ControllableAggregateRefreshSleep() + clocks[id] = clock + return clock + } + + func sleep(id: String, interval: Duration) async throws { + try await clock(id).sleep(for: interval) + } + + func resumeInitial(_ id: String) async { + let clock = clock(id) + _ = await clock.waitUntilRequested(count: 1) + await clock.resume() + } +} + +private actor PassiveLiveServer: WebSocketConnecting { + struct Subscription: Sendable { + let connection: PassiveLiveConnection + let requestID: Int + let payload: JSONValue? + } + private let configs = MultiEnvironmentConfigurationServer() + private var archiveGates: [String: PassiveRequestGate] = [:] + func holdArchive(host: String, gate: PassiveRequestGate) { archiveGates[host] = gate } + private var subscriptions: [String: [Subscription]] = [:] + private var interrupts: [String: Int] = [:] + private var waiters: [(String, Int, Bool, CheckedContinuation)] = [] + + func connect(to url: URL) -> any WebSocketConnection { + PassiveLiveConnection(host: url.host ?? "", server: self) + } + + func request(_ value: JSONValue, host: String, connection: PassiveLiveConnection) async throws -> JSONValue? { + if let tag = value["tag"]?.stringValue, + tag == RPCMethod.subscribeShell.rawValue || tag == RPCMethod.subscribeThread.rawValue, + case let .number(id)? = value["id"] { + let key = tag == RPCMethod.subscribeThread.rawValue ? host + "#detail" : host + subscriptions[key, default: []].append(Subscription(connection: connection, requestID: Int(id), payload: value["payload"])) + resumeWaiters() + return nil + } + if value["_tag"]?.stringValue == "Interrupt", case let .number(id)? = value["requestId"] { + for key in [host, host + "#detail"] where subscriptions[key, default: []].contains(where: { + $0.connection === connection && $0.requestID == Int(id) + }) { + interrupts[key, default: 0] += 1 + } + resumeWaiters() + } + if value["_tag"]?.stringValue == "Ping" { return .object(["_tag": .string("Pong")]) } + if value["tag"]?.stringValue == RPCMethod.dispatchCommand.rawValue, + ["thread.meta.update", "thread.unarchive"].contains(value["payload"]?["type"]?.stringValue ?? ""), + case let .number(id)? = value["id"] { + return .object([ + "_tag": .string("Exit"), "requestId": .number(id), + "exit": .object(["_tag": .string("Success"), "value": try JSONValue.encode(DispatchResult(sequence: 10))]), + ]) + } + if value["tag"]?.stringValue == RPCMethod.getArchivedShellSnapshot.rawValue, + let gate = archiveGates.removeValue(forKey: host) { await gate.enter() } + return try await configs.response(to: value, host: host) + } + + func lastPayload(host: String) -> JSONValue? { subscriptions[host]?.last?.payload } + + func waitForSubscriptions(host: String, count: Int) async { await wait(host: host, count: count, interrupt: false) } + func waitForInterrupts(host: String, count: Int) async { await wait(host: host, count: count, interrupt: true) } + + private func count(_ host: String, interrupt: Bool) -> Int { + interrupt ? interrupts[host, default: 0] : subscriptions[host, default: []].count + } + + private func wait(host: String, count: Int, interrupt: Bool) async { + guard self.count(host, interrupt: interrupt) < count else { return } + await withCheckedContinuation { waiters.append((host, count, interrupt, $0)) } + } + + private func resumeWaiters() { + let ready = waiters.filter { count($0.0, interrupt: $0.2) >= $0.1 } + waiters.removeAll { count($0.0, interrupt: $0.2) >= $0.1 } + ready.forEach { $0.3.resume() } + } + + func emit(_ values: [JSONValue], host: String) async throws { + guard let subscription = subscriptions[host]?.last else { throw URLError(.notConnectedToInternet) } + try await subscription.connection.enqueue(.object([ + "_tag": .string("Chunk"), "requestId": .number(Double(subscription.requestID)), "values": .array(values), + ])) + } + + func snapshot(_ shell: OrchestrationShellSnapshot, host: String) async throws { + try await emit([.object(["kind": .string("snapshot"), "snapshot": try JSONValue.encode(shell)])], host: host) + } + + func upsert(_ thread: OrchestrationThreadShell, sequence: Int, host: String) async throws { + try await emit([.object([ + "kind": .string("thread-upserted"), "sequence": .number(Double(sequence)), "thread": try JSONValue.encode(thread), + ])], host: host) + } + + func finish(host: String) async throws { + guard let subscription = subscriptions[host]?.last else { throw URLError(.notConnectedToInternet) } + try await subscription.connection.enqueue(.object([ + "_tag": .string("Exit"), "requestId": .number(Double(subscription.requestID)), + "exit": .object(["_tag": .string("Success"), "value": .null]), + ])) + } + + func subscriptionCount(host: String) -> Int { subscriptions[host, default: []].count } + + func closeLatest(host: String) async { await subscriptions[host]?.last?.connection.close() } +} + +private actor PassiveLiveConnection: WebSocketConnection { + private let host: String + private let server: PassiveLiveServer + private var responses: [Data] = [] + private var receiver: CheckedContinuation? + private var closed = false + + init(host: String, server: PassiveLiveServer) { self.host = host; self.server = server } + + func send(_ data: Data) async throws { + guard !closed else { throw URLError(.networkConnectionLost) } + let value = try JSONDecoder.t3.decode(JSONValue.self, from: data) + if let response = try await server.request(value, host: host, connection: self) { try enqueue(response) } + } + + func enqueue(_ value: JSONValue) throws { + let data = try JSONEncoder.t3.encode(value) + if let receiver { self.receiver = nil; receiver.resume(returning: data) } + else { responses.append(data) } + } + + func receive() async throws -> Data { + guard !closed else { throw URLError(.networkConnectionLost) } + if !responses.isEmpty { return responses.removeFirst() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + closed = true + receiver?.resume(throwing: URLError(.networkConnectionLost)) + receiver = nil + } +} + +@Suite("Native incremental bootstrap") +@MainActor +struct NativeIncrementalBootstrapTests { + @Test("Pairing resets the previous environment's hydration and archive lifetime", .timeLimit(.minutes(1)), arguments: [false, true]) + func pairingReplacesHydrationLifetime(previousHydrated: Bool) async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + if !previousHydrated { await fixture.transport.holdNextShell(host: "one.example", gate: held) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + do { + if previousHydrated { try await receipts.waitForArchive("one") } + else { try await held.waitUntilEnteredCancellable() } + await server.waitForSubscriptions(host: "two.example", count: 1) + try await fixture.client.pair(endpoint: "https://two.example", token: "fixture-bootstrap") + await server.waitForSubscriptions(host: "two.example", count: 2) + let shell = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "New active hydration", snapshotSequence: 20) + await fixture.transport.setShell(shell, host: "two.example") + try await server.upsert(shell.threads[0], sequence: 20, host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "New active hydration" } } + try await receipts.waitForArchive("two") + if !previousHydrated { #expect(await held.isHeld) } + } catch { + await held.release() + await fixture.client.disconnect() + throw error + } + await held.release() + await fixture.client.disconnect() + } + + @Test("Foreground restores bootstrap before a client has been adopted", .timeLimit(.minutes(1))) + func foregroundRestartsUnadoptedClient() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let recorder = BootstrapSnapshotRecorder( + seed: FeatureSnapshot(connection: .init(state: .disconnected)), events: fixture.client.events() + ) + defer { recorder.stop() } + fixture.client.suspendForBackground() + await fixture.client.resumeAfterBackground(reconnect: false) + let restored = try await recorder.wait { $0.threads.contains { $0.environmentID == "one" } } + #expect(restored.environments.contains { $0.id == "one" }) + await fixture.client.disconnect() + } + + @Test("An archive refresh reads the current shell after the archive RPC settles", .timeLimit(.minutes(1))) + func archiveRefreshReadsShellAfterArchive() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + try await receipts.waitForArchive("one") + let archive = PassiveRequestGate() + await server.holdArchive(host: "one.example", gate: archive) + let refresh = Task { try await fixture.client.setThreadArchived(id: "thread-one", archived: false) } + do { + try await archive.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "Current after archive", snapshotSequence: 20 + ), host: "one.example") + await archive.release() + try await refresh.value + #expect(recorder.history.last?.threads.contains { $0.title == "Current after archive" } == true) + } catch { + await archive.release() + refresh.cancel() + _ = try? await refresh.value + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("Metadata returns while active HTTP and catalogue are held; a healthy peer publishes", .timeLimit(.minutes(1))) + func heldActiveDoesNotBlockHealthyPeer() async throws { + let connector = BootstrapHeldConnector() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, webSocketConnector: connector, + rpcConnectionWaitTimeout: .seconds(60), fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let active = PassiveRequestGate() + let passive = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: active) + await fixture.transport.holdNextShell(host: "two.example", gate: passive) + let seed = try await fixture.client.initialSnapshot() + #expect(Set(seed.environments.map(\.id)) == ["one", "two", "three"]) + #expect(seed.threads.isEmpty) + #expect(seed.environments.allSatisfy { $0.connectionState != .connected }) + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + try await active.waitUntilEnteredCancellable() + try await passive.waitUntilEnteredCancellable() + try await connector.gate.waitUntilEnteredCancellable() + let healthy = try await recorder.wait { snapshot in + snapshot.threads.contains { $0.environmentID == "three" } + && snapshot.environments.first { $0.id == "three" }?.connectionState == .connected + } + #expect(await active.isHeld) + #expect(await passive.isHeld) + #expect(await connector.gate.isHeld) + #expect(healthy.environments.first { $0.id == "one" }?.connectionState != .connected) + await active.release() + await passive.release() + await connector.gate.release() + await fixture.client.disconnect() + } + + @Test("Catalog and synchronized markers cannot authorize an empty outbox view", .timeLimit(.minutes(1))) + func authorityAndRowsPublishTogether() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: held) + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + let submission = FeatureQueuedSubmission( + environmentID: "one", identity: .init(threadID: "thread-one"), + threadID: FeatureScopedID.thread(environmentID: "one", wireID: "thread-one"), + text: "Queued before hydration", selection: nil, runtimeMode: .fullAccess, + interactionMode: .standard, attachments: [] + ) + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: seed) == .wait) + try await held.waitUntilEnteredCancellable() + await server.waitForSubscriptions(host: "one.example", count: 1) + try await server.emit([.object(["kind": .string("synchronized")])], host: "one.example") + try await receipts.waitForConfiguration("one") + let shell = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Authoritative", snapshotSequence: 10) + try await server.snapshot(shell, host: "one.example") + let ready = try await recorder.wait { $0.threads.contains { $0.title == "Authoritative" } } + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: ready) == .send) + for snapshot in recorder.history { + let connected = snapshot.environments.first { $0.id == "one" }?.connectionState == .connected + #expect(!connected || snapshot.threads.contains { $0.id == submission.threadID }) + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: snapshot) != .discard) + } + #expect(await held.isHeld) + await held.release() + await fixture.client.disconnect() + } + + @Test("Same-client reload rejects a late old HTTP response and retains cached rows", .timeLimit(.minutes(1))) + func reloadRetainsCacheAndRejectsOldRead() async throws { + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { $0.threads.count == 2 } + let old = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Stale old request", snapshotSequence: 999), host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: old) + let cached = try await fixture.client.initialSnapshot() + #expect(cached.threads.count == 2) + #expect(cached.environments.allSatisfy { $0.connectionState != .connected }) + try await old.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Current reload", snapshotSequence: 2), host: "one.example") + _ = try await fixture.client.initialSnapshot() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Current reload" } } + await old.release() + try await receipts.waitForHTTP("one", count: 3) + #expect(!recorder.history.contains { $0.threads.contains { $0.title == "Stale old request" } }) + await fixture.client.disconnect() + } + + @Test("A current HTTP command refresh keeps the live header reconnecting while its rows are ready", .timeLimit(.minutes(1))) + func currentHTTPCommandPreservesReconnectingHeader() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + do { + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "HTTP command result", snapshotSequence: 2 + ), host: "one.example") + try await fixture.client.renameThread(id: "thread-one", title: "HTTP command result") + let updated = try await recorder.wait { $0.threads.contains { $0.title == "HTTP command result" } } + #expect(updated.connection.state == .reconnecting) + #expect(updated.environments.first { $0.id == "one" }?.connectionState == .connected) + #expect(await fixture.transport.dispatchRecords().count == 1) + } catch { + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("A public rename's old optional shell cannot authorize a replacement bootstrap", .timeLimit(.minutes(1))) + func oldCommandRefreshCannotAuthorizeReload() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + let oldRead = PassiveRequestGate() + let replacementRead = PassiveRequestGate() + let stale = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Old command shell", snapshotSequence: 999) + await fixture.transport.setShell(stale, host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: oldRead) + let command = Task { try await fixture.client.renameThread(id: "thread-one", title: "Rename once") } + do { + try await oldRead.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Replacement bootstrap", snapshotSequence: 2), host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: replacementRead) + let seed = try await fixture.client.initialSnapshot() + recorder.record(seed) + try await replacementRead.waitUntilEnteredCancellable() + await oldRead.release() + try await command.value + #expect(recorder.history.last?.environments.first { $0.id == "one" }?.connectionState != .connected) + #expect(!recorder.history.contains { $0.threads.contains { $0.title == "Old command shell" } }) + await replacementRead.release() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Replacement bootstrap" } } + } catch { + await oldRead.release() + await replacementRead.release() + command.cancel() + _ = try? await command.value + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("A cancelled old archive cannot authorize a same-client reload", .timeLimit(.minutes(1))) + func oldArchiveCannotAuthorizeReload() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let archive = PassiveRequestGate() + await server.holdArchive(host: "one.example", gate: archive) + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { $0.threads.contains { $0.environmentID == "one" } } + try await archive.waitUntilEnteredCancellable() + let heldShell = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: heldShell) + let reload = try await fixture.client.initialSnapshot() + recorder.record(reload) + let startIndex = recorder.history.count - 1 + try await heldShell.waitUntilEnteredCancellable() + await archive.release() + try await receipts.waitForArchive("one") + for snapshot in recorder.history.dropFirst(startIndex) { + #expect(snapshot.environments.first { $0.id == "one" }?.connectionState != .connected) + } + #expect(await heldShell.isHeld) + await heldShell.release() + await fixture.client.disconnect() + } + + @Test("Unknown active events repair once, then a replacement stream delivers deltas", .timeLimit(.minutes(1))) + func unknownActiveStreamRepairsAndResubscribes() async throws { + let server = PassiveLiveServer() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregateStreamRetrySleep: { id, interval in + if id == "one" { try await retries.sleep(for: interval) } + }, aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + await server.waitForSubscriptions(host: "one.example", count: 1) + let baseline = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active live baseline", snapshotSequence: 10) + try await server.snapshot(baseline, host: "one.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active live baseline" } } + let held = PassiveRequestGate() + let repaired = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active HTTP repaired", snapshotSequence: 20) + await fixture.transport.setShell(repaired, host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: held) + try await server.emit([.object(["kind": .string("unknown-active-event")])], host: "one.example") + try await held.waitUntilEnteredCancellable() + #expect(await retries.waitUntilRequested(count: 1) == .seconds(20)) + // These are delivered to the discarded subscription and cannot poison repair. + for sequence in 21...23 { try await server.upsert(baseline.threads[0], sequence: sequence, host: "one.example") } + await held.release() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active HTTP repaired" } } + #expect(await server.subscriptionCount(host: "one.example") == 1) + await retries.resume() + await server.waitForSubscriptions(host: "one.example", count: 2) + try await server.snapshot(repaired, host: "one.example") + let delta = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active delta recovered", snapshotSequence: 24) + try await server.upsert(delta.threads[0], sequence: 24, host: "one.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active delta recovered" } } + await fixture.client.disconnect() + } + + @Test("A failed initial socket attempt cannot invalidate its independent HTTP read", .timeLimit(.minutes(1))) + func failedSocketDoesNotStarveHTTP() async throws { + let connector = FailedBootstrapAttemptConnector() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: connector, fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: held) + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + do { + try await held.waitUntilEnteredCancellable() + try await connector.waitForFailure() + await held.release() + let ready = try await recorder.wait { + $0.environments.first { $0.id == "one" }?.connectionState == .connected + } + #expect(ready.threads.contains { $0.environmentID == "one" }) + #expect(await fixture.transport.shellReadCount(host: "one.example") == 1) + } catch { + await held.release() + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } +} + +/// One collector per fixture; any number of predicates observe the same snapshots. +@MainActor +final class BootstrapSnapshotRecorder { + private(set) var history: [FeatureSnapshot] + private var task: Task? + private var waiters: [UUID: (@MainActor (FeatureSnapshot) -> Bool, CheckedContinuation)] = [:] + private var finished = false + + init(seed: FeatureSnapshot, events: AsyncStream) { + history = [seed] + task = Task { [weak self] in + for await event in events { + guard !Task.isCancelled else { break } + guard let self else { break } + switch event { + case let .snapshot(snapshot): self.record(snapshot) + case let .thread(thread): + guard var snapshot = self.history.last else { continue } + if let index = snapshot.threads.firstIndex(where: { $0.id == thread.id }) { + snapshot.threads[index] = thread + } else { snapshot.threads.append(thread) } + self.record(snapshot) + case let .threadRemoved(id): + guard var snapshot = self.history.last else { continue } + snapshot.threads.removeAll { $0.id == id } + self.record(snapshot) + default: break + } + } + self?.stop() + } + } + + func record(_ snapshot: FeatureSnapshot) { + history.append(snapshot) + let ready = waiters.filter { $0.value.0(snapshot) } + for (id, waiter) in ready { + waiters[id] = nil + waiter.1.resume(returning: snapshot) + } + } + + func wait(_ predicate: @escaping @MainActor (FeatureSnapshot) -> Bool) async throws -> FeatureSnapshot { + try Task.checkCancellation() + if let snapshot = history.last, predicate(snapshot) { return snapshot } + guard !finished else { throw CancellationError() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (predicate, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + } + } + + func stop() { + finished = true + task?.cancel() + task = nil + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.1.resume(throwing: CancellationError()) } + } + + deinit { task?.cancel() } +} + +private actor BootstrapHeldConnector: WebSocketConnecting { + let gate = PassiveRequestGate() + func connect(to url: URL) async throws -> any WebSocketConnection { + if url.host == "one.example" { await gate.enter() } + throw URLError(.cannotConnectToHost) + } +} + +@Suite("Native bootstrap durable outbox") +@MainActor +struct NativeBootstrapRootOutboxTests { + @Test("A persisted follow-up waits through another peer's update and sends once after its owner hydrates", .timeLimit(.minutes(1))) + func restoredFollowUpWaitsForOwningShell() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let outbox = FeatureOutboxStore(fileURL: fixture.directory.appendingPathComponent("root-outbox.json")) + let drafts = FeatureComposerDraftStore(fileURL: fixture.directory.appendingPathComponent("root-drafts.json")) + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + let identity = FeatureSubmissionIdentity(threadID: "thread-two", commandID: "bootstrap-root-command", messageID: "bootstrap-root-message") + let submission = FeatureQueuedSubmission( + environmentID: "two", identity: identity, threadID: threadID, + text: "Persisted before the peer loaded", selection: nil, + runtimeMode: .fullAccess, interactionMode: .standard, attachments: [] + ) + try await outbox.enqueue(submission) + let heldShell = PassiveRequestGate() + let heldDispatch = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "two.example", gate: heldShell) + await fixture.transport.holdNextDispatch(host: "two.example", gate: heldDispatch) + let model = FeatureRootModel(client: fixture.client, outboxStore: outbox, draftStore: drafts) + let run = Task { await model.start() } + do { + try await heldShell.waitUntilEnteredCancellable() + let healthy = RootObservationWaiter { + !model.isLoading && model.snapshot.threads.contains { $0.environmentID == "one" } + } + try await healthy.wait() + #expect(model.snapshot.environments.first { $0.id == "two" }?.connectionState != .connected) + #expect(try await outbox.submissions().map(\.id) == [submission.id]) + #expect(await fixture.transport.dispatchRecords().isEmpty) + #expect(await heldShell.isHeld) + + await heldShell.release() + try await heldDispatch.waitUntilEnteredCancellable() + // Load the real detail while acceptance is held; the root attaches its + // restored queued message, giving an observable completion after removal. + _ = await model.detail(for: threadID) + #expect(model.details[threadID]?.messages.first { $0.id == identity.messageID }?.state == .queued) + let completed = RootObservationWaiter { + model.details[threadID]?.messages.first { $0.id == identity.messageID }?.state == .complete + } + await heldDispatch.release() + try await completed.wait() + // Message acceptance may precede durable deletion. Wait for the actual + // owned outbox file change, independent of that presentation ordering. + try await OwnedOutboxRemovalWaiter(fileURL: fixture.directory.appendingPathComponent("root-outbox.json")).wait() + #expect(try await outbox.submissions().isEmpty) + let commands = await fixture.transport.dispatchRecords() + #expect(commands.count == 1) + #expect(commands.first?.host == "two.example") + #expect(commands.first?.command["commandId"]?.stringValue == identity.commandID) + #expect(commands.first?.command["threadId"]?.stringValue == identity.threadID) + #expect(commands.first?.command["message"]?["messageId"]?.stringValue == identity.messageID) + #expect(model.errorMessage == nil) + } catch { + await heldShell.release() + await heldDispatch.release() + run.cancel() + await fixture.client.disconnect() + await run.value + throw error + } + await heldShell.release() + await heldDispatch.release() + run.cancel() + await fixture.client.disconnect() + await run.value + } +} + +/// Observe the root's applied state without taking a second client event iterator. +@MainActor +private final class RootObservationWaiter { + private let predicate: @MainActor () -> Bool + private var continuation: CheckedContinuation? + + init(_ predicate: @escaping @MainActor () -> Bool) { self.predicate = predicate } + + func wait() async throws { + try Task.checkCancellation() + if predicate() { return } + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor [weak self] in + self?.finish(.failure(CancellationError())) + } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { predicate() } onChange: { [weak self] in + Task { @MainActor in self?.observe() } + } + if ready { finish(.success(())) } + } + + private func finish(_ result: Result) { + let pending = continuation + continuation = nil + pending?.resume(with: result) + } +} + + +/// The test watches only its owned directory. Atomic store writes replace the +/// file, so observing the directory avoids retaining an obsolete file inode. +@MainActor +private final class OwnedOutboxRemovalWaiter { + private let fileURL: URL + private var source: DispatchSourceFileSystemObject? + private var continuation: CheckedContinuation? + + init(fileURL: URL) { self.fileURL = fileURL } + + func wait() async throws { + try Task.checkCancellation() + let descriptor = open(fileURL.deletingLastPathComponent().path, O_EVTONLY) + guard descriptor >= 0 else { throw POSIXError(.EIO) } + let source = DispatchSource.makeFileSystemObjectSource( + fileDescriptor: descriptor, eventMask: [.write, .rename], queue: .global() + ) + self.source = source + source.setCancelHandler { close(descriptor) } + source.setEventHandler { [weak self] in + Task { @MainActor in self?.check() } + } + source.resume() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + check() + } + } onCancel: { + Task { @MainActor [weak self] in self?.finish(.failure(CancellationError())) } + } + } + + private func check() { + guard continuation != nil else { return } + do { + let document = try JSONSerialization.jsonObject(with: Data(contentsOf: fileURL)) as? [String: Any] + guard let submissions = document?["submissions"] as? [Any] else { throw URLError(.cannotParseResponse) } + if submissions.isEmpty { finish(.success(())) } + } catch { finish(.failure(error)) } + } + + private func finish(_ result: Result) { + let pending = continuation + continuation = nil + source?.cancel() + source = nil + pending?.resume(with: result) + } +} + + +/// Observe the failed connector attempt itself: the RPC subscription deliberately +/// keeps waiting for its first usable socket instead of failing with a UI header. +private actor FailedBootstrapAttemptConnector: WebSocketConnecting { + private let failures = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + + func connect(to url: URL) async throws -> any WebSocketConnection { + if url.host == "one.example" { failures.continuation.yield(()) } + throw URLError(.cannotConnectToHost) + } + + func waitForFailure() async throws { + try Task.checkCancellation() + var iterator = failures.stream.makeAsyncIterator() + guard await iterator.next() != nil else { throw CancellationError() } + try Task.checkCancellation() + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index c3a5a4f146bc..fd0bda3de996 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -353,31 +353,21 @@ final class NativeRetryIdentityTests: XCTestCase { let settingsStore = UserDefaults(suiteName: settingsSuite)! defer { settingsStore.removePersistentDomain(forName: settingsSuite) } let client = NativeFeatureClient(runtime: runtime, settingsStore: settingsStore) - var events = client.events().makeAsyncIterator() - let initial = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + let initial = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() var updated = initial.settings updated.textSize = FeatureTextSizeAdjustment(steps: 2) updated.codeSize = FeatureTextSizeAdjustment(steps: -1) try await client.saveSettings(updated) await connection.failReceive() - - var receivedReconnect = false - while let event = await events.next() { - guard case let .connection(state, _) = event, - state.state == .reconnecting else { - continue - } - receivedReconnect = true - break + let snapshot = try await recorder.wait { + $0.connection.state == .reconnecting && $0.settings.textSize.steps == 2 } - XCTAssertTrue(receivedReconnect) - // A reconnect patches the connection instead of republishing the - // snapshot. The next snapshot the client builds must still carry the - // saved sizes. - let republished = try await client.backgroundSnapshot() - XCTAssertEqual(republished.settings.textSize.steps, 2) - XCTAssertEqual(republished.settings.codeSize.steps, -1) + XCTAssertEqual(snapshot.settings.textSize.steps, 2) + XCTAssertEqual(snapshot.settings.codeSize.steps, -1) await client.disconnect() } @@ -1550,6 +1540,13 @@ private final class AcceptedCommandSnapshotRecorder { guard var snapshot = self.history.last else { continue } snapshot.threads.removeAll { $0.id == id } self.record(snapshot) + case let .connection(connection, environmentID): + guard var snapshot = self.history.last else { continue } + snapshot.connection = connection + if let index = snapshot.environments.firstIndex(where: { $0.id == environmentID }) { + snapshot.environments[index].connectionState = connection.state + } + self.record(snapshot) default: break } } diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index 267a00c1a3e3..9fe19f16868c 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -1452,15 +1452,18 @@ private struct CatchUpFixture { requests: requests.continuation, completionMarker: completionMarker ) ) + let readiness = CatchUpBootstrapReadiness() let client = NativeFeatureClient( runtime: runtime, settingsStore: UserDefaults(suiteName: UUID().uuidString)!, fallbackPollingInitialDelay: .seconds(3_600), aggregateRefreshInterval: .seconds(3_600), + aggregateRefreshReceipt: { readiness.record($0) }, detailPublicationSleep: detailPublicationSleep, catchUpDelay: { try await delay.wait() }, threadRetryDelay: threadRetryDelay ) _ = try await client.initialSnapshot() + try await readiness.wait() return Self(client: client, http: http, requests: requests.stream, delay: delay, directory: directory) } @@ -1942,3 +1945,36 @@ private final class CatchUpPublicationClock: Sendable { func release() async { await gate.release() } } + +@MainActor +private final class CatchUpBootstrapReadiness { + private var hasShell = false + private var hasConfig = false + private var waiters: [UUID: CheckedContinuation] = [:] + + func record(_ receipt: NativePassiveShellReceipt) { + switch receipt { + case .shellApplied(environmentID: "one", sequence: _): hasShell = true + case .configurationApplied(environmentID: "one"): hasConfig = true + default: break + } + if hasShell && hasConfig { + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.resume() } + } + } + + func wait() async throws { + try Task.checkCancellation() + guard !hasShell || !hasConfig else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { waiters[id] = $0 } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } + } + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift b/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift index 259fdca781fa..71782c698f3c 100644 --- a/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift @@ -39,7 +39,9 @@ final class T3ConnectNativeCapabilityTests: XCTestCase { guard case let .snapshot(snapshot)? = event else { return XCTFail("Managed connect did not publish its initial Home snapshot") } - XCTAssertEqual(snapshot.connection.state, .connected) + // HTTP hydration publishes content before the deliberately blocked live stream. + XCTAssertEqual(snapshot.connection.state, .reconnecting) + XCTAssertEqual(snapshot.environments.first(where: { $0.id == "managed-1" })?.connectionState, .connected) XCTAssertEqual(snapshot.connection.environmentName, "Managed Studio") let saved = try await store.load() From dd9d9eabe1905528898ad9a43cde36e6783be7bd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:02:29 +1000 Subject: [PATCH 10/19] fix(swift-ios): keep applying config events after a catalog read fails A transient environment catalog read failure ended the active client's configuration subscription, so later provider and settings updates were ignored. Skip that publish instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index b1af76250544..e18d8ebed789 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -4221,7 +4221,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, case .unrelated: continue } - let environments = try await self.runtime.environments() + // A failed catalog read skips this publish; later events still apply. + guard let environments = try? await self.runtime.environments() else { continue } guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), environments.contains(activeClient.environment), let activeEnvironment = self.activeEnvironment else { return } From e3e7ec126bdded0ecb43e0b0ae55d95a738e0226 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:04:35 +1000 Subject: [PATCH 11/19] fix(swift-ios): keep a peer's newer config over an in-flight catalogue probe A passive peer's catalogue probe started at bootstrap could finish after the user saved a preference there and replace it with the older value. Discard the probe when a config already arrived. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index e18d8ebed789..d3ebec4fb9db 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -4626,6 +4626,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let config = try? await probe.serverConfig() await probe.disconnect() guard let environments = try await currentEnvironments(for: environment) else { return } + // A save or config event that landed during the probe is newer. + guard owner?.serverConfigsByEnvironmentID[environment.id] == nil else { return } if let config { applyCatalogue(config, environment: environment, saved: environments) return From 9cd6d823df1cc8dc19e1fd745a774b647eb3afec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:16:45 +1000 Subject: [PATCH 12/19] fix(swift-ios): stop probing a peer's catalogue once it needs pairing A peer whose credential was rejected before its catalogue loaded kept opening a probe client every 20 seconds. The probe's own error cannot show the rejection (the socket ticket 401 surfaces as a timeout), so stop once the peer's shell read has marked it as needing pairing; re-pairing starts a new worker. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index d3ebec4fb9db..b52522c07586 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -4618,7 +4618,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func refreshCatalogue(_ environment: Environment) async { - while isCurrent && owner?.serverConfigsByEnvironmentID[environment.id] == nil { + // A rejected peer needs pairing; re-pairing starts a new worker. + while isCurrent && owner?.serverConfigsByEnvironmentID[environment.id] == nil + && owner?.environmentConnectionStates[environment.id] != .needsPairing { do { guard try await currentEnvironments(for: environment) != nil else { return } // Never disconnect the shared client if this peer becomes selected. From dca56985baf014f79258fd2dd56c5b218e2a7edf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:39:38 +1000 Subject: [PATCH 13/19] perf(swift-ios): skip validated stale detail replay --- apps/swift-ios/App/NativeFeatureClient.swift | 14 ++- .../NativeThreadCatchUpTests.swift | 111 ++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index b52522c07586..59f2b600d2c6 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -5196,7 +5196,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, scheduleDetailRefresh(threadID: route.uiID, client: route.client, force: true) return } - let reduction = NativeThreadDetailReducer.apply(event, to: current) + let reduction = NativeThreadDetailReducer.apply( + event, to: current, afterSequence: activeThreadSequence ?? 0 + ) if reduction.sequence < 0 { threadHistoryEpoch &+= 1 detailSnapshotRequiredAfterEpoch = threadHistoryEpoch @@ -8379,7 +8381,8 @@ struct NativeThreadDetailReduction: Equatable { enum NativeThreadDetailReducer { static func apply( _ event: JSONValue, - to thread: OrchestrationThread + to thread: OrchestrationThread, + afterSequence: Int? = nil ) -> NativeThreadDetailReduction { guard case let .object(object) = event, let type = object["type"]?.stringValue, @@ -8394,6 +8397,13 @@ enum NativeThreadDetailReducer { ) } + // Validate ownership and the common envelope before skipping replayed events. + if let afterSequence, sequence >= 0, sequence <= afterSequence { + return NativeThreadDetailReduction( + sequence: sequence, result: .unchanged, renderMutation: .none + ) + } + let result: NativeThreadDetailReductionResult var renderMutation = NativeDetailRenderMutation.metadata switch type { diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index 9fe19f16868c..610fcddb2e5d 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -5,6 +5,117 @@ import XCTest @MainActor @available(iOS 18.0, *) final class NativeThreadCatchUpTests: XCTestCase { + func testStaleDetailReplaySkipsReductionOnlyAfterEnvelopeValidation() throws { + let thread = multiEnvironmentDetail( + projectID: "project", threadID: "first", snapshotSequence: 2, messages: [] + ).thread + let event = replayMessage(sequence: 2, text: "Duplicate") + let ordinary = NativeThreadDetailReducer.apply(event, to: thread) + guard case .updated = ordinary.result else { + return XCTFail("The control must exercise a real message reduction.") + } + let skipped = NativeThreadDetailReducer.apply(event, to: thread, afterSequence: 2) + XCTAssertEqual(skipped.sequence, 2) + guard case .unchanged = skipped.result, case .none = skipped.renderMutation else { + return XCTFail("A validated stale event must bypass message reduction.") + } + let newer = NativeThreadDetailReducer.apply(event, to: thread, afterSequence: 1) + guard case .updated = newer.result else { return XCTFail("New events must still reduce.") } + } + + func testStaleDetailReplayPreservesNewerMessageAndExplicitMarker() async throws { + let fixture = try await CatchUpFixture.make(completionMarker: false) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + var values = (0..<100).map { index in + JSONValue.object(["kind": .string("event"), "event": replayMessage( + sequence: Double(index % 3), text: "Must not appear" + )]) + } + // These valid envelopes were ignored by the existing post-reduction cursor guard. + for type in ["future.event", "thread.message-sent"] { + values.append(.object(["kind": .string("event"), "event": .object([ + "type": .string(type), "sequence": .number(2), + "occurredAt": .string("2026-09-02T12:00:00Z"), + "payload": .object(["threadId": .string("first")]), + ])])) + } + values.append(.object(["kind": .string("event"), "event": replayMessage( + sequence: 3, text: "New content" + )])) + values.append(.object(["kind": .string("synchronized")])) + try await stream.socket.chunk(id: stream.id, values: values) + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages, ["New content"]) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 1) + await fixture.client.disconnect() + } catch { + await fixture.client.disconnect() + throw error + } + } + + func testMalformedStaleLookingDetailEnvelopesStillRepair() async throws { + var malformed: [JSONValue] = [.null] + for field in ["type", "occurredAt"] { + var object = try replayMessage(sequence: 2, text: "Invalid").decode([String: JSONValue].self) + object.removeValue(forKey: field) + malformed.append(.object(object)) + } + var wrongThread = try replayMessage(sequence: 2, text: "Invalid").decode([String: JSONValue].self) + wrongThread["payload"] = .object(["threadId": .string("another-thread")]) + malformed.append(.object(wrongThread)) + malformed.append(replayMessage(sequence: 1.5, text: "Invalid")) + malformed.append(replayMessage(sequence: -1, text: "Invalid")) + var items = malformed.map { JSONValue.object(["kind": .string("event"), "event": $0]) } + items.append(.object(["kind": .string("unknown")])) + for (index, item) in items.enumerated() { + let fixture = try await CatchUpFixture.make(completionMarker: false) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Recovered \(index)", sequence: 10) + try await stream.socket.chunk(id: stream.id, values: [ + item, .object(["kind": .string("synchronized")]), + ]) + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages, ["Recovered \(index)"], "Malformed case \(index) must repair.") + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } catch { + await fixture.client.disconnect() + throw error + } + } + } + + private func replayMessage(sequence: Double, text: String) -> JSONValue { + .object([ + "type": .string("thread.message-sent"), "sequence": .number(sequence), + "occurredAt": .string("2026-09-02T12:00:00Z"), "payload": .object([ + "threadId": .string("first"), "messageId": .string("replay-message"), + "role": .string("assistant"), "text": .string(text), "streaming": .bool(false), + "createdAt": .string("2026-09-02T12:00:00Z"), + "updatedAt": .string("2026-09-02T12:00:00Z"), + ]), + ]) + } + func testLegacyReplayPublishesOncePerReceivedBatchAndResumesAfterAppliedEvents() async throws { for batchSize in [1, 16, 500] { let fixture = try await CatchUpFixture.make(completionMarker: false) From 8ef7e74c91a4a92494c5527448d0980ab30162d6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:46:02 +1000 Subject: [PATCH 14/19] fix(swift-ios): reconcile silent selected-thread streams --- apps/swift-ios/App/NativeFeatureClient.swift | 219 +++- .../NativeMultiEnvironmentTests.swift | 74 +- .../NativeThreadCatchUpTests.swift | 1006 ++++++++++++++++- 3 files changed, 1257 insertions(+), 42 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index 59f2b600d2c6..568229734bcb 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -21,6 +21,11 @@ private struct T3ConnectManagedCleanupError: LocalizedError { } } +enum NativeSelectedThreadReconciliationReceipt: Equatable { + case finished(threadID: String) + case deferred(threadID: String) +} + /// Composes the transport-focused Core layer with the UI-focused Features layer. @MainActor final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, @@ -61,6 +66,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private let threadSnapshotTimeoutInterval: TimeInterval private let detailPublicationSleep: @Sendable () async throws -> Void private let catchUpDelay: @Sendable () async throws -> Void + private let selectedThreadReconciliationSleep: @Sendable (Duration) async throws -> Void + private let selectedThreadReconciliationNow: @MainActor @Sendable () -> ContinuousClock.Instant + private let selectedThreadReconciliationReceipt: @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void private let threadRetryDelay: @Sendable (Int) async throws -> Void private let aggregateEnvironmentLoader: @Sendable (EnvironmentRuntime) async throws -> [Environment] private let stream: AsyncStream @@ -151,6 +159,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ] = [:] private var shellPublishTask: Task? private var archivedRefreshTask: Task? + private var selectedThreadReconciliationTask: Task? + private var selectedThreadReconciliationRefreshGeneration: Int? + private var selectedThreadLastProgressAt: ContinuousClock.Instant? private var detailRefreshTask: Task? private var detailStreamTask: Task? private var detailCatchUpTask: Task? @@ -207,6 +218,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, catchUpDelay: @escaping @Sendable () async throws -> Void = { try await Task.sleep(for: .seconds(2)) }, + selectedThreadReconciliationSleep: @escaping @Sendable (Duration) async throws -> Void = { + try await Task.sleep(for: $0) + }, + selectedThreadReconciliationNow: @escaping @MainActor @Sendable () -> ContinuousClock.Instant = { .now }, + selectedThreadReconciliationReceipt: @escaping @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void = { _ in }, threadRetryDelay: @escaping @Sendable (Int) async throws -> Void = { attempt in try await Task.sleep(for: .seconds(min(5, 0.25 * pow(2, Double(min(5, attempt - 1)))))) }, @@ -250,6 +266,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.threadSnapshotTimeoutInterval = threadSnapshotTimeoutInterval self.detailPublicationSleep = detailPublicationSleep self.catchUpDelay = catchUpDelay + self.selectedThreadReconciliationSleep = selectedThreadReconciliationSleep + self.selectedThreadReconciliationNow = selectedThreadReconciliationNow + self.selectedThreadReconciliationReceipt = selectedThreadReconciliationReceipt self.threadRetryDelay = threadRetryDelay self.aggregateEnvironmentLoader = aggregateEnvironmentLoader let pair = AsyncStream.makeStream() @@ -258,6 +277,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } deinit { + selectedThreadReconciliationTask?.cancel() activeHydrationTask?.cancel() pollingTask?.cancel() fallbackPollingTask?.cancel() @@ -289,7 +309,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } let environment = activeClient.environment - await adoptEnvironment(environment, client: activeClient) + guard try await adoptEnvironment(environment, client: activeClient) else { throw CancellationError() } guard bootstrapID == foregroundBootstrapID, isCurrentSession(client: activeClient, generation: environmentGeneration) else { throw CancellationError() @@ -430,6 +450,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func suspendForBackground() { + stopSelectedThreadReconciliation() isForeground = false beginForegroundBootstrap() } @@ -519,7 +540,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } else { pairedClient = try await runtime.pair(url: endpoint, clientLabel: "T3 Code Swift") } - await adoptEnvironment(pairedClient.environment, client: pairedClient) + guard try await adoptEnvironment(pairedClient.environment, client: pairedClient) else { throw CancellationError() } startAggregateRefresh(pairedClient) startPolling(pairedClient) } @@ -584,7 +605,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environment, credential: savedCredential ) - await adoptEnvironment(environment, client: managedClient) + guard try await adoptEnvironment(environment, client: managedClient) else { throw CancellationError() } do { try await refresh(client: managedClient) } catch { @@ -649,12 +670,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func setEnvironmentEnabled(id: String, enabled: Bool) async throws { + try await runtime.setEnabled(id: id, enabled: enabled) if !enabled { shellConnectionIDsByEnvironmentID[id] = nil aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() - } - try await runtime.setEnabled(id: id, enabled: enabled) - if !enabled { cancelAcceptedCommandRefreshes(environmentID: id) environmentConnectionStates[id] = .disconnected environmentConnectionDetails[id] = nil @@ -669,16 +688,16 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func removeEnvironment(id: String) async throws { - shellConnectionIDsByEnvironmentID[id] = nil - aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() let removesActiveEnvironment = activeEnvironment?.id == id let environment = try await runtime.environments().first { $0.id == id } if environment?.kind == .managedDPoP { try await runtime.revokeCredential(id: id) } - cancelAcceptedCommandRefreshes(environmentID: id) try await runtime.remove(id: id) saveGitHubRoutingGrants(gitHubRoutingGrants.filter { $0.environmentID != id }) + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() + cancelAcceptedCommandRefreshes(environmentID: id) if removesActiveEnvironment { await clearActiveEnvironment(disconnectClient: false) } @@ -1159,20 +1178,20 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func adoptEnvironment( _ environment: Environment, client newClient: T3Client - ) async { + ) async throws -> Bool { + let bootstrapID = foregroundBootstrapID + let generation = environmentGeneration + let adoptedConnectionID = await newClient.currentConnectionID() + let selectedClient = try await runtime.activeClient() + guard bootstrapID == foregroundBootstrapID, generation == environmentGeneration, + selectedClient === newClient, selectedClient?.environment == environment else { return false } cancelAggregateRefresh() if activeEnvironment?.id == environment.id, client === newClient { activeEnvironment = environment environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - return + return true } - let bootstrapID = foregroundBootstrapID - let generation = environmentGeneration - let adoptedConnectionID = await newClient.currentConnectionID() - let selectedEnvironment = try? await runtime.activeEnvironment() - guard bootstrapID == foregroundBootstrapID, generation == environmentGeneration, - selectedEnvironment == environment else { return } pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() @@ -1195,6 +1214,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, latestShell = shellsByEnvironmentID[environment.id] // Runtime owns shared transports. The former inbox client may now // serve a passive shell or a selected detail, so adoption does not close it. + return true } private func clearActiveEnvironment(disconnectClient: Bool = true) async { @@ -4859,11 +4879,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func scheduleDetailRefresh( threadID: String, client: T3Client, - force: Bool = false + force: Bool = false, + reconcile: Bool = false ) { guard activeThreadID == threadID, activeThreadEnvironmentID == client.environment.id else { return } - guard force || detailStreamTask == nil else { return } + guard force || reconcile || detailStreamTask == nil else { return } if force { detailWasSynchronized = false // This required read owns recovery now. An older fallback must not @@ -4881,11 +4902,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detailRefreshGeneration &+= 1 let generation = detailRefreshGeneration let sessionGeneration = environmentGeneration + let streamGeneration = detailStreamGeneration detailRefreshTask = Task { [weak self] in do { // Shell updates can be coalesced. A required replacement cannot // apply more thread events until its snapshot arrives. - if !force { + if !force && !reconcile { try await Task.sleep(for: .milliseconds(250)) } } catch { @@ -4901,11 +4923,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, generation: sessionGeneration ) { do { - try await self.refreshThread(id: threadID, client: client) + try await self.refreshThread( + id: threadID, client: client, + expectedStreamGeneration: reconcile ? streamGeneration : nil, + reconcile: reconcile + ) } catch is CancellationError { // Closing a thread cancels its read without changing its status. } catch { - if !Task.isCancelled, + if !reconcile, !Task.isCancelled, self.detailRefreshGeneration == generation, self.activeThreadID == threadID, self.activeRawThread == nil || self.detailStreamTask == nil { @@ -4930,6 +4956,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, id: route.uiID, state: warmConnectionID != nil ? .live : .catchingUp )) ensureDetailCatchUpFallback(route, generation: streamGeneration) + startSelectedThreadReconciliation(route, generation: streamGeneration) let retryDelay = threadRetryDelay detailStreamTask = Task { [weak self] in var failedAttempts = 0 @@ -5054,6 +5081,60 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, continuation.yield(.threadSync(id: route.uiID, state: .failed(message))) } + /// The selected transcript gets a bounded HTTP check even when its socket + /// remains healthy but subscription delivery stops. Other threads are untouched. + private func startSelectedThreadReconciliation(_ route: NativeThreadRoute, generation: Int) { + stopSelectedThreadReconciliation() + guard isForeground else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() + let sessionGeneration = environmentGeneration + let sleep = selectedThreadReconciliationSleep + selectedThreadReconciliationTask = Task { [weak self] in + while !Task.isCancelled { + let remaining = self.map { owner in + max(.zero, .seconds(30) - (owner.selectedThreadLastProgressAt ?? owner.selectedThreadReconciliationNow()) + .duration(to: owner.selectedThreadReconciliationNow())) + } ?? .seconds(30) + do { try await sleep(remaining) } catch { return } + guard !Task.isCancelled, let self, self.isForeground, + self.environmentGeneration == sessionGeneration, + self.isCurrentDetail(route, generation: generation) else { return } + if let progress = self.selectedThreadLastProgressAt, + progress.duration(to: self.selectedThreadReconciliationNow()) < .seconds(30) { + self.selectedThreadReconciliationReceipt(.deferred(threadID: route.uiID)) + continue + } + guard self.detailRefreshTask == nil, self.detailCatchUpTask == nil, + self.acceptedCommandRefreshes[route.uiID] == nil, + self.activeRawThread != nil, + self.activeThreadPage?.isLoading != true || self.pendingOlderThreadPage != nil else { + self.selectedThreadLastProgressAt = self.selectedThreadReconciliationNow() + self.selectedThreadReconciliationReceipt(.deferred(threadID: route.uiID)) + continue + } + self.scheduleDetailRefresh(threadID: route.uiID, client: route.client, reconcile: true) + self.selectedThreadReconciliationRefreshGeneration = self.detailRefreshGeneration + let refresh = self.detailRefreshTask + await refresh?.value + guard !Task.isCancelled, self.isForeground, + self.environmentGeneration == sessionGeneration, + self.isCurrentDetail(route, generation: generation) else { return } + self.selectedThreadReconciliationRefreshGeneration = nil + self.selectedThreadLastProgressAt = self.selectedThreadReconciliationNow() + self.selectedThreadReconciliationReceipt(.finished(threadID: route.uiID)) + } + } + } + + private func stopSelectedThreadReconciliation() { + selectedThreadReconciliationTask?.cancel() + selectedThreadReconciliationTask = nil + if selectedThreadReconciliationRefreshGeneration == detailRefreshGeneration { + resetDetailRefresh() + } + selectedThreadReconciliationRefreshGeneration = nil + } + private func isCurrentDetail(_ route: NativeThreadRoute, generation: Int) -> Bool { detailStreamGeneration == generation && activeThreadID == route.uiID @@ -5170,6 +5251,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, subscriptionEpoch < requiredEpoch { return } guard snapshot.snapshotSequence >= (activeThreadSequence ?? 0), activeRawThread == nil || snapshot.snapshotSequence > (activeThreadSequence ?? 0) else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() beginWarmReplayIfNeeded(route) resetDetailRefresh() detailSnapshotRequiredAfterEpoch = nil @@ -5209,6 +5291,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, return } guard reduction.sequence > (activeThreadSequence ?? 0) else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() beginWarmReplayIfNeeded(route) switch reduction.result { case let .updated(thread): @@ -5309,6 +5392,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func resetDetailStream() { + stopSelectedThreadReconciliation() detailStreamGeneration &+= 1 detailCompletionReceived = false detailWasSynchronized = false @@ -5680,7 +5764,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func refreshThread( - id: String, client: T3Client, expectedStreamGeneration: Int? = nil, + id: String, client: T3Client, expectedStreamGeneration: Int? = nil, reconcile: Bool = false, isSuperseded: (() -> Bool)? = nil ) async throws { let route = try threadRoute(for: id) @@ -5693,11 +5777,89 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let supportsPagination = serverConfigsByEnvironmentID[ environment.id ]?.threadSnapshotPagination == true - let snapshot = try await client.threadSnapshot( + let loadedUsers = reconcile ? activeRawThread?.messages.filter { $0.role == "user" } ?? [] : [] + let loadedAnchor = loadedUsers.first?.id + let pendingHistory = reconcile ? pendingOlderThreadPage : nil + if let pendingHistory { + guard pendingHistory.epoch == historyEpoch, pendingHistory.threadID == route.uiID, + pendingHistory.environmentID == route.environmentID, + (pendingHistory.snapshot.page?.threadSequence ?? 0) >= (activeThreadSequence ?? 0) else { return } + } + let loadedIDs = Set(loadedUsers.map(\.id)) + // Pending rows determine the requested extent only. The published data + // always comes from a new authoritative snapshot, never a cached union. + let pendingUsers = pendingHistory?.snapshot.thread.messages.filter { + $0.role == "user" && !loadedIDs.contains($0.id) + } ?? [] + let retainedUsers = pendingUsers + loadedUsers + let retainedIDs = Set(retainedUsers.map(\.id)) + let retainedPage = activeThreadPage + let retainedThreads = reconcile ? [activeRawThread, pendingHistory?.snapshot.thread].compactMap { $0 } : [] + let retainedMessageIDs = Set(retainedThreads.flatMap { $0.messages.map(\.id) }) + let retainedActivityIDs = Set(retainedThreads.flatMap { $0.activities.map(\.id) }) + let retainedCheckpointTurns = Set(retainedThreads.flatMap { $0.checkpoints.map(\.turnId) }) + let turnLimit = max(Self.initialThreadUserTurnLimit, retainedUsers.count) + + func ownsReconciliationExtent() -> Bool { + !reconcile || ( + pendingOlderThreadPage == pendingHistory + && threadHistoryEpoch == historyEpoch + && activeThreadPage == retainedPage + && (activeThreadPage?.isLoading != true || pendingHistory != nil) + && loadedAnchor == activeRawThread?.messages.first(where: { $0.role == "user" })?.id + ) + } + func coversRetainedHistory(_ thread: OrchestrationThread) -> Bool { + // A server page can hit its raw-turn cap before reaching even one + // user row. Preserve every loaded collection, not just user anchors. + retainedMessageIDs.isSubset(of: Set(thread.messages.map(\.id))) + && retainedActivityIDs.isSubset(of: Set(thread.activities.map(\.id))) + && retainedCheckpointTurns.isSubset(of: Set(thread.checkpoints.map(\.turnId))) + } + func additionalTailUsers(in thread: OrchestrationThread) -> Int? { + let users = thread.messages.filter { $0.role == "user" } + guard let lastKnown = users.lastIndex(where: { retainedIDs.contains($0.id) }) else { return nil } + return users.distance(from: users.index(after: lastKnown), to: users.endIndex) + } + var snapshot = try await client.threadSnapshot( id: route.wireID, - turnLimit: supportsPagination ? Self.initialThreadUserTurnLimit : nil, + turnLimit: supportsPagination ? turnLimit : nil, timeoutInterval: threadSnapshotTimeoutInterval ) + guard !Task.isCancelled, + expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true, + isKnownClient(client, environmentID: environment.id, generation: generation) else { + throw CancellationError() + } + guard ownsReconciliationExtent() else { return } + if reconcile, pendingHistory == nil, let current = activeRawThread, + snapshot.snapshotSequence >= (activeThreadSequence ?? 0), + snapshot.page?.threadSequence.map({ $0 <= (activeThreadSequence ?? 0) }) == true + || snapshot.snapshotSequence == activeThreadSequence || snapshot.thread == current { + return + } + if reconcile, snapshot.page?.hasMore == true, + !coversRetainedHistory(snapshot.thread) { + if let added = additionalTailUsers(in: snapshot.thread), added > 0 { + // One extra user turn needs one extra slot, not an automatic + // page of older history. Do not keep expanding a moving head. + snapshot = try await client.threadSnapshot( + id: route.wireID, turnLimit: turnLimit + added, + timeoutInterval: threadSnapshotTimeoutInterval + ) + guard !Task.isCancelled, ownsReconciliationExtent(), + expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { return } + guard additionalTailUsers(in: snapshot.thread) == added else { return } + } + if snapshot.page?.hasMore == true, + !coversRetainedHistory(snapshot.thread) { + // Missing retained rows or a raw-turn cap get one full read. Never + // resurrect old rows by unioning them into a rewound thread. + snapshot = try await client.threadSnapshot( + id: route.wireID, timeoutInterval: threadSnapshotTimeoutInterval + ) + } + } guard !Task.isCancelled, isKnownClient(client, environmentID: environment.id, generation: generation), expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { @@ -5705,6 +5867,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } // A newer accepted command makes this snapshot stale; its owner reads again. if isSuperseded?() == true { return } + guard ownsReconciliationExtent() else { return } + if let pendingWatermark = pendingHistory?.snapshot.page?.threadSequence, + snapshot.snapshotSequence < pendingWatermark { return } if activeThreadID == route.uiID { if activeRawThread == nil, historyEpoch != threadHistoryEpoch { return @@ -5747,7 +5912,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, wasSynchronized: false, connectionID: nil ) - if activeThreadID == route.uiID, + if !reconcile, activeThreadID == route.uiID, detailCompletionReceived || serverConfigsByEnvironmentID[environment.id]?.threadResumeCompletionMarker != true { markDetailSynchronized(route) @@ -9023,7 +9188,7 @@ private struct NativeProjectRoute { let client: T3Client } -private struct PendingOlderThreadPage { +private struct PendingOlderThreadPage: Equatable { let snapshot: OrchestrationThreadDetailSnapshot let epoch: Int let threadID: String diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index 7b48469a2790..f0a55a27aa47 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -1560,6 +1560,46 @@ struct NativePassiveThreadRefreshTests { await fixture.client.disconnect() } + @Test("Failed environment writes preserve the passive worker", arguments: [false, true]) + func failedEnvironmentWritePreservesWorker(removal: Bool) async throws { + let clock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + aggregatePeerRefreshSleep: { _, interval in try await clock.sleep(for: interval) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + _ = await clock.waitUntilRequested(count: 1) + let worker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let catalog = fixture.directory.appendingPathComponent("environments.json") + let data = try Data(contentsOf: catalog) + try FileManager.default.removeItem(at: catalog) + try FileManager.default.createDirectory(at: catalog, withIntermediateDirectories: false) + do { + if removal { try await fixture.client.removeEnvironment(id: "two") } + else { try await fixture.client.setEnvironmentEnabled(id: "two", enabled: false) } + Issue.record("The blocked catalog write unexpectedly succeeded") + } catch { + #expect(!worker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] != nil) + } + try FileManager.default.removeItem(at: catalog) + try data.write(to: catalog) + #expect(try await fixture.runtime.environments().contains { $0.id == "two" && $0.isEnabled }) + let probe = ThreadTitleEventProbe( + events: fixture.client.events(), + threadID: FeatureScopedID.thread(environmentID: "two", wireID: "thread-two"), + title: "Still updating" + ) + probe.start() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Still updating", snapshotSequence: 2 + ), host: "two.example") + await clock.resume() + await probe.waitUntilObserved() + #expect(probe.didObserveTitle()) + await fixture.client.disconnect() + } + @Test("Removal cancels a held peer and its late response never returns a row", .timeLimit(.minutes(1))) func removedPeerCannotPublishLateResponse() async throws { let removedClock = ControllableAggregateRefreshSleep() @@ -2803,7 +2843,9 @@ struct NativePassiveLiveShellTests { let nextWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) try await fixture.client.removeEnvironment(id: "two") await nextWorker.value - await server.waitForInterrupts(host: "two.example", count: 2) + // Successful removal closes the transport before cancelling the worker. + // There is no live socket on which to deliver another Interrupt frame. + #expect(await server.latestConnectionIsClosed(host: "two.example")) #expect(nextWorker.isCancelled) #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) let snapshot = try await fixture.client.backgroundSnapshot() @@ -3168,10 +3210,15 @@ private actor PassiveLiveServer: WebSocketConnecting { func subscriptionCount(host: String) -> Int { subscriptions[host, default: []].count } + func latestConnectionIsClosed(host: String) async -> Bool { + await subscriptions[host]?.last?.connection.isClosed ?? false + } + func closeLatest(host: String) async { await subscriptions[host]?.last?.connection.close() } } private actor PassiveLiveConnection: WebSocketConnection { + var isClosed: Bool { closed } private let host: String private let server: PassiveLiveServer private var responses: [Data] = [] @@ -3208,6 +3255,31 @@ private actor PassiveLiveConnection: WebSocketConnection { @Suite("Native incremental bootstrap") @MainActor struct NativeIncrementalBootstrapTests { + @Test("Environment-store failures remain visible to the root model") + func environmentStoreFailureIsNotCancellation() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let catalog = directory.appendingPathComponent("environments.json") + try Data("invalid catalog".utf8).write(to: catalog) + let runtime = EnvironmentRuntime(environmentStore: EnvironmentStore(fileURL: catalog), + credentialStore: InMemoryCredentialStore()) + let client = NativeFeatureClient(runtime: runtime) + do { + _ = try await client.initialSnapshot() + Issue.record("The invalid catalog unexpectedly loaded") + } catch { + #expect(error is DecodingError) + #expect(!(error is CancellationError)) + } + let model = FeatureRootModel(client: client, + outboxStore: FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")), + draftStore: FeatureComposerDraftStore(fileURL: directory.appendingPathComponent("drafts.json"))) + await model.reload() + #expect(model.errorMessage?.isEmpty == false) + await model.disconnect() + } + @Test("Pairing resets the previous environment's hydration and archive lifetime", .timeLimit(.minutes(1)), arguments: [false, true]) func pairingReplacesHydrationLifetime(previousHydrated: Bool) async throws { let server = PassiveLiveServer() diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index 610fcddb2e5d..7b017837146c 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -5,6 +5,804 @@ import XCTest @MainActor @available(iOS 18.0, *) final class NativeThreadCatchUpTests: XCTestCase { + func testSelectedReconciliationPreservesExplicitOlderRawFanoutBelowTenUsers() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300) + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.messages.count, 150) + XCTAssertEqual(initial.messages.filter { $0.role == .user }.count, 1) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let loaded = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(loaded?.messages.count, 300) + XCTAssertEqual(loaded?.messages.filter { $0.role == .user }.count, 2) + XCTAssertEqual(loaded?.messages.first?.text, "Raw 0") + XCTAssertEqual(loaded?.page?.hasMore, false) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.last?.text, "Changed raw tail") + XCTAssertEqual(value?.messages.count, 300, "Quiet repair must preserve explicitly loaded raw-turn history even below ten user rows.") + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertNotEqual(value?.page?.hasMore, true) + print("RAW_CAP_PROOF initial=\(initial.messages.count) loaded=\(loaded?.messages.count ?? -1) loadedUsers=\(loaded?.messages.filter { $0.role == .user }.count ?? -1) repaired=\(value?.messages.count ?? -1)") + print("RAW_CAP_REQUESTS \(await fixture.http.threadRequests.map { $0.url!.absoluteString })") + } + + func testSelectedReconciliationPreservesExplicitOlderRawFanoutWithZeroUsers() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.messages.count, 150) + XCTAssertEqual(initial.messages.filter { $0.role == .user }.count, 0) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let loaded = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(loaded?.messages.count, 300) + XCTAssertEqual(loaded?.messages.filter { $0.role == .user }.count, 0) + XCTAssertEqual(loaded?.messages.first?.text, "Raw 0") + XCTAssertEqual(loaded?.page?.hasMore, false) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.last?.text, "Changed raw tail") + XCTAssertEqual(value?.messages.count, 300, "Quiet repair must preserve explicitly loaded raw-turn history even below ten user rows.") + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertNotEqual(value?.page?.hasMore, true) + print("RAW_CAP_PROOF initial=\(initial.messages.count) loaded=\(loaded?.messages.count ?? -1) loadedUsers=\(loaded?.messages.filter { $0.role == .user }.count ?? -1) repaired=\(value?.messages.count ?? -1)") + print("RAW_CAP_REQUESTS \(await fixture.http.threadRequests.map { $0.url!.absoluteString })") + } + + func testSelectedReconciliationRetainsRawCollectionsAndAcceptsAuthoritativeDeletion() async throws { + for collection in ["activities", "checkpoints"] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: [], collection: collection) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + var raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(raw.messages.count, 0) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 300) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release(); _ = await completed.next() + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 300) + var urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, collection == "activities" ? 4 : 3) + XCTAssertEqual(urls.last!.contains("turnLimit="), collection == "checkpoints", + "Only windowed activities need full fallback for an ordinary changed tail.") + await fixture.http.removePaginatedUser(id: "raw-0", sequence: 302) + let pendingDelete = await ticks.next(); let deleteTick = try XCTUnwrap(pendingDelete) + clock.advance(by: .seconds(30)); deleteTick.release(); _ = await completed.next() + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) + XCTAssertFalse(raw.activities.contains { $0.id == "activity-raw-0" }) + XCTAssertFalse(raw.checkpoints.contains { $0.turnId == "turn-raw-0" }) + urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, collection == "activities" ? 6 : 5) + XCTAssertFalse(urls.last!.contains("turnLimit=")) + let pendingUnchanged = await ticks.next(); let unchangedTick = try XCTUnwrap(pendingUnchanged) + clock.advance(by: .seconds(30)); unchangedTick.release(); _ = await completed.next() + let unchangedURLs = await fixture.http.threadRequests + XCTAssertEqual(unchangedURLs.count, collection == "activities" ? 7 : 6, "Unchanged capped recent reads must not repeat full fallback.") + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) + } + } + + private func rawThreadForHistoryTest(_ client: NativeFeatureClient) throws -> OrchestrationThread { + // Read-only test inspection: checkpoints are retained raw state and have + // no standalone published collection in FeatureThreadDetail. + try XCTUnwrap(Mirror(reflecting: client).children.first { $0.label == "activeRawThread" }?.value as? OrchestrationThread) + } + + func testSelectedReconciliationRepairsSilentDetailWithoutSyncFlicker() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Quiet HTTP repair", sequence: 20) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + var messages: [String] = [] + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): messages = value.messages.map(\.text) + case .threadSync(fixture.firstID, .live): break + case .threadSync(fixture.firstID, .catchingUp): XCTFail("Quiet repair must not flash catch-up.") + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(messages, ["Quiet HTTP repair"]) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } + + func testSelectedReconciliationUnchangedSnapshotPublishesNothing() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + var publications = 0 + while let event = await events.next(isolation: #isolation) { + if case .detail = event { publications += 1 } + if case .detailDelta = event { publications += 1 } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(publications, 0) + await fixture.client.disconnect() + } + + func testSelectedReconciliationActualProgressPostponesButMarkerDoesNot() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pendingFirst = await ticks.next(); let first = try XCTUnwrap(pendingFirst) + clock.advance(by: .seconds(20)) + try await detail.sendMessage(text: "Live progress", sequence: 3) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + clock.advance(by: .seconds(10)); first.release() + let deferred = await completed.next() + XCTAssertEqual(deferred, .deferred(threadID: fixture.firstID)) + let pendingNext = await ticks.next(); let next = try XCTUnwrap(pendingNext) + XCTAssertEqual(next.duration, .seconds(20)) + clock.advance(by: .seconds(10)) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + clock.advance(by: .seconds(10)); next.release() + let finished = await completed.next() + XCTAssertEqual(finished, .finished(threadID: fixture.firstID)) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } + + func testSelectedReconciliationRejectsOlderHeldHTTPAfterNewerDetail() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Old HTTP", sequence: 3) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + try await detail.sendMessage(text: "Newer live response", sequence: 4) + try await detail.synchronize() + let live = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(live, ["Newer live response"]) + held.succeed() + _ = await completed.next() + try await detail.synchronize() + let replaced = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertTrue(replaced.isEmpty, "Older HTTP must not republish over newer detail.") + await fixture.client.disconnect() + } + + func testSelectedReconciliationBackgroundCancelsItsHeldRead() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + fixture.client.suspendForBackground() + held.succeed() + let cancelled = await held.finished.first { _ in true } + XCTAssertEqual(cancelled, true) + await fixture.client.disconnect() + } + + func testSelectedReconciliationRequiredReadKeepsItsFailureAfterQuietRead() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + try await detail.invalidate(sequence: 3) + await nextCatchUp(&events, threadID: fixture.firstID) + quiet.succeed() + let required = try await nextHeldRead(&reads) + required.fail() + var failure: String? + while let event = await events.next(isolation: #isolation) { + if case let .threadSync(id, .failed(message)) = event, id == fixture.firstID { + failure = message; break + } + } + XCTAssertEqual(failure, URLError(.notConnectedToInternet).localizedDescription) + await fixture.client.disconnect() + } + + private func retainForSelectedReconciliationTest(_ fixture: CatchUpFixture, clock: SelectedReconciliationClock? = nil) { + addTeardownBlock { + await MainActor.run { clock?.cancelAll() } + await fixture.http.cancelHeldReads() + await fixture.client.disconnect() + await MainActor.run { fixture.cleanUp() } + } + } + + func testSelectedReconciliationPreservesLoadedHistoryWhenUnchanged() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let older = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(older?.messages.count, 30) + let retainedPage = older?.page + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let unchanged = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertTrue(unchanged.isEmpty, "An unchanged enlarged window must not publish extra history.") + let earlier = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(retainedPage?.beforeCursor, "user-20") + XCTAssertEqual(earlier?.messages.count, 50) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("beforeCursor=user-20")) + } + + func testSelectedReconciliationExpandedGapAndDeletedBoundaryUseAuthoritativeFullRead() async throws { + for deletedBoundary in [false, true] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let lower = deletedBoundary ? 21 : 0 + await fixture.http.setPaginatedUserMessages(lower..<110, sequence: 200) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages.first, "User \(lower)") + XCTAssertEqual(messages.last, "User 109") + XCTAssertEqual(messages.count, 110 - lower) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertFalse(urls[3].contains("turnLimit=")) + } + } + + func testSelectedReconciliationUnchangedReadDoesNotInvalidatePendingOlderPage() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed() + _ = await completed.next() + olderRead.succeed() + let older = try await olderTask.value + XCTAssertEqual(older?.messages.count, 30) + XCTAssertEqual(older?.page?.beforeCursor, "user-20") + XCTAssertEqual(older?.page?.isLoading, false) + } + + func testSelectedReconciliationNavigationRejectsHeldResponse() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Must not arrive after navigation", sequence: 20) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + fixture.client.releaseThread(id: fixture.firstID) + await fixture.http.holdThreadReads(false) + _ = try await fixture.client.loadThread(id: fixture.secondID) + _ = try await nextThreadRequest(&requests) + held.succeed() + let cancelled = await held.finished.first { _ in true } + XCTAssertEqual(cancelled, true) + fixture.client.releaseThread(id: fixture.secondID) + let restored = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertFalse(restored.messages.contains { $0.text == "Must not arrive after navigation" }) + } + + func testSelectedReconciliationUsesSelectedPassivePeerRoute() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + includePeer: true, reconciliationClock: clock, + reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + let peerID = FeatureScopedID.thread(environmentID: "two", wireID: "first") + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: peerID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: peerID) + await fixture.http.setResponse(text: "Peer HTTP repair", sequence: 20) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: peerID)) + try await detail.synchronize() + let messages = await messagesBeforeLive(&events, threadID: peerID) + XCTAssertEqual(messages, ["Peer HTTP repair"]) + let reads = await fixture.http.threadRequests + XCTAssertEqual(reads.count, 2) + XCTAssertTrue(reads.allSatisfy { $0.url?.host == "two.example" }) + } + + func testSelectedReconciliationDefaultDeadlineRepairsMessageAndWorkingState() async throws { + let fixture = try await CatchUpFixture.make() + retainForSelectedReconciliationTest(fixture) + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.thread.state, .idle) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setRunningResponse(text: "Default deadline repair", sequence: 20) + let start = ContinuousClock.now + var repaired: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + if value.thread.id == fixture.firstID, + value.messages.contains(where: { $0.text == "Default deadline repair" }) { + repaired = value + } + case .threadSync(fixture.firstID, .catchingUp): XCTFail("Quiet recovery must not flash catch-up.") + default: break + } + if repaired != nil { break } + } + let elapsed = start.duration(to: .now) + XCTAssertEqual(repaired?.thread.state, .working) + XCTAssertGreaterThanOrEqual(elapsed, .seconds(29)) + XCTAssertLessThan(elapsed, .seconds(40)) + let reads = await fixture.http.threadRequests + XCTAssertEqual(reads.count, 2) + print("DEFAULT_RECONCILIATION elapsed=\(elapsed) state=\(String(describing: repaired?.thread.state)) reads=\(reads.count) bytes=\(await fixture.http.threadResponseBytes)") + } + + func testSelectedReconciliationRepeatedTailEditsKeepExplicitHistoryExtent() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + for edit in 1...3 { + await fixture.http.editLatestUserText("Tail edit \(edit)", sequence: 100 + edit) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 70") + XCTAssertEqual(value?.messages.last?.text, "Tail edit \(edit)") + XCTAssertEqual(value?.page?.beforeCursor, "user-70") + } + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 5) + XCTAssertTrue(urls.dropFirst(2).allSatisfy { $0.contains("turnLimit=30") }) + } + + func testSelectedReconciliationNewUserAddsOnlyRequiredWindowExtent() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 31) + XCTAssertEqual(value?.messages.first?.text, "User 70") + XCTAssertEqual(value?.messages.last?.text, "User 100") + XCTAssertEqual(value?.page?.beforeCursor, "user-70") + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + if urls.count > 3 { XCTAssertTrue(urls[3].contains("turnLimit=31")) } + } + + func testSelectedReconciliationUserlessPendingAheadPageRecoversWithoutStreamProgress() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("New head", sequence: 301) + let pendingPage = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(pendingPage?.page?.isLoading, true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 300) + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertEqual(value?.messages.last?.text, "New head") + XCTAssertNotEqual(value?.page?.isLoading, true) + } + + func testSelectedReconciliationPendingAheadPageRecoversWithoutStreamProgress() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("New head", sequence: 51) + let pendingPage = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(pendingPage?.page?.isLoading, true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 20") + XCTAssertEqual(value?.messages.last?.text, "New head") + XCTAssertEqual(value?.page?.isLoading, false) + } + + func testSelectedReconciliationUserlessChangedQuietReadYieldsToOlderPageRequest() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed head", sequence: 301) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed(); _ = await completed.next() + olderRead.succeed() + let waitingOlder = try await olderTask.value + XCTAssertEqual(waitingOlder?.page?.isLoading, true, "Optional quiet read must not invalidate the user's older-page epoch.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let nextTick = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); nextTick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 300) + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertEqual(value?.messages.last?.text, "Changed head") + XCTAssertNotEqual(value?.page?.isLoading, true) + } + + func testSelectedReconciliationChangedQuietReadYieldsToOlderPageRequest() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed head", sequence: 51) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed(); _ = await completed.next() + olderRead.succeed() + let waitingOlder = try await olderTask.value + XCTAssertEqual(waitingOlder?.page?.isLoading, true, "Optional quiet read must not invalidate the user's older-page epoch.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let nextTick = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); nextTick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 20") + XCTAssertEqual(value?.messages.last?.text, "Changed head") + XCTAssertEqual(value?.page?.isLoading, false) + } + + private func selectedDetailBeforeLive(_ events: inout AsyncStream.Iterator, threadID: String) async -> FeatureThreadDetail? { + var value: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(detail), let .detailDelta(detail, _): + if detail.thread.id == threadID { value = detail } + case .threadSync(threadID, .live): return value + default: break + } + } + return value + } + + func testSelectedReconciliationMovingAdaptiveHeadDefersWithoutFullRead() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let first = try await nextHeldRead(&reads) + await fixture.http.setPaginatedUserMessages(0..<102, sequence: 102) + first.succeed() + let adaptive = try await nextHeldRead(&reads) + adaptive.succeed(); _ = await completed.next() + var urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("turnLimit=31")) + try await detail.synchronize() + let unchanged = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertNil(unchanged, "A moving second read must not widen or replace the retained range.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let next = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); next.release(); _ = await completed.next() + try await detail.synchronize() + let repaired = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired?.messages.count, 32) + XCTAssertEqual(repaired?.messages.first?.text, "User 70") + XCTAssertEqual(repaired?.messages.last?.text, "User 101") + urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 6) + XCTAssertTrue(urls[5].contains("turnLimit=32")) + } + + func testSelectedReconciliationDeletedAnchorAfterAdaptiveReadUsesOneFullFallback() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + await fixture.http.removePaginatedUser(id: "user-70", sequence: 102) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release(); _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 100) + XCTAssertFalse(value?.messages.contains(where: { $0.text == "User 70" }) ?? true) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 5) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("turnLimit=31")) + XCTAssertFalse(urls[4].contains("turnLimit=")) + } + func testStaleDetailReplaySkipsReductionOnlyAfterEnvelopeValidation() throws { let thread = multiEnvironmentDetail( projectID: "project", threadID: "first", snapshotSequence: 2, messages: [] @@ -1536,6 +2334,10 @@ private struct CatchUpFixture { static func make( completionMarker: Bool? = true, + includePeer: Bool = false, + reconciliationClock: SelectedReconciliationClock? = nil, + reconciliationReceipt: @escaping @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void = { _ in }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, activities: [OrchestrationActivity] = [], detailPublicationSleep: @escaping @Sendable () async throws -> Void = { try await Task.sleep(for: .milliseconds(80)) @@ -1549,7 +2351,10 @@ private struct CatchUpFixture { try await store.save([Environment( id: "one", label: "Computer", httpBaseURL: URL(string: "https://one.example")!, webSocketBaseURL: URL(string: "wss://one.example/ws")! - )]) + )] + (includePeer ? [Environment( + id: "two", label: "Peer", httpBaseURL: URL(string: "https://two.example")!, + webSocketBaseURL: URL(string: "wss://two.example/ws")! + )] : [])) try await store.setActiveEnvironment(id: "one") let http = CatchUpHTTPTransport() await http.setActivities(activities) @@ -1557,24 +2362,40 @@ private struct CatchUpFixture { let delay = CatchUpDelay() let runtime = EnvironmentRuntime( environmentStore: store, - credentialStore: InMemoryCredentialStore(credentials: ["one": .init(accessToken: "test")]), + credentialStore: InMemoryCredentialStore(credentials: ["one": .init(accessToken: "test"), "two": .init(accessToken: "test")]), httpTransport: http, webSocketConnector: CatchUpConnector( requests: requests.continuation, completionMarker: completionMarker ) ) let readiness = CatchUpBootstrapReadiness() + let peerReadiness = CatchUpBootstrapReadiness() let client = NativeFeatureClient( runtime: runtime, settingsStore: UserDefaults(suiteName: UUID().uuidString)!, fallbackPollingInitialDelay: .seconds(3_600), aggregateRefreshInterval: .seconds(3_600), - aggregateRefreshReceipt: { readiness.record($0) }, + aggregateRefreshReceipt: { + readiness.record($0) + switch $0 { + case let .shellApplied("two", sequence): peerReadiness.record(.shellApplied(environmentID: "one", sequence: sequence)) + case .configurationApplied("two"): peerReadiness.record(.configurationApplied(environmentID: "one")) + default: break + } + aggregateRefreshReceipt($0) + }, detailPublicationSleep: detailPublicationSleep, catchUpDelay: { try await delay.wait() }, + selectedThreadReconciliationSleep: { duration in + if let reconciliationClock { try await reconciliationClock.sleep(for: duration) } + else { try await Task.sleep(for: duration) } + }, + selectedThreadReconciliationNow: { reconciliationClock?.now ?? .now }, + selectedThreadReconciliationReceipt: reconciliationReceipt, threadRetryDelay: threadRetryDelay ) _ = try await client.initialSnapshot() try await readiness.wait() + if includePeer { try await peerReadiness.wait() } return Self(client: client, http: http, requests: requests.stream, delay: delay, directory: directory) } @@ -1583,10 +2404,16 @@ private struct CatchUpFixture { private actor CatchUpHTTPTransport: HTTPTransport { private(set) var threadRequests: [URLRequest] = [] + private(set) var threadResponseBytes: [Int] = [] private var messages: [OrchestrationMessage] = [] + private var paginatedMessages: [OrchestrationMessage]? + private var rawTurnCap: Int? + private var rawCollection: String? + private var pendingHeldReads: [CatchUpHTTPRead] = [] private var activities: [OrchestrationActivity] = [] private var completionResponse = false private var responsePage: OrchestrationThreadDetailPage? + private var runningResponse = false private var sequence = 2 private var holdsThreadReads = false private let heldReadContinuation: AsyncStream.Continuation @@ -1598,6 +2425,47 @@ private actor CatchUpHTTPTransport: HTTPTransport { heldReadContinuation = reads.continuation } + func setRawFanoutMessages(sequence: Int, userSlots: Set = [0, 200], collection: String? = nil) { + rawTurnCap = 150 + rawCollection = collection + paginatedMessages = (0..<300).map { index in + let stamp = String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60) + return OrchestrationMessage(id: "raw-\(index)", role: userSlots.contains(index) ? "user" : "assistant", + text: "Raw \(index)", attachments: [], turnId: userSlots.contains(index) ? nil : "turn-\(index)", + streaming: false, createdAt: stamp, updatedAt: stamp) + } + self.sequence = sequence + } + + func setPaginatedUserMessages(_ range: Range, sequence: Int) { + paginatedMessages = range.map { index in + OrchestrationMessage( + id: "user-\(index)", role: "user", text: "User \(index)", attachments: [], + turnId: nil, streaming: false, + createdAt: String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60), + updatedAt: String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60) + ) + } + self.sequence = sequence + } + func removePaginatedUser(id: String, sequence: Int) { + paginatedMessages?.removeAll { $0.id == id } + self.sequence = sequence + } + func editLatestUserText(_ text: String, sequence: Int) { + guard let latest = paginatedMessages?.popLast() else { return } + paginatedMessages?.append(OrchestrationMessage( + id: latest.id, role: latest.role, text: text, attachments: latest.attachments, + turnId: latest.turnId, streaming: latest.streaming, createdAt: latest.createdAt, + updatedAt: latest.updatedAt + )) + self.sequence = sequence + } + func cancelHeldReads() { + pendingHeldReads.forEach { $0.fail() } + pendingHeldReads.removeAll() + } + func setResponse(text: String, sequence: Int, attachment: ChatAttachment? = nil) { messages = [catchUpMessage(text, index: 0, attachment: attachment)] self.sequence = sequence @@ -1610,6 +2478,11 @@ private actor CatchUpHTTPTransport: HTTPTransport { self.sequence = sequence } + func setRunningResponse(text: String, sequence: Int) { + setResponse(text: text, sequence: sequence) + runningResponse = true + } + func setCompletionResponse(text: String, sequence: Int, streaming: Bool = false) { messages = [OrchestrationMessage( id: "answer-0", role: "assistant", text: text, attachments: [], turnId: "turn-1", @@ -1643,24 +2516,77 @@ private actor CatchUpHTTPTransport: HTTPTransport { throw URLError(.unsupportedURL) } threadRequests.append(request) + var responseMessages = messages + var page: OrchestrationThreadDetailPage? + if let paginatedMessages { + let query = URLComponents(url: request.url!, resolvingAgainstBaseURL: false)?.queryItems ?? [] + let limit = query.first { $0.name == "turnLimit" }?.value.flatMap(Int.init) + let cursor = query.first { $0.name == "beforeCursor" }?.value + let end = cursor.flatMap { value in paginatedMessages.firstIndex { $0.id == value } } ?? paginatedMessages.count + var start = max(0, end - (limit ?? end)) + if let cap = rawTurnCap, let limit { + // One fixture message per raw turn. Match server candidates LIMIT150, + // then walk descending until the user-anchored turn limit is reached. + start = max(0, end - cap) + var usersSeen = 0 + for index in stride(from: end - 1, through: start, by: -1) { + if paginatedMessages[index].role == "user" { usersSeen += 1 } + if usersSeen == limit { start = index; break } + } + } + responseMessages = Array(paginatedMessages[start.. 0, + snapshotSequence: sequence, threadSequence: sequence + ) + } + } let snapshot = multiEnvironmentDetail( projectID: "project", threadID: request.url!.lastPathComponent, - snapshotSequence: sequence, messages: messages + snapshotSequence: sequence, messages: responseMessages ) var thread = snapshot.thread thread.activities = activities + if let rawCollection { + thread.messages = [] + if rawCollection == "activities" { + thread.activities = responseMessages.map { row in + OrchestrationActivity(id: "activity-\(row.id)", tone: "info", kind: "fixture.raw-turn", + summary: row.text, payload: .null, turnId: "turn-\(row.id)", sequence: nil, createdAt: row.createdAt) + } + } else { + // Server checkpoints are unwindowed even when messages/activities + // hit the raw-turn cap. Keep this fixture faithful to that contract. + thread.checkpoints = (paginatedMessages ?? responseMessages).enumerated().map { offset, row in + CheckpointSummary(turnId: "turn-\(row.id)", checkpointTurnCount: offset, + checkpointRef: row.text, status: "completed", files: [], assistantMessageId: nil, completedAt: row.createdAt) + } + } + } + if runningResponse { + var object = try JSONValue.encode(thread).decode([String: JSONValue].self) + object["latestTurn"] = .object([ + "turnId": .string("synthetic-turn"), "state": .string("running"), + "requestedAt": .string("2026-09-02T12:00:00Z"), "startedAt": .string("2026-09-02T12:00:00Z"), + "completedAt": .null, "assistantMessageId": .null + ]) + object["session"] = catchUpRunningSession(threadID: thread.id, turnID: "synthetic-turn") + thread = try JSONValue.object(object).decode(OrchestrationThread.self) + } if completionResponse { var object = try JSONValue.encode(thread).decode([String: JSONValue].self) object["latestTurn"] = catchUpCompletedTurn(assistantMessageID: "answer-0") thread = try JSONValue.object(object).decode(OrchestrationThread.self) } value = try .encode(OrchestrationThreadDetailSnapshot( - snapshotSequence: snapshot.snapshotSequence, thread: thread, page: responsePage ?? snapshot.page + snapshotSequence: snapshot.snapshotSequence, thread: thread, page: responsePage ?? page ?? snapshot.page )) } let response = (try JSONEncoder.t3.encode(value), HTTPURLResponse( url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: nil )!) + if request.url!.path.hasPrefix("/api/orchestration/threads/") { threadResponseBytes.append(response.0.count) } if holdsThreadReads, request.url!.path.hasPrefix("/api/orchestration/threads/") { let finished = AsyncStream.makeStream() defer { @@ -1668,21 +2594,30 @@ private actor CatchUpHTTPTransport: HTTPTransport { finished.continuation.finish() } return try await withCheckedThrowingContinuation { continuation in - heldReadContinuation.yield(CatchUpHTTPRead( - response: response, continuation: continuation, finished: finished.stream - )) + let read = CatchUpHTTPRead(response: response, continuation: continuation, finished: finished.stream) + pendingHeldReads.append(read) + heldReadContinuation.yield(read) } } return response } } -private struct CatchUpHTTPRead: Sendable { +private final class CatchUpHTTPRead: @unchecked Sendable { let response: (Data, HTTPURLResponse) - let continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error> + private var continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error>? + private let lock = NSLock() let finished: AsyncStream - func succeed() { continuation.resume(returning: response) } - func fail() { continuation.resume(throwing: URLError(.notConnectedToInternet)) } + init(response: (Data, HTTPURLResponse), continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error>, finished: AsyncStream) { + self.response = response; self.continuation = continuation; self.finished = finished + } + private func take() -> CheckedContinuation<(Data, HTTPURLResponse), any Error>? { + lock.lock(); defer { lock.unlock() } + defer { continuation = nil } + return continuation + } + func succeed() { take()?.resume(returning: response) } + func fail() { take()?.resume(throwing: URLError(.notConnectedToInternet)) } } private func catchUpMessage( @@ -1702,7 +2637,7 @@ private struct CatchUpConnector: WebSocketConnecting { let requests: AsyncStream.Continuation let completionMarker: Bool? func connect(to url: URL) async throws -> any WebSocketConnection { - CatchUpSocket(requests: requests, completionMarker: completionMarker) + CatchUpSocket(requests: requests, completionMarker: completionMarker, publishInitialShell: url.host == "two.example") } } @@ -1875,9 +2810,12 @@ private actor CatchUpSocket: WebSocketConnection { private var receiver: CheckedContinuation? private var closed = false - init(requests: AsyncStream.Continuation, completionMarker: Bool?) { + private let publishInitialShell: Bool + + init(requests: AsyncStream.Continuation, completionMarker: Bool?, publishInitialShell: Bool = false) { self.requests = requests self.completionMarker = completionMarker + self.publishInitialShell = publishInitialShell } func send(_ data: Data) throws { @@ -1887,6 +2825,12 @@ private actor CatchUpSocket: WebSocketConnection { try enqueue(.object(["_tag": .string("Pong")])) } guard let tag = request["tag"]?.stringValue, case let .number(id) = request["id"] else { return } + if publishInitialShell, tag == RPCMethod.subscribeShell.rawValue { + let snapshot = multiEnvironmentShell(projectID: "project", threadID: "first", title: "Peer") + try chunk(id: Int(id), values: [.object([ + "kind": .string("snapshot"), "snapshot": try .encode(snapshot) + ])]) + } if tag == RPCMethod.assetsCreateURL.rawValue { assetRequestCount += 1 } if tag == RPCMethod.subscribeServerConfig.rawValue { var config: [String: JSONValue] = [ @@ -2089,3 +3033,37 @@ private final class CatchUpBootstrapReadiness { } } } +@MainActor +private final class SelectedReconciliationClock { + var now = ContinuousClock.now + let requests = AsyncStream.makeStream() + private var pending: [UUID: CheckedContinuation] = [:] + struct Wait: Sendable { + let duration: Duration + let release: @MainActor @Sendable () -> Void + } + func advance(by duration: Duration) { now = now.advanced(by: duration) } + func cancelAll() { + let waits = pending.values + pending.removeAll() + waits.forEach { $0.resume(throwing: CancellationError()) } + requests.continuation.finish() + } + func sleep(for duration: Duration) async throws { + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else { + pending[id] = continuation + requests.continuation.yield(Wait(duration: duration, release: { [weak self] in + self?.pending.removeValue(forKey: id)?.resume() + })) + } + } + pending.removeValue(forKey: id) + } onCancel: { Task { @MainActor [weak self] in + self?.pending.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } } + } +} From 4238100683e30d769e31e8bf1e91a3732979e70e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:37:18 +1000 Subject: [PATCH 15/19] fix(swift-ios): retain selected-thread ownership across navigation --- apps/swift-ios/App/NativeFeatureClient.swift | 82 +++++++-- .../Features/Chat/ThreadDetailView.swift | 30 ++-- .../Features/Root/FeatureRootModel.swift | 58 ++++++ .../Workspace/WorkspaceThreadSelection.swift | 25 +++ .../Features/Workspace/WorkspaceView.swift | 60 +++++-- .../FeatureTests/FeatureRootModelTests.swift | 166 ++++++++++++++++++ .../NativeMultiEnvironmentTests.swift | 131 ++++++++++++++ .../NativeThreadCatchUpTests.swift | 60 ++++++- .../WorkspaceThreadPresentationTests.swift | 33 ++++ 9 files changed, 596 insertions(+), 49 deletions(-) create mode 100644 apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift create mode 100644 apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index 568229734bcb..bc11d8752b2d 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -54,6 +54,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private let projectFaviconStore: FeatureProjectFaviconStore private let fallbackPollingInitialDelay: Duration private let fallbackPollingInterval: Duration + private static let shellReconciliationInterval: Duration = .seconds(30) + private let shellReconciliationSleep: @Sendable (String, Duration) async throws -> Void private let aggregateRefreshInterval: Duration private let aggregateIdleRefreshInterval: Duration private let aggregateFailureRefreshInterval: Duration @@ -149,6 +151,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var activeStreamIsAuthoritative = false private var pollingTask: Task? private var fallbackPollingTask: Task? + private var shellReconciliationTask: Task? private var configurationTask: Task? private var aggregateRefreshTask: Task? private var aggregateRefreshID: UUID? @@ -194,6 +197,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, projectFaviconStore: FeatureProjectFaviconStore = FeatureProjectFaviconStore(), fallbackPollingInitialDelay: Duration = .seconds(3), fallbackPollingInterval: Duration = .seconds(2), + shellReconciliationSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, aggregateRefreshInterval: Duration = NativeFeatureClient.defaultAggregateRefreshInterval, aggregateIdleRefreshInterval: Duration = NativeFeatureClient.defaultAggregateIdleRefreshInterval, aggregateFailureRefreshInterval: Duration = NativeFeatureClient.defaultAggregateFailureRefreshInterval, @@ -254,6 +260,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.projectFaviconStore = projectFaviconStore self.fallbackPollingInitialDelay = fallbackPollingInitialDelay self.fallbackPollingInterval = fallbackPollingInterval + self.shellReconciliationSleep = shellReconciliationSleep self.aggregateRefreshInterval = aggregateRefreshInterval self.aggregateIdleRefreshInterval = aggregateIdleRefreshInterval self.aggregateFailureRefreshInterval = aggregateFailureRefreshInterval @@ -281,6 +288,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, activeHydrationTask?.cancel() pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() aggregateRefreshTask?.cancel() aggregatePublishTask?.cancel() @@ -346,6 +354,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, activeStreamIsAuthoritative = false pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() archivedRefreshTask?.cancel() archivedRefreshTask = nil @@ -398,7 +407,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.activeHydrationPending = false self.acceptActiveShell(shell, client: activeClient, environments: environments, socketAuthoritative: false, connectHeader: false) - } else { + } else if !self.activeStreamIsAuthoritative { self.emitConnection( self.activeHasHydrated ? .reconnecting : .disconnected, detail: "Server unreachable. Retrying automatically." @@ -1194,10 +1203,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil + shellReconciliationTask = nil configurationTask = nil aggregateRefreshTask = nil aggregateRefreshID = nil @@ -1222,11 +1233,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let previousClient = client pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() cancelAggregateRefresh() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil + shellReconciliationTask = nil configurationTask = nil aggregateRefreshTask = nil aggregateRefreshID = nil @@ -4053,6 +4066,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func startPolling(_ activeClient: T3Client) { pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() let generation = environmentGeneration let bootstrapID = foregroundBootstrapID @@ -4167,11 +4181,25 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, if !self.activeStreamIsAuthoritative { self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) } - do { - try await Task.sleep(for: fallbackPollingInterval) - } catch { - return - } + do { try await Task.sleep(for: fallbackPollingInterval) } + catch { return } + } + } + let reconciliationSleep = shellReconciliationSleep + shellReconciliationTask = Task { [weak self] in + while !Task.isCancelled { + // Ping health cannot prove shell completeness. Keep the faster + // outage fallback independent of this quiet reconciliation. + do { try await reconciliationSleep(activeClient.environment.id, Self.shellReconciliationInterval) } + catch { return } + guard let self, + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) + else { return } + guard self.activeStreamIsAuthoritative else { continue } + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + // One owner coalesces all HTTP requests. The next interval starts + // after this read drains, including failures and supersession. + await self.activeHydrationTask?.value } } configurationTask = Task { [weak self] in @@ -4312,6 +4340,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let shellTimeout: TimeInterval let sleep: @Sendable (Duration) async throws -> Void let peerSleep: @Sendable (String, Duration) async throws -> Void + let reconciliationSleep: @Sendable (String, Duration) async throws -> Void let loadEnvironments: @Sendable (EnvironmentRuntime) async throws -> [Environment] let streamRetrySleep: @Sendable (String, Duration) async throws -> Void let publishSleep: @Sendable () async throws -> Void @@ -4331,6 +4360,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, shellTimeout = owner.environmentShellTimeoutInterval sleep = owner.aggregateRefreshSleep peerSleep = owner.aggregatePeerRefreshSleep + reconciliationSleep = owner.shellReconciliationSleep loadEnvironments = owner.aggregateEnvironmentLoader streamRetrySleep = owner.aggregateStreamRetrySleep publishSleep = owner.aggregatePublishSleep @@ -4401,19 +4431,27 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, var interval = fastInterval while owns(environment) { do { - if state.isLive { await state.waitForRepair() } - if !state.needsHTTP { + let wasLive = state.isLive + if wasLive || !state.needsHTTP { let delay = interval await withTaskGroup(of: Void.self) { group in - group.addTask { try? await self.peerSleep(environment.id, delay) } - group.addTask { await state.waitForRepair() } + group.addTask { + if wasLive { + try? await self.reconciliationSleep(environment.id, NativeFeatureClient.shellReconciliationInterval) + } else { + try? await self.peerSleep(environment.id, delay) + } + } + group.addTask { await state.waitForRepair(untilLive: !wasLive) } _ = await group.next() group.cancelAll() } } guard owns(environment) else { return } + // A newly authoritative snapshot starts the quiet cadence; + // it must not turn a fallback wake-up into a duplicate read. + if !wasLive && state.isLive { continue } state.needsHTTP = false - guard !state.isLive else { continue } guard try await currentEnvironments(for: environment) != nil else { return } let client = await runtime.client(for: environment) let epoch = state.epoch @@ -4429,8 +4467,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let currentConnection = await client.currentConnectionID() guard let environments = try await currentEnvironments(for: environment) else { return } guard owns(environment), epoch == state.epoch, - authority == state.authorityRevision, connectionID == currentConnection, - !state.isLive else { continue } + authority == state.authorityRevision, connectionID == currentConnection else { continue } + // HTTP failure alone says nothing about the connected stream. + if shell == nil && state.isLive { continue } if let error = hydrationError, error.isRejectedAuthorization { owner?.applyEnvironmentLoad(EnvironmentShellLoad( environment: environment, client: client, shell: nil, config: nil, @@ -4531,8 +4570,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, owner.shellConnectionIDsByEnvironmentID[environment.id] = state.connectionID state.authorityRevision &+= 1 state.cacheEpoch = state.epoch + let wasLive = state.isLive state.isLive = true state.needsHTTP = false + if !wasLive { state.wakeWaiter() } case .synchronized: // A completion marker cannot make an arbitrary cached shell authoritative. return state.isLive @@ -4682,16 +4723,20 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func requestRepair() { needsHTTP = true + wakeWaiter() + } + + func wakeWaiter() { waiter?.continuation.resume() waiter = nil } - func waitForRepair() async { - guard !needsHTTP else { return } + func waitForRepair(untilLive: Bool = false) async { + guard !needsHTTP, !(untilLive && isLive) else { return } let id = UUID() await withTaskCancellationHandler { await withCheckedContinuation { continuation in - if Task.isCancelled || needsHTTP { continuation.resume() } + if Task.isCancelled || needsHTTP || (untilLive && isLive) { continuation.resume() } else { waiter = (id, continuation) } } } onCancel: { @@ -5832,10 +5877,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, throw CancellationError() } guard ownsReconciliationExtent() else { return } + // A later delivered cursor does not prove that every earlier row arrived. + // Independent HTTP repair must compare content, even at an equal watermark. if reconcile, pendingHistory == nil, let current = activeRawThread, snapshot.snapshotSequence >= (activeThreadSequence ?? 0), - snapshot.page?.threadSequence.map({ $0 <= (activeThreadSequence ?? 0) }) == true - || snapshot.snapshotSequence == activeThreadSequence || snapshot.thread == current { + snapshot.thread == current { return } if reconcile, snapshot.page?.hasMore == true, diff --git a/apps/swift-ios/Features/Chat/ThreadDetailView.swift b/apps/swift-ios/Features/Chat/ThreadDetailView.swift index 6483b15c0ef4..95dad3b0885b 100644 --- a/apps/swift-ios/Features/Chat/ThreadDetailView.swift +++ b/apps/swift-ios/Features/Chat/ThreadDetailView.swift @@ -14,6 +14,7 @@ public struct ThreadDetailView: View { let submitMessage: (FeatureMessageSubmission) async -> Bool let onNavigateBack: () -> Void private let draftStore: FeatureComposerDraftStore + private let managesThreadPresentation: Bool @State private var draft = "" @State private var composerContext: OrchestrationMessageContext? @@ -52,20 +53,22 @@ public struct ThreadDetailView: View { thread: FeatureThread, submitMessage: @escaping (FeatureMessageSubmission) async -> Bool, onNavigateBack: @escaping () -> Void = {}, - draftStore: FeatureComposerDraftStore = .shared + draftStore: FeatureComposerDraftStore = .shared, + managesThreadPresentation: Bool = true ) { self.model = model self.thread = thread self.submitMessage = submitMessage self.onNavigateBack = onNavigateBack self.draftStore = draftStore + self.managesThreadPresentation = managesThreadPresentation } private var threadContent: some View { Group { if let detail { timeline(detail) - } else if isLoading { + } else if isOpening { FeatureThreadOpeningView() } else { ContentUnavailableView { @@ -102,9 +105,11 @@ public struct ThreadDetailView: View { } } .task(id: thread.id) { + guard managesThreadPresentation else { return } isLoading = true - _ = await model.detail(for: thread.id, force: true) - isLoading = false + await model.runThreadPresentation(id: thread.id) { + isLoading = false + } } .task(id: thread.id) { // A cached thread can already show its composer while the server @@ -133,7 +138,7 @@ public struct ThreadDetailView: View { .onChange(of: threadConnectionState) { _, state in if state == .connected, case .failed = model.detailLoadStates[thread.id], - !isLoading { + !isOpening { reloadThread() } } @@ -143,7 +148,6 @@ public struct ThreadDetailView: View { } } .onDisappear { - model.releaseThread(thread.id) persistDraftBeforeLeaving() } .sheet(item: $toolSurface) { surface in @@ -277,6 +281,12 @@ public struct ThreadDetailView: View { } } + private var isOpening: Bool { + if managesThreadPresentation { return isLoading } + if case .failed = model.detailLoadStates[thread.id] { return false } + return detail == nil || model.detailLoadStates[thread.id] == .loading + } + private var detail: FeatureThreadDetail? { model.details[thread.id] } @@ -683,10 +693,8 @@ public struct ThreadDetailView: View { } private func reloadThread() { - isLoading = true - Task { - _ = await model.detail(for: thread.id, force: true, fresh: true) - isLoading = false + if model.refreshThreadPresentation(id: thread.id, onLoaded: { isLoading = false }) { + isLoading = true } } @@ -699,7 +707,7 @@ public struct ThreadDetailView: View { ThreadRefreshPresentation.resolve( loadState: model.detailLoadStates[thread.id], connectionState: threadConnectionState, - isOpening: isLoading, + isOpening: isOpening, syncState: model.threadSyncStates[thread.id] ) } diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index c9fb47fc99d0..3ad36c6bc102 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -99,6 +99,10 @@ public final class FeatureRootModel { private var pendingSettlementMutations: [String: PendingSettlementMutation] = [:] private var pendingCompletionSubmissionIDs: Set = [] private var pendingDiscardSubmissionIDs: Set = [] + @ObservationIgnored private var activeDetailPresentation: ( + threadID: String, owner: UUID, lifetime: AsyncStream.Continuation, + refreshTask: Task? + )? private var detailRecency: [String] = [] private var detailLoadGeneration: UInt64 = 0 private var detailLoadRevisions: [String: UInt64] = [:] @@ -726,6 +730,55 @@ public final class FeatureRootModel { } } + /// Holds selected-thread transport for the owning presentation task, including after loading. + func runThreadPresentation(id: String, onLoaded: @MainActor () -> Void = {}) async { + guard !Task.isCancelled else { return } + if let previous = activeDetailPresentation { + releaseThread(previous.threadID) + } + let owner = UUID() + let lifetime = AsyncStream.makeStream() + activeDetailPresentation = (id, owner, lifetime.continuation, nil) + await withTaskCancellationHandler { + defer { releaseThreadPresentation(id: id, owner: owner) } + _ = await detail(for: id, force: true) + guard !Task.isCancelled, + activeDetailPresentation?.owner == owner else { return } + onLoaded() + for await _ in lifetime.stream {} + } onCancel: { + // Cancellation must release even when the detail read is still suspended. + Task { @MainActor [weak self] in + self?.releaseThreadPresentation(id: id, owner: owner) + } + } + } + + /// Retries belong to the current presentation and cannot outlive a close or replacement. + @discardableResult + func refreshThreadPresentation( + id: String, fresh: Bool = true, onLoaded: @escaping @MainActor () -> Void = {} + ) -> Bool { + guard let presentation = activeDetailPresentation, presentation.threadID == id else { return false } + presentation.refreshTask?.cancel() + let owner = presentation.owner + activeDetailPresentation?.refreshTask = Task { [weak self] in + guard let self, !Task.isCancelled, + self.activeDetailPresentation?.owner == owner else { return } + _ = await self.detail(for: id, force: true, fresh: fresh) + guard !Task.isCancelled, + self.activeDetailPresentation?.owner == owner else { return } + onLoaded() + } + return true + } + + private func releaseThreadPresentation(id: String, owner: UUID) { + guard activeDetailPresentation?.threadID == id, + activeDetailPresentation?.owner == owner else { return } + releaseThread(id) + } + public func detail(for id: String, force: Bool = false, fresh: Bool = false) async -> FeatureThreadDetail? { if !force, let cached = details[id] { return cached @@ -804,6 +857,11 @@ public final class FeatureRootModel { /// Ends any selected-thread transport work when its detail view closes. public func releaseThread(_ id: String) { + if let presentation = activeDetailPresentation, presentation.threadID == id { + activeDetailPresentation = nil + presentation.refreshTask?.cancel() + presentation.lifetime.finish() + } client.releaseThread(id: id) markDetailRecentlyUsed(id) evictOldThreadDetailsIfNeeded() diff --git a/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift b/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift new file mode 100644 index 000000000000..70c524635174 --- /dev/null +++ b/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift @@ -0,0 +1,25 @@ +import Foundation + +struct WorkspaceThreadSelection: Equatable { + private(set) var selectedID: String? + private(set) var lastOpenedID: String? + + var highlightedID: String? { selectedID ?? lastOpenedID } + + mutating func open(_ id: String) { + selectedID = id + lastOpenedID = id + } + + func presentedID(isCompact: Bool, showsDetailColumn: Bool) -> String? { + isCompact && !showsDetailColumn ? nil : selectedID + } + + mutating func reconcilePresentation(isCompact: Bool, showsDetailColumn: Bool) { + if isCompact && !showsDetailColumn { close() } + } + + mutating func close() { + selectedID = nil + } +} diff --git a/apps/swift-ios/Features/Workspace/WorkspaceView.swift b/apps/swift-ios/Features/Workspace/WorkspaceView.swift index 2a0d3a448149..9e13c977dda4 100644 --- a/apps/swift-ios/Features/Workspace/WorkspaceView.swift +++ b/apps/swift-ios/Features/Workspace/WorkspaceView.swift @@ -18,24 +18,9 @@ struct FeatureWorkspaceNavigationRequest: Equatable, Sendable { } } -struct WorkspaceThreadSelection: Equatable { - private(set) var selectedID: String? - private(set) var lastOpenedID: String? - - var highlightedID: String? { selectedID ?? lastOpenedID } - - mutating func open(_ id: String) { - selectedID = id - lastOpenedID = id - } - - mutating func close() { - selectedID = nil - } -} - public struct WorkspaceView: View { @SwiftUI.Environment(\.dynamicTypeSize) private var dynamicTypeSize + @SwiftUI.Environment(\.horizontalSizeClass) private var horizontalSizeClass @Bindable var model: FeatureRootModel private let navigationRequest: FeatureWorkspaceNavigationRequest? @@ -81,8 +66,8 @@ public struct WorkspaceView: View { self.submitMessage = { submission in await model.sendMessage(submission) } } - public var body: some View { - NavigationSplitView(preferredCompactColumn: $preferredCompactColumn) { + private var workspaceNavigation: some View { + NavigationSplitView(preferredCompactColumn: compactColumnBinding) { sidebar .navigationSplitViewColumnWidth( min: T3Metrics.minimumSidebarWidth, @@ -93,6 +78,17 @@ public struct WorkspaceView: View { detail } .navigationSplitViewStyle(.balanced) + .task(id: presentedThreadID) { [id = presentedThreadID] in + guard let id else { return } + await model.runThreadPresentation(id: id) + } + .onChange(of: horizontalSizeClass) { _, _ in + reconcileThreadPresentation() + } + } + + public var body: some View { + workspaceNavigation .sheet(isPresented: $showingNewTask) { NewThreadView( model: model, @@ -297,7 +293,8 @@ public struct WorkspaceView: View { model: model, thread: thread, submitMessage: submitMessage, - onNavigateBack: closeSelectedThread + onNavigateBack: closeSelectedThread, + managesThreadPresentation: false ) .id(id) } else { @@ -600,6 +597,31 @@ public struct WorkspaceView: View { return model.snapshot.projects.contains { $0.id == selectedProjectID } } + private var compactColumnBinding: Binding { + Binding( + get: { preferredCompactColumn }, + set: { column in + preferredCompactColumn = column + reconcileThreadPresentation() + } + ) + } + + private var presentedThreadID: String? { + guard let id = threadSelection.presentedID( + isCompact: horizontalSizeClass == .compact, + showsDetailColumn: preferredCompactColumn == .detail + ), model.snapshot.threads.contains(where: { $0.id == id }) else { return nil } + return id + } + + private func reconcileThreadPresentation() { + threadSelection.reconcilePresentation( + isCompact: horizontalSizeClass == .compact, + showsDetailColumn: preferredCompactColumn == .detail + ) + } + private func openThread(_ id: String) { threadSelection.open(id) preferredCompactColumn = .detail diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 7c3aacb4ec9f..9e0704ff076e 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -9,6 +9,163 @@ import XCTest @MainActor @Suite("Feature root model") struct FeatureRootModelTests { + @Test(arguments: [["a", "b", "a"], ["a", "a"]]) + func presentationTaskReplacementAndCloseReleaseExactlyOnce(threadIDs: [String]) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + var loads = loaded.stream.makeAsyncIterator() + var previous: Task? + for id in threadIDs { + let next = Task { + await model.runThreadPresentation(id: id) { loaded.continuation.yield(id) } + } + #expect(await loads.next() == id) + // Replacement finishes the old lifetime even before SwiftUI cancels its task. + await previous?.value + previous?.cancel() + previous = next + } + #expect(client.loadedThreadIDs == threadIDs) + #expect(client.releasedThreadIDs == Array(threadIDs.dropLast())) + previous?.cancel() + await previous?.value + #expect(client.releasedThreadIDs == threadIDs) + loaded.continuation.finish() + } + + @Test + func cancelledPresentationTaskCannotReplaceVisibleThread() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let current = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + let cancelled = Task { @MainActor in + withUnsafeCurrentTask { $0?.cancel() } + await model.runThreadPresentation(id: "a") + } + await cancelled.value + #expect(client.loadedThreadIDs == ["a"]) + #expect(client.releasedThreadIDs.isEmpty) + current.cancel() + await current.value + #expect(client.releasedThreadIDs == ["a"]) + loaded.continuation.finish() + } + + @Test(arguments: [false, true]) + func cancellationReleasesSuspendedPresentationBeforeItsLoadReturns(cancelBeforeReplacement: Bool) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let started = AsyncStream.makeStream() + let released = AsyncStream.makeStream() + let loaded = AsyncStream.makeStream() + client.onReleaseThread = { released.continuation.yield($0) } + var pendingLoad: CheckedContinuation? + client.loadThreadHandler = { _ in + try await withCheckedThrowingContinuation { continuation in + pendingLoad = continuation + started.continuation.yield(()) + } + } + var oldLoaded = false + let old = Task { await model.runThreadPresentation(id: "a") { oldLoaded = true } } + var starts = started.stream.makeAsyncIterator() + await starts.next() + var releases = released.stream.makeAsyncIterator() + if cancelBeforeReplacement { + old.cancel() + #expect(await releases.next() == "a") + } + #expect(pendingLoad != nil) + #expect(!oldLoaded) + + client.loadThreadHandler = nil + let replacement = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + if !cancelBeforeReplacement { + // The old cancellation handler runs only after the same-ID replacement owns transport. + old.cancel() + #expect(await releases.next() == "a") + } + pendingLoad?.resume(returning: FeatureThreadDetail(thread: client.createdThread)) + pendingLoad = nil + await old.value + #expect(!oldLoaded) + #expect(client.releasedThreadIDs == ["a"]) + replacement.cancel() + await replacement.value + #expect(client.releasedThreadIDs == ["a", "a"]) + #expect(client.loadedThreadIDs == ["a", "a"]) + started.continuation.finish() + released.continuation.finish() + loaded.continuation.finish() + } + + @Test + func presentationCloseStillReleasesAfterConnectionReload() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + await model.reloadAfterConnection() + presentation.cancel() + await presentation.value + #expect(client.releasedThreadIDs == ["a"]) + loaded.continuation.finish() + } + + @Test + func presentationCloseCancelsRetryAndRejectsLateRetry() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let started = AsyncStream.makeStream() + let cancelled = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + var pendingRetry: CheckedContinuation? + client.loadThreadHandler = { _ in + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + pendingRetry = continuation + started.continuation.yield(()) + } + } onCancel: { + cancelled.continuation.yield(()) + } + } + #expect(model.refreshThreadPresentation(id: "a")) + var retries = started.stream.makeAsyncIterator() + await retries.next() + presentation.cancel() + await presentation.value + var cancellations = cancelled.stream.makeAsyncIterator() + await cancellations.next() + #expect(client.releasedThreadIDs == ["a"]) + #expect(!model.refreshThreadPresentation(id: "a")) + pendingRetry?.resume(throwing: CancellationError()) + pendingRetry = nil + #expect(client.loadedThreadIDs == ["a", "a"]) + loaded.continuation.finish() + started.continuation.finish() + cancelled.continuation.finish() + } + @Test func rewindLocksSendingAndSavesRecoveredInputAfterLeavingTheThread() async throws { let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) @@ -3924,6 +4081,14 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { try await prepareRecovery(.init(message: message, attachments: rewindAttachments)) _ = try await rewindHandler(threadID, messageID) } + + var loadedThreadIDs: [String] = [] + var releasedThreadIDs: [String] = [] + var onReleaseThread: ((String) -> Void)? + func releaseThread(id: String) { + releasedThreadIDs.append(id) + onReleaseThread?(id) + } var foregroundReconnects: [Bool] = [] var backgroundSuspends = 0 func suspendForBackground() { backgroundSuspends += 1 } @@ -4104,6 +4269,7 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { } func loadThread(id: String, fresh: Bool) async throws -> FeatureThreadDetail { + loadedThreadIDs.append(id) if let loadThreadError { throw loadThreadError } diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index f0a55a27aa47..10ff9f34149f 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -1157,6 +1157,9 @@ final class NativeMultiEnvironmentTests: XCTestCase { rpcConnectionWaitTimeout: Duration = .milliseconds(5), fallbackPollingInitialDelay: Duration = .seconds(3), fallbackPollingInterval: Duration = .seconds(2), + shellReconciliationSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, aggregateRefreshInterval: Duration = NativeFeatureClient.defaultAggregateRefreshInterval, aggregateIdleRefreshInterval: Duration = NativeFeatureClient.defaultAggregateIdleRefreshInterval, aggregateFailureRefreshInterval: Duration = NativeFeatureClient.defaultAggregateFailureRefreshInterval, @@ -1277,6 +1280,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { settingsStore: settings, fallbackPollingInitialDelay: fallbackPollingInitialDelay, fallbackPollingInterval: fallbackPollingInterval, + shellReconciliationSleep: shellReconciliationSleep, aggregateRefreshInterval: aggregateRefreshInterval, aggregateIdleRefreshInterval: aggregateIdleRefreshInterval, aggregateFailureRefreshInterval: aggregateFailureRefreshInterval, @@ -2646,6 +2650,90 @@ private actor GatedPassiveCatalogueConnection: WebSocketConnection { @Suite("Native passive live shells") @MainActor struct NativePassiveLiveShellTests { + @Test("Silent connected shells reconcile through bounded HTTP", arguments: ["one", "two"], [false, true]) + func silentConnectedShellReconciles(environmentID: String, disconnect: Bool) async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clock = SilentShellClock() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .zero, + fallbackPollingInterval: .milliseconds(1), + shellReconciliationSleep: { id, interval in + if id == environmentID { try await clock.sleep(interval) } + else { try await Task.sleep(for: interval) } + }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + let host = environmentID + ".example" + let threadID = "thread-" + environmentID + try await receipts.waitForHTTP(environmentID, count: 1) + await server.waitForSubscriptions(host: host, count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "Live seed", snapshotSequence: 10 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + let reads = await fixture.transport.shellReadCount(host: host) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "HTTP repaired silent shell", snapshotSequence: 11 + ), host: host) + let held = PassiveRequestGate() + let operation = Task { @MainActor in + try await clock.waitForRequest() + #expect(await clock.intervals == [.seconds(30)]) + #expect(await fixture.transport.shellReadCount(host: host) == reads) + await clock.advance() + _ = try await recorder.wait { $0.threads.contains { $0.title == "HTTP repaired silent shell" } } + #expect(await fixture.transport.shellReadCount(host: host) == reads + 1) + try await clock.waitForRequest(2) + await fixture.transport.holdNextShell(host: host, gate: held) + await clock.advance() + try await held.waitUntilEnteredCancellable() + #expect(await clock.intervals.count == 2) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 2) + let completions = receipts.httpCount(environmentID) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "Newer live shell", snapshotSequence: 20 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Newer live shell" } } + await held.release() + try await receipts.waitForHTTP(environmentID, count: completions + 1) + try await clock.waitForRequest(3) + #expect(recorder.history.last?.threads.contains { $0.title == "Newer live shell" } == true) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 2) + // A failed quiet check neither disconnects the live stream nor + // retries until another full reconciliation interval has elapsed. + await fixture.transport.setShellReadsEnabled(false, host: host) + await clock.advance() + try await clock.waitForRequest(4) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 3) + #expect(recorder.history.last?.environments.first { $0.id == environmentID }?.connectionState == .connected) + #expect(await clock.intervals == Array(repeating: .seconds(30), count: 4)) + if disconnect { await fixture.client.disconnect() } + else { fixture.client.suspendForBackground() } + try await clock.waitForCancellation() + #expect(await fixture.transport.shellReadCount(host: host) == reads + 3) + } + // Failure watchdog only; successful verification advances an explicit clock. + let watchdog = Task { + try? await Task.sleep(for: .seconds(3)) + if !Task.isCancelled { operation.cancel() } + } + do { try await operation.value } + catch { Issue.record("Silent connected shell never scheduled or applied its HTTP repair: \(error)") } + watchdog.cancel() + await held.release() + recorder.stop() + await fixture.client.disconnect() + } + @Test("Live peer bursts publish together without polling", .timeLimit(.minutes(1))) func livePeerBurstsPublishTogetherWithoutPolling() async throws { let server = PassiveLiveServer() @@ -3890,3 +3978,46 @@ private actor FailedBootstrapAttemptConnector: WebSocketConnecting { try Task.checkCancellation() } } + +private actor SilentShellClock { + private(set) var intervals: [Duration] = [] + private let cancelled = PassiveRequestGate() + private var continuation: CheckedContinuation? + private var requestWaiters: [UUID: (Int, CheckedContinuation)] = [:] + + func sleep(_ interval: Duration) async throws { + intervals.append(interval) + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else { + self.continuation = continuation + let ready = requestWaiters.filter { intervals.count >= $0.value.0 } + for (id, waiter) in ready { + requestWaiters[id] = nil + waiter.1.resume() + } + } + } + } onCancel: { Task { await self.cancel() } } + } + func waitForRequest(_ count: Int = 1) async throws { + try Task.checkCancellation() + guard intervals.count < count else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { requestWaiters[id] = (count, $0) } + } onCancel: { Task { await self.cancelWaiter(id) } } + } + private func cancelWaiter(_ id: UUID) { + requestWaiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + func advance() { continuation?.resume(); continuation = nil } + func waitForCancellation() async throws { try await cancelled.waitUntilEnteredCancellable() } + func cancel() async { + continuation?.resume(throwing: CancellationError()) + continuation = nil + await cancelled.release() + await cancelled.enter() + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index 7b017837146c..678d7a5c6761 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -5,6 +5,51 @@ import XCTest @MainActor @available(iOS 18.0, *) final class NativeThreadCatchUpTests: XCTestCase { + func testFaultInjectedOmittedMessageRepairsAtEqualWatermark() async throws { + // Deliberate application-delivery omission, not simulated TCP packet loss. + // HTTP is authoritative for both messages; only event 4 reaches the client. + for paginated in [false, true] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, + reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + // The missing event would have sequence 3; later event 4 is valid. + try await detail.sendMessage(text: "Later delivered message", sequence: 4) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let before = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(before.messages.map(\.id), ["message-4"]) + await fixture.http.setFaultInjectionMessages(paginated: paginated) + for attempt in 1...2 { + let pendingTick = await ticks.next() + let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)) + tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + let raw = try rawThreadForHistoryTest(fixture.client) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, attempt + 1) + print("FAULT_INJECTED_EQUAL_WATERMARK paginated=\(paginated) attempt=\(attempt) reads=\(reads) messageIDs=\(raw.messages.map(\.id))") + XCTAssertEqual(Set(raw.messages.map(\.id)), Set(["message-3", "message-4"]), + "An authoritative equal-watermark read must repair omitted message 3; attempt \(attempt).") + } + await fixture.client.disconnect() + } + } + + func testSelectedReconciliationPreservesExplicitOlderRawFanoutBelowTenUsers() async throws { let clock = SelectedReconciliationClock() let receipts = AsyncStream.makeStream() @@ -115,7 +160,7 @@ final class NativeThreadCatchUpTests: XCTestCase { let pendingUnchanged = await ticks.next(); let unchangedTick = try XCTUnwrap(pendingUnchanged) clock.advance(by: .seconds(30)); unchangedTick.release(); _ = await completed.next() let unchangedURLs = await fixture.http.threadRequests - XCTAssertEqual(unchangedURLs.count, collection == "activities" ? 7 : 6, "Unchanged capped recent reads must not repeat full fallback.") + XCTAssertEqual(unchangedURLs.count, collection == "activities" ? 8 : 6, "A matching cursor cannot certify omitted rows outside a capped recent window.") raw = try rawThreadForHistoryTest(fixture.client) XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) } @@ -2403,6 +2448,19 @@ private struct CatchUpFixture { } private actor CatchUpHTTPTransport: HTTPTransport { + func setFaultInjectionMessages(paginated: Bool) { + let authoritative = [3, 4].map { index in + OrchestrationMessage(id: "message-\(index)", role: "assistant", + text: index == 3 ? "Omitted message" : "Later delivered message", + attachments: [], turnId: nil, streaming: false, + createdAt: "2026-09-02T12:00:00Z", updatedAt: "2026-09-02T12:00:00Z") + } + messages = authoritative + paginatedMessages = paginated ? authoritative : nil + sequence = 4 + } + + private(set) var threadRequests: [URLRequest] = [] private(set) var threadResponseBytes: [Int] = [] private var messages: [OrchestrationMessage] = [] diff --git a/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift b/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift new file mode 100644 index 000000000000..3ae6755bf29b --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift @@ -0,0 +1,33 @@ +import Foundation +import Testing +@testable import T3Code + +struct WorkspaceThreadPresentationTests { + @Test(arguments: [false, true], [false, true]) + func presentationTracksCompactColumnWithoutHidingRegularDetail(isCompact: Bool, showsDetail: Bool) { + var selection = WorkspaceThreadSelection() + selection.open("a") + let isHidden = isCompact && !showsDetail + #expect(selection.presentedID(isCompact: isCompact, showsDetailColumn: showsDetail) == (isHidden ? nil : "a")) + selection.reconcilePresentation(isCompact: isCompact, showsDetailColumn: showsDetail) + #expect(selection.selectedID == (isHidden ? nil : "a")) + #expect(selection.highlightedID == "a") + } + + @Test + func compactBackAndRegularToCompactHideCannotResurrectClosedThread() { + var selection = WorkspaceThreadSelection() + selection.open("a") + selection.reconcilePresentation(isCompact: false, showsDetailColumn: false) + #expect(selection.selectedID == "a") + selection.reconcilePresentation(isCompact: true, showsDetailColumn: false) + #expect(selection.selectedID == nil) + selection.reconcilePresentation(isCompact: false, showsDetailColumn: false) + #expect(selection.presentedID(isCompact: false, showsDetailColumn: false) == nil) + #expect(selection.highlightedID == "a") + selection.open("a") + #expect(selection.presentedID(isCompact: true, showsDetailColumn: true) == "a") + selection.close() + #expect(selection.presentedID(isCompact: true, showsDetailColumn: true) == nil) + } +} From 30626ead8b893a0845092f82f2dca2b0c2a601ea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 17:51:59 +1000 Subject: [PATCH 16/19] fix(swift-ios): finish draft restore when a reopened thread view is dismissed early Reopening a thread in the compact split view can report the new thread view as disappeared right after it appears while it stays on screen. That cancelled the draft restore, so the composer showed a busy send button until relaunch. Restore outside the view task's lifetime. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Features/Chat/ThreadDetailView.swift | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/swift-ios/Features/Chat/ThreadDetailView.swift b/apps/swift-ios/Features/Chat/ThreadDetailView.swift index 95dad3b0885b..d6a545b7f4f2 100644 --- a/apps/swift-ios/Features/Chat/ThreadDetailView.swift +++ b/apps/swift-ios/Features/Chat/ThreadDetailView.swift @@ -114,9 +114,18 @@ public struct ThreadDetailView: View { .task(id: thread.id) { // A cached thread can already show its composer while the server // is catching up. Local drafts must not wait for that request. - await model.checkRewindRecovery(for: currentThread) - guard !didRestoreDraft else { return } - await restoreDraft(from: composerDraft, key: draftKey) + // When a thread is reopened in the compact split view, SwiftUI can + // report this view as disappeared right after it appears while it + // stays on screen. That cancels this task and it never re-runs, so + // the composer would stay busy. Finish the restore outside the + // view's task lifetime; each thread has its own view identity, so + // a late restore cannot reach another thread's composer. + let restore = Task { @MainActor in + await model.checkRewindRecovery(for: currentThread) + guard !didRestoreDraft else { return } + await restoreDraft(from: composerDraft, key: draftKey) + } + await restore.value } .task(id: pullRequestObservationID) { await observeThreadPullRequest() From d8cf546daa5d225de466d051768110a43b671850 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:37:49 +1000 Subject: [PATCH 17/19] fix(swift-ios): stop quiet shell reconciliation after pairing is rejected Pairing rejection cancelled the polling tasks but left the reconciliation loop issuing rejected shell reads every 30 seconds. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index bc11d8752b2d..c43e9b4d7dd1 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -5595,9 +5595,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() + // Rejected credentials cannot hydrate; stop the quiet shell reads too. + shellReconciliationTask?.cancel() pollingTask = nil fallbackPollingTask = nil configurationTask = nil + shellReconciliationTask = nil + activeStreamIsAuthoritative = false if activeEnvironment?.kind == .managedDPoP { emitConnection(.disconnected, detail: detail) } else { From c2d24486787b2e4869d1848d4a4e1656f7223f93 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 19:49:21 +1000 Subject: [PATCH 18/19] fix(swift-ios): back off live peers whose credential is rejected A passive peer whose socket stayed open kept its 30-second quiet shell reads after a 401, and showed as connected. Handle the rejected credential before the live-stream shortcut and switch to the long back-off. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 11 +++- .../NativeMultiEnvironmentTests.swift | 59 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index c43e9b4d7dd1..104724e7b43c 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -4429,14 +4429,17 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func refreshShell(_ environment: Environment, state: PassiveShellState) async { var interval = fastInterval + // A rejected credential outlives the open socket; stop the quiet cadence. + var credentialRejected = false while owns(environment) { do { let wasLive = state.isLive if wasLive || !state.needsHTTP { let delay = interval + let reconciles = wasLive && !credentialRejected await withTaskGroup(of: Void.self) { group in group.addTask { - if wasLive { + if reconciles { try? await self.reconciliationSleep(environment.id, NativeFeatureClient.shellReconciliationInterval) } else { try? await self.peerSleep(environment.id, delay) @@ -4468,9 +4471,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard let environments = try await currentEnvironments(for: environment) else { return } guard owns(environment), epoch == state.epoch, authority == state.authorityRevision, connectionID == currentConnection else { continue } - // HTTP failure alone says nothing about the connected stream. - if shell == nil && state.isLive { continue } + // A rejected credential applies even while the old socket stays open. if let error = hydrationError, error.isRejectedAuthorization { + credentialRejected = true owner?.applyEnvironmentLoad(EnvironmentShellLoad( environment: environment, client: client, shell: nil, config: nil, credentialRejected: true, failureDetail: error.localizedDescription @@ -4479,6 +4482,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, interval = .seconds(24 * 60 * 60) continue } + // Other HTTP failures say nothing about the connected stream. + if shell == nil && state.isLive { continue } // A validated HTTP snapshot remains in this socket's cache // epoch without claiming that the live stream is complete. if shell != nil { owner?.shellConnectionIDsByEnvironmentID[environment.id] = connectionID } diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index 10ff9f34149f..83985691fd86 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -1979,6 +1979,7 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { private var detailData: [String: [String: Data]] = [:] private var reachableHosts: Set private var shellReadsEnabledHosts: Set + private var rejectedShellHosts: Set = [] private var shellReadCounts: [String: Int] = [:] private var dispatched: [MultiEnvironmentDispatchRecord] = [] private var hostsDroppingNextCreateReply = Set() @@ -2003,6 +2004,11 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { } } + /// Answers this host's shell reads with 401, as after a revoked credential. + func rejectShellReads(host: String) { + rejectedShellHosts.insert(host) + } + func setShellReadsEnabled(_ enabled: Bool, host: String) { if enabled { shellReadsEnabledHosts.insert(host) @@ -2058,6 +2064,15 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { } if path == "/api/orchestration/shell" { shellReadCounts[host, default: 0] += 1 + if rejectedShellHosts.contains(host) { + return ( + Data(#"{"error":"Unauthorized"}"#.utf8), + HTTPURLResponse( + url: request.url!, statusCode: 401, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + )! + ) + } if let gate = nextShellGates.removeValue(forKey: host), let data = shellData[host] { await gate.enter() return (data, multiEnvironmentResponse(request)) @@ -2650,6 +2665,50 @@ private actor GatedPassiveCatalogueConnection: WebSocketConnection { @Suite("Native passive live shells") @MainActor struct NativePassiveLiveShellTests { + @Test("A live peer whose credential is rejected stops quiet reconciliation", .timeLimit(.minutes(1))) + func livePeerRejectedCredentialStopsReconciling() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clock = SilentShellClock() + let (peerSleeps, peerSleepRecorder) = AsyncStream.makeStream() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .zero, + fallbackPollingInterval: .milliseconds(1), + shellReconciliationSleep: { id, interval in + if id == "two" { try await clock.sleep(interval) } + else { try await Task.sleep(for: interval) } + }, + aggregatePeerRefreshSleep: { id, interval in + if id == "two" { peerSleepRecorder.yield(interval) } + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + try await receipts.waitForHTTP("two", count: 1) + await server.waitForSubscriptions(host: "two.example", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Live seed", snapshotSequence: 10 + ), host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + try await clock.waitForRequest() + await fixture.transport.rejectShellReads(host: "two.example") + await clock.advance() + // The socket is still open, but the credential is gone: show it and back off. + _ = try await recorder.wait { + $0.environments.first { $0.id == "two" }?.connectionState == .needsPairing + } + // The next wait is the long rejected-credential back-off, not another quiet read. + var sleeps = peerSleeps.makeAsyncIterator() + while let next = await sleeps.next(), next != .seconds(24 * 60 * 60) {} + #expect(await clock.intervals.count == 1) + await fixture.client.disconnect() + } + @Test("Silent connected shells reconcile through bounded HTTP", arguments: ["one", "two"], [false, true]) func silentConnectedShellReconciles(environmentID: String, disconnect: Bool) async throws { let server = PassiveLiveServer() From a082dccec67d398dbd910f1270ef442de53e320a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:30:46 +1000 Subject: [PATCH 19/19] fix(swift-ios): share a peer's credential rejection with its live stream The open socket's updates marked a rejected peer connected again, and stream repairs woke its HTTP loop before the back-off. Keep the rejection in the peer's shared state. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/swift-ios/App/NativeFeatureClient.swift | 27 ++++++++++++------- .../NativeMultiEnvironmentTests.swift | 22 +++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index 104724e7b43c..62ea4e7c2af7 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -4429,14 +4429,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func refreshShell(_ environment: Environment, state: PassiveShellState) async { var interval = fastInterval - // A rejected credential outlives the open socket; stop the quiet cadence. - var credentialRejected = false while owns(environment) { do { let wasLive = state.isLive if wasLive || !state.needsHTTP { let delay = interval - let reconciles = wasLive && !credentialRejected + // A rejected credential outlives the open socket; stop the quiet cadence. + let reconciles = wasLive && !state.credentialRejected await withTaskGroup(of: Void.self) { group in group.addTask { if reconciles { @@ -4473,7 +4472,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, authority == state.authorityRevision, connectionID == currentConnection else { continue } // A rejected credential applies even while the old socket stays open. if let error = hydrationError, error.isRejectedAuthorization { - credentialRejected = true + state.credentialRejected = true owner?.applyEnvironmentLoad(EnvironmentShellLoad( environment: environment, client: client, shell: nil, config: nil, credentialRejected: true, failureDetail: error.localizedDescription @@ -4538,15 +4537,17 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, // A read begun before the stream lost completeness cannot // repair that gap, even if the socket identity is unchanged. state.authorityRevision &+= 1 - markStreamPaused(environment) + markStreamPaused(environment, state: state) state.requestRepair() do { try await streamRetrySleep(environment.id, failureInterval) } catch { return } } } - func markStreamPaused(_ environment: Environment) { + func markStreamPaused(_ environment: Environment, state: PassiveShellState) { guard let owner, owns(environment) else { return } owner.shellConnectionIDsByEnvironmentID[environment.id] = nil + // A rejected credential still needs pairing; keep Pair again visible. + guard !state.credentialRejected else { return } owner.environmentConnectionStates[environment.id] = .reconnecting owner.environmentConnectionDetails[environment.id] = "Live updates paused. Refreshing over HTTP." schedulePublication(environment.id) @@ -4578,7 +4579,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let wasLive = state.isLive state.isLive = true state.needsHTTP = false - if !wasLive { state.wakeWaiter() } + // A rejected peer keeps its back-off even when a new stream seeds it. + if !wasLive && !state.credentialRejected { state.wakeWaiter() } case .synchronized: // A completion marker cannot make an arbitrary cached shell authoritative. return state.isLive @@ -4627,8 +4629,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, || current?.threads.map(\.id) != shell.threads.map(\.id) owner.shellsByEnvironmentID[environment.id] = shell owner.environmentClients[environment.id] = client - owner.environmentConnectionStates[environment.id] = .connected - owner.environmentConnectionDetails[environment.id] = nil + // Stream deltas still arrive on the old socket; keep showing the rejection. + if !state.credentialRejected { + owner.environmentConnectionStates[environment.id] = .connected + owner.environmentConnectionDetails[environment.id] = nil + } if membershipChanged { owner.rebuildEntityIndexes(savedEnvironments) } owner.synchronizeActiveDetail(with: shell, environment: environment) refreshReceipt(.shellApplied(environmentID: environment.id, sequence: shell.snapshotSequence)) @@ -4724,9 +4729,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, var authorityRevision = 0 var isLive = false var needsHTTP = false + /// Set when HTTP rejects this peer's credential; cleared by a new worker after re-pairing. + var credentialRejected = false private var waiter: (id: UUID, continuation: CheckedContinuation)? func requestRepair() { + // Stream failures must not wake a rejected peer before its back-off. + guard !credentialRejected else { return } needsHTTP = true wakeWaiter() } diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index 83985691fd86..7c5e2f9be863 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -2683,6 +2683,7 @@ struct NativePassiveLiveShellTests { if id == "two" { peerSleepRecorder.yield(interval) } try await Task.sleep(for: interval) }, + aggregateStreamRetrySleep: { _, _ in }, aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } ) @@ -2706,6 +2707,27 @@ struct NativePassiveLiveShellTests { var sleeps = peerSleeps.makeAsyncIterator() while let next = await sleeps.next(), next != .seconds(24 * 60 * 60) {} #expect(await clock.intervals.count == 1) + // Updates still arriving on the open socket must not hide the rejection. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "After rejection", snapshotSequence: 11 + ), host: "two.example") + let afterUpdate = try await recorder.wait { $0.threads.contains { $0.title == "After rejection" } } + #expect(afterUpdate.environments.first { $0.id == "two" }?.connectionState == .needsPairing) + // A closed stream and its fresh snapshot must neither hide the rejection nor end the back-off. + let reads = await fixture.transport.shellReadCount(host: "two.example") + try await server.finish(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 2) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "After reconnect", snapshotSequence: 12 + ), host: "two.example") + let afterReconnect = try await recorder.wait { $0.threads.contains { $0.title == "After reconnect" } } + #expect(afterReconnect.environments.first { $0.id == "two" }?.connectionState == .needsPairing) + // A woken fallback read would land before this later update is published. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Still backed off", snapshotSequence: 13 + ), host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Still backed off" } } + #expect(await fixture.transport.shellReadCount(host: "two.example") == reads) await fixture.client.disconnect() }