diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index a011101359bc..a23f0804c5ba 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -21,6 +21,11 @@ private struct T3ConnectManagedCleanupError: LocalizedError { } } +enum NativeSelectedThreadReconciliationReceipt: Equatable { + case finished(threadID: String) + case deferred(threadID: String) +} + /// Composes the transport-focused Core layer with the UI-focused Features layer. @MainActor final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, @@ -49,13 +54,23 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private let projectFaviconStore: FeatureProjectFaviconStore private let fallbackPollingInitialDelay: Duration private let fallbackPollingInterval: Duration + private static let shellReconciliationInterval: Duration = .seconds(30) + private let shellReconciliationSleep: @Sendable (String, Duration) async throws -> Void private let aggregateRefreshInterval: Duration private let aggregateIdleRefreshInterval: Duration private let aggregateFailureRefreshInterval: Duration private let aggregateRefreshSleep: @Sendable (Duration) async throws -> Void + private let aggregateRefreshReceipt: @MainActor @Sendable (NativePassiveShellReceipt) -> Void + private let aggregateStreamRetrySleep: @Sendable (String, Duration) async throws -> Void + private let aggregatePublishSleep: @Sendable () async throws -> Void + private let aggregatePeerRefreshSleep: @Sendable (String, Duration) async throws -> Void private let environmentShellTimeoutInterval: TimeInterval private let threadSnapshotTimeoutInterval: TimeInterval + private let detailPublicationSleep: @Sendable () async throws -> Void private let catchUpDelay: @Sendable () async throws -> Void + private let selectedThreadReconciliationSleep: @Sendable (Duration) async throws -> Void + private let selectedThreadReconciliationNow: @MainActor @Sendable () -> ContinuousClock.Instant + private let selectedThreadReconciliationReceipt: @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void private let threadRetryDelay: @Sendable (Int) async throws -> Void private let aggregateEnvironmentLoader: @Sendable (EnvironmentRuntime) async throws -> [Environment] private let stream: AsyncStream @@ -104,6 +119,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var pendingTurnSubmissions: [String: PendingTurnSubmission] = [:] private var projectSettingsWriteTask: Task? private var projectSettingsWriteGeneration: UInt64 = 0 + private struct AcceptedCommandRefresh: Sendable { + let id: UUID + let client: T3Client + let generation: Int + let task: Task + var pending = false + var needsDetail: Bool + } + private var acceptedCommandRefreshes: [String: AcceptedCommandRefresh] = [:] private var approvalRoutes: [String: PendingRequestRoute] = [:] private var inputRoutes: [String: PendingRequestRoute] = [:] private var relayDeviceSessionIDs: Set = [] @@ -115,13 +139,32 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var terminalSnapshots: [TerminalKey: FeatureTerminalSnapshot] = [:] // Keep versions unique when a terminal cache is evicted or an environment reconnects. private var terminalLifecycleVersion = 0 + private var foregroundBootstrapID = UUID() + private var activeHydrationTask: Task? + private var activeHydrationID: UUID? + private var activeHydrationPending = false + private var activeHTTPAuthorityRevision = 0 + private var shellConnectionIDsByEnvironmentID: [String: UUID] = [:] + private var activeShellConnectionID: UUID? + private var activeShellEpochHasSnapshot = false + private var activeHasHydrated = false + private var activeStreamIsAuthoritative = false private var pollingTask: Task? private var fallbackPollingTask: Task? + private var shellReconciliationTask: Task? private var configurationTask: Task? private var aggregateRefreshTask: Task? private var aggregateRefreshID: UUID? + private var aggregatePublishTask: Task? + private var isForeground = true + private(set) var aggregateRefreshWorkers: [ + String: (environment: Environment, task: Task) + ] = [:] private var shellPublishTask: Task? private var archivedRefreshTask: Task? + private var selectedThreadReconciliationTask: Task? + private var selectedThreadReconciliationRefreshGeneration: Int? + private var selectedThreadLastProgressAt: ContinuousClock.Instant? private var detailRefreshTask: Task? private var detailStreamTask: Task? private var detailCatchUpTask: Task? @@ -135,7 +178,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var detailStreamGeneration = 0 private var pendingDetailRenderMutations = NativeDetailRenderMutations() private var environmentGeneration = 0 - private var lastShellEventAt: Date? private var activeRawThread: OrchestrationThread? private var activeThreadSequence: Int? private var activeThreadPage: FeatureThreadPage? @@ -155,17 +197,38 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, projectFaviconStore: FeatureProjectFaviconStore = FeatureProjectFaviconStore(), fallbackPollingInitialDelay: Duration = .seconds(3), fallbackPollingInterval: Duration = .seconds(2), + shellReconciliationSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, aggregateRefreshInterval: Duration = NativeFeatureClient.defaultAggregateRefreshInterval, aggregateIdleRefreshInterval: Duration = NativeFeatureClient.defaultAggregateIdleRefreshInterval, aggregateFailureRefreshInterval: Duration = NativeFeatureClient.defaultAggregateFailureRefreshInterval, aggregateRefreshSleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }, + aggregatePeerRefreshSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(250)) + }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, environmentShellTimeoutInterval: TimeInterval = 6, threadSnapshotTimeoutInterval: TimeInterval = 8, + detailPublicationSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(80)) + }, catchUpDelay: @escaping @Sendable () async throws -> Void = { try await Task.sleep(for: .seconds(2)) }, + selectedThreadReconciliationSleep: @escaping @Sendable (Duration) async throws -> Void = { + try await Task.sleep(for: $0) + }, + selectedThreadReconciliationNow: @escaping @MainActor @Sendable () -> ContinuousClock.Instant = { .now }, + selectedThreadReconciliationReceipt: @escaping @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void = { _ in }, threadRetryDelay: @escaping @Sendable (Int) async throws -> Void = { attempt in try await Task.sleep(for: .seconds(min(5, 0.25 * pow(2, Double(min(5, attempt - 1)))))) }, @@ -197,13 +260,22 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.projectFaviconStore = projectFaviconStore self.fallbackPollingInitialDelay = fallbackPollingInitialDelay self.fallbackPollingInterval = fallbackPollingInterval + self.shellReconciliationSleep = shellReconciliationSleep self.aggregateRefreshInterval = aggregateRefreshInterval self.aggregateIdleRefreshInterval = aggregateIdleRefreshInterval self.aggregateFailureRefreshInterval = aggregateFailureRefreshInterval + self.aggregateRefreshReceipt = aggregateRefreshReceipt + self.aggregateStreamRetrySleep = aggregateStreamRetrySleep + self.aggregatePublishSleep = aggregatePublishSleep + self.aggregatePeerRefreshSleep = aggregatePeerRefreshSleep self.aggregateRefreshSleep = aggregateRefreshSleep self.environmentShellTimeoutInterval = environmentShellTimeoutInterval self.threadSnapshotTimeoutInterval = threadSnapshotTimeoutInterval + self.detailPublicationSleep = detailPublicationSleep self.catchUpDelay = catchUpDelay + self.selectedThreadReconciliationSleep = selectedThreadReconciliationSleep + self.selectedThreadReconciliationNow = selectedThreadReconciliationNow + self.selectedThreadReconciliationReceipt = selectedThreadReconciliationReceipt self.threadRetryDelay = threadRetryDelay self.aggregateEnvironmentLoader = aggregateEnvironmentLoader let pair = AsyncStream.makeStream() @@ -212,10 +284,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } deinit { + selectedThreadReconciliationTask?.cancel() + activeHydrationTask?.cancel() pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() aggregateRefreshTask?.cancel() + aggregatePublishTask?.cancel() + aggregateRefreshWorkers.values.forEach { $0.task.cancel() } shellPublishTask?.cancel() archivedRefreshTask?.cancel() detailRefreshTask?.cancel() @@ -223,50 +300,179 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detailCatchUpTask?.cancel() detailPublishTask?.cancel() projectFaviconRefreshTasks.values.forEach { $0.cancel() } + acceptedCommandRefreshes.values.forEach { $0.task.cancel() } continuation.finish() } func initialSnapshot() async throws -> FeatureSnapshot { + beginForegroundBootstrap() + let bootstrapID = foregroundBootstrapID let environments = try await runtime.environments() + guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } guard let activeClient = try await runtime.activeClient() else { - await clearActiveEnvironment() + await clearActiveEnvironment(disconnectClient: false) let snapshot = disconnectedSnapshot(environments: environments) latestSnapshot = snapshot return snapshot } - // The runtime actor can change its active selection at any suspension - // point. Derive both values from one client so the snapshot cannot pair - // one environment with another environment's connection. + guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } let environment = activeClient.environment - - await adoptEnvironment(environment, client: activeClient) - let generation = environmentGeneration - let loads = await loadEnvironmentShells(environments.filter(\.isEnabled)) - guard isCurrentSession(client: activeClient, generation: generation) else { + guard try await adoptEnvironment(environment, client: activeClient) else { throw CancellationError() } + guard bootstrapID == foregroundBootstrapID, + isCurrentSession(client: activeClient, generation: environmentGeneration) else { throw CancellationError() } - reconcileEnvironmentLoads(loads, savedEnvironments: environments) - latestShell = shellsByEnvironmentID[environment.id] - startPolling(activeClient) - let activeLoad = loads.first { $0.environment.id == environment.id } - let activeIsReachable = activeLoad?.shell != nil - if activeIsReachable { - scheduleArchivedRefresh(client: activeClient, environment: environment) - } - if activeLoad?.credentialRejected == true { - markActiveEnvironmentNeedsPairing(detail: activeLoad?.failureDetail) + reconcileEnvironmentLoads([], savedEnvironments: environments) + for saved in environments where saved.isEnabled { + environmentConnectionStates[saved.id] = .reconnecting + environmentConnectionDetails[saved.id] = nil } + latestShell = shellsByEnvironmentID[environment.id] let snapshot = makeSnapshot( - environments: environments, - activeEnvironment: environment, - connectionState: environmentConnectionStates[environment.id] ?? .disconnected, - connectionDetail: environmentConnectionDetails[environment.id] + environments: environments, activeEnvironment: environment, + connectionState: .reconnecting ) latestSnapshot = snapshot + // No suspension after assigning the seed: the root installs it before + // it consumes buffered hydration events through its single iterator. + if isForeground { + startPolling(activeClient) + requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + startAggregateRefresh(activeClient, refreshImmediately: true) + } return snapshot } + private func beginForegroundBootstrap() { + foregroundBootstrapID = UUID() + activeHydrationTask?.cancel() + activeHydrationTask = nil + activeHydrationID = nil + activeHydrationPending = false + activeHTTPAuthorityRevision &+= 1 + activeHasHydrated = false + activeStreamIsAuthoritative = false + pollingTask?.cancel() + fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() + configurationTask?.cancel() + archivedRefreshTask?.cancel() + archivedRefreshTask = nil + shellPublishTask?.cancel() + shellPublishTask = nil + cancelAggregateRefresh() + } + + private func isCurrentForeground(_ client: T3Client, generation: Int, bootstrapID: UUID) -> Bool { + !Task.isCancelled && isForeground && foregroundBootstrapID == bootstrapID + && isCurrentSession(client: client, generation: generation) + } + + /// Bootstrap, fallback polling, and unknown stream events share one HTTP + /// request owner. An invalidated request drains before its pending repair. + private func requestActiveShellHydration(_ activeClient: T3Client, bootstrapID: UUID, force: Bool = false) { + let generation = environmentGeneration + guard isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } + guard activeHydrationTask == nil else { + if force { activeHydrationPending = true } + return + } + let requestID = UUID() + let revision = activeHTTPAuthorityRevision + let runtime = runtime + let timeout = environmentShellTimeoutInterval + activeHydrationID = requestID + activeHydrationPending = false + activeHydrationTask = Task { [weak self] in + let shell: OrchestrationShellSnapshot? + var hydrationError: (any Error)? + do { shell = try await activeClient.shellSnapshot(timeoutInterval: timeout) } + catch { shell = nil; hydrationError = error } + let environments = try? await runtime.environments() + guard let self else { return } + defer { self.aggregateRefreshReceipt(.httpFinished(environmentID: activeClient.environment.id)) } + guard self.activeHydrationID == requestID, + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) + else { return } + self.activeHydrationTask = nil + self.activeHydrationID = nil + guard let environments, environments.contains(activeClient.environment), + activeClient.environment.isEnabled else { return } + if revision == self.activeHTTPAuthorityRevision { + if let error = hydrationError, error.isRejectedAuthorization { + self.markActiveEnvironmentNeedsPairing(detail: error.localizedDescription) + return + } + if let shell { + self.activeHydrationPending = false + self.acceptActiveShell(shell, client: activeClient, environments: environments, + socketAuthoritative: false, connectHeader: false) + } else if !self.activeStreamIsAuthoritative { + self.emitConnection( + self.activeHasHydrated ? .reconnecting : .disconnected, + detail: "Server unreachable. Retrying automatically." + ) + } + } + if self.activeHydrationPending && !self.activeStreamIsAuthoritative { + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + } + } + } + + private func acceptActiveShell( + _ candidate: OrchestrationShellSnapshot, client: T3Client, + environments: [Environment], socketAuthoritative: Bool, connectHeader: Bool + ) { + let shell: OrchestrationShellSnapshot + if activeShellEpochHasSnapshot, let cached = latestShell, + cached.snapshotSequence > candidate.snapshotSequence { shell = cached } + else { shell = candidate } + let firstHydration = !activeHasHydrated + activeShellEpochHasSnapshot = true + activeHasHydrated = true + if socketAuthoritative { + shellConnectionIDsByEnvironmentID[client.environment.id] = activeShellConnectionID + activeStreamIsAuthoritative = true + activeHTTPAuthorityRevision &+= 1 + activeHydrationPending = false + } + let removed = Set(shellsByEnvironmentID[client.environment.id]?.threads.map(\.id) ?? []) + .subtracting(shell.threads.map(\.id)) + for id in removed { + clearRemovedThreadDetail(FeatureScopedID.thread(environmentID: client.environment.id, wireID: id)) + } + latestShell = shell + shellsByEnvironmentID[client.environment.id] = shell + environmentConnectionStates[client.environment.id] = .connected + environmentConnectionDetails[client.environment.id] = nil + rebuildEntityIndexes(environments) + synchronizeActiveDetail(with: shell, environment: client.environment) + guard let activeEnvironment else { return } + publish(makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: connectHeader ? .connected : (latestSnapshot?.connection.state ?? .reconnecting), + connectionDetail: connectHeader ? nil : latestSnapshot?.connection.detail + )) + aggregateRefreshReceipt(.shellApplied(environmentID: client.environment.id, sequence: shell.snapshotSequence)) + if firstHydration { scheduleArchivedRefresh(client: client, environment: client.environment) } + } + + func suspendForBackground() { + stopSelectedThreadReconciliation() + isForeground = false + beginForegroundBootstrap() + } + func resumeAfterBackground(reconnect: Bool) async { + isForeground = true + if client == nil { + if let snapshot = try? await initialSnapshot(), isForeground { + continuation.yield(.snapshot(snapshot)) + } + return + } + if let client { startAggregateRefresh(client) } let sessionGeneration = environmentGeneration let selectedRoute = activeThreadID.flatMap { try? threadRoute(for: $0) } detailWasSynchronized = false @@ -291,7 +497,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } guard sessionGeneration == environmentGeneration else { return } - if let client { startPolling(client) } + if let client { + startPolling(client) + requestActiveShellHydration(client, bootstrapID: foregroundBootstrapID) + } if let selectedRoute, activeThreadID == selectedRoute.uiID, isKnownClient(selectedRoute.client, environmentID: selectedRoute.environmentID, generation: sessionGeneration) { resetDetailRefresh() @@ -340,7 +549,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } else { pairedClient = try await runtime.pair(url: endpoint, clientLabel: "T3 Code Swift") } - await adoptEnvironment(pairedClient.environment, client: pairedClient) + guard try await adoptEnvironment(pairedClient.environment, client: pairedClient) else { throw CancellationError() } + startAggregateRefresh(pairedClient) startPolling(pairedClient) } @@ -404,7 +614,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environment, credential: savedCredential ) - await adoptEnvironment(environment, client: managedClient) + guard try await adoptEnvironment(environment, client: managedClient) else { throw CancellationError() } do { try await refresh(client: managedClient) } catch { @@ -417,6 +627,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) publish(snapshot) } + startAggregateRefresh(managedClient) startPolling(managedClient) } @@ -470,6 +681,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func setEnvironmentEnabled(id: String, enabled: Bool) async throws { try await runtime.setEnabled(id: id, enabled: enabled) if !enabled { + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() + cancelAcceptedCommandRefreshes(environmentID: id) environmentConnectionStates[id] = .disconnected environmentConnectionDetails[id] = nil environmentClients[id] = nil @@ -490,6 +704,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } try await runtime.remove(id: id) saveGitHubRoutingGrants(gitHubRoutingGrants.filter { $0.environmentID != id }) + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() + cancelAcceptedCommandRefreshes(environmentID: id) if removesActiveEnvironment { await clearActiveEnvironment(disconnectClient: false) } @@ -970,46 +1187,59 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func adoptEnvironment( _ environment: Environment, client newClient: T3Client - ) async { + ) async throws -> Bool { + let bootstrapID = foregroundBootstrapID + let generation = environmentGeneration + let adoptedConnectionID = await newClient.currentConnectionID() + let selectedClient = try await runtime.activeClient() + guard bootstrapID == foregroundBootstrapID, generation == environmentGeneration, + selectedClient === newClient, selectedClient?.environment == environment else { return false } + cancelAggregateRefresh() if activeEnvironment?.id == environment.id, client === newClient { activeEnvironment = environment environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - startAggregateRefresh(newClient) - return + return true } - let previousClient = client pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() - aggregateRefreshTask?.cancel() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil + shellReconciliationTask = nil configurationTask = nil aggregateRefreshTask = nil aggregateRefreshID = nil archivedRefreshTask = nil + // A passive snapshot is authoritative for adoption only while its + // exact socket is still current. A replacement socket starts a new epoch. + activeShellConnectionID = adoptedConnectionID + activeShellEpochHasSnapshot = adoptedConnectionID != nil + && shellConnectionIDsByEnvironmentID[environment.id] == adoptedConnectionID clearEnvironmentState(preserveEnvironmentSnapshots: true) activeEnvironment = environment client = newClient environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - if let previousClient, previousClient !== newClient { - await previousClient.disconnect() - } - startAggregateRefresh(newClient) + // Runtime owns shared transports. The former inbox client may now + // serve a passive shell or a selected detail, so adoption does not close it. + return true } private func clearActiveEnvironment(disconnectClient: Bool = true) async { + beginForegroundBootstrap() let previousClient = client pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() - aggregateRefreshTask?.cancel() + cancelAggregateRefresh() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil + shellReconciliationTask = nil configurationTask = nil aggregateRefreshTask = nil aggregateRefreshID = nil @@ -1023,7 +1253,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func clearEnvironmentState(preserveEnvironmentSnapshots: Bool = false) { + activeHydrationTask?.cancel() + activeHydrationTask = nil + activeHydrationID = nil + activeHydrationPending = false + activeHTTPAuthorityRevision &+= 1 + activeHasHydrated = false + activeStreamIsAuthoritative = false environmentGeneration &+= 1 + cancelAcceptedCommandRefreshes() resetDetailRefresh() resetDetailStream() archivedRefreshTask?.cancel() @@ -1031,10 +1269,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, shellPublishTask?.cancel() shellPublishTask = nil latestShell = nil - lastShellEventAt = nil latestServerConfig = nil if !preserveEnvironmentSnapshots { environmentClients.removeAll() + shellConnectionIDsByEnvironmentID.removeAll() shellsByEnvironmentID.removeAll() shellProjectionCache.removeAll() indexedShellMembership = nil @@ -1984,6 +2222,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func deleteThread(id: String) async throws { let route = try threadRoute(for: id) _ = try await route.client.delete(threadID: route.wireID) + // A pending accepted-command refresh would publish the deleted thread again. + cancelAcceptedCommandRefreshes(threadID: route.uiID) archivedThreadsByEnvironmentID[route.environmentID]?.removeAll { $0.id == route.uiID } @@ -2012,6 +2252,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func loadThread(id: String, fresh: Bool) async throws -> FeatureThreadDetail { let route = try threadRoute(for: id) + if let previous = activeThreadID, previous != route.uiID { + cancelAcceptedCommandRefreshes(threadID: previous) + } let client = route.client let environment = client.environment let generation = environmentGeneration @@ -2051,9 +2294,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, activeThreadSequence = cached.sequence activeThreadPage = cached.page detail.page = cached.page + if latestDetails[route.uiID]?.page != detail.page { + publish(detail, threadID: route.uiID, renderCacheIsSource: true) + } markThreadCacheRecentlyUsed(route.uiID) startDetailStream(route, warmConnectionID: warmConnectionID) - return detail + if let shell = shellsByEnvironmentID[environment.id] { + synchronizeActiveDetail(with: shell, environment: environment) + } + return latestDetails[route.uiID] ?? detail } continuation.yield(.threadSync(id: route.uiID, state: .catchingUp)) let snapshot: OrchestrationThreadDetailSnapshot @@ -2154,6 +2403,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func releaseThread(id: String) { + cancelAcceptedCommandRefreshes(threadID: id) guard activeThreadID == id else { return } retainActiveThread() resetDetailRefresh() @@ -2273,11 +2523,67 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, if pendingTurnSubmissions[route.uiID]?.identity == pending.identity { pendingTurnSubmissions[route.uiID] = nil } - // Live sync reconciles these snapshots. Refreshes are opportunistic - // after the accepted command so transient reads cannot invite a - // duplicate user turn. - try? await refreshThread(id: route.uiID, client: client) - try? await refresh(client: client) + scheduleAcceptedCommandRefresh(threadID: route.uiID, client: client, generation: generation) + } + + /// Complete accepted commands without waiting for optional reads. A send + /// needs detail and shell recovery; Stop retains its shell-only refresh. + private func scheduleAcceptedCommandRefresh( + threadID: String, client: T3Client, generation: Int, refreshDetail: Bool = true + ) { + guard isKnownClient(client, environmentID: client.environment.id, generation: generation) else { + return + } + if let current = acceptedCommandRefreshes[threadID], + current.client === client, current.generation == generation { + acceptedCommandRefreshes[threadID]?.pending = true + acceptedCommandRefreshes[threadID]?.needsDetail = current.needsDetail || refreshDetail + return + } + cancelAcceptedCommandRefreshes(threadID: threadID) + let id = UUID() + let task = Task { [weak self] in + while self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true { + let needsDetail = self?.acceptedCommandRefreshes[threadID]?.needsDetail == true + self?.acceptedCommandRefreshes[threadID]?.needsDetail = false + self?.acceptedCommandRefreshes[threadID]?.pending = false + if needsDetail { + try? await self?.refreshThread(id: threadID, client: client) { [weak self] in + guard self?.acceptedCommandRefreshes[threadID]?.pending == true else { return false } + // The discarded detail must be read again, even for a shell-only Stop. + self?.acceptedCommandRefreshes[threadID]?.needsDetail = true + return true + } + } + guard self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true else { break } + try? await self?.refresh(client: client) + guard self?.acceptedCommandRefreshes[threadID]?.pending == true else { break } + } + if self?.acceptedCommandRefreshes[threadID]?.id == id { + self?.acceptedCommandRefreshes[threadID] = nil + } + } + acceptedCommandRefreshes[threadID] = AcceptedCommandRefresh( + id: id, client: client, generation: generation, task: task, needsDetail: refreshDetail + ) + } + + private func isCurrentAcceptedCommandRefresh(threadID: String, id: UUID) -> Bool { + guard !Task.isCancelled, let refresh = acceptedCommandRefreshes[threadID], refresh.id == id else { + return false + } + return isKnownClient( + refresh.client, environmentID: refresh.client.environment.id, generation: refresh.generation + ) + } + + private func cancelAcceptedCommandRefreshes(threadID: String? = nil, environmentID: String? = nil) { + for (key, refresh) in acceptedCommandRefreshes { + guard threadID.map({ $0 == key }) ?? true, + environmentID.map({ $0 == refresh.client.environment.id }) ?? true else { continue } + refresh.task.cancel() + acceptedCommandRefreshes[key] = nil + } } private func messageWasCommitted( @@ -2291,14 +2597,46 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, return snapshot.thread.messages.contains { $0.id == messageID } } + func stopStatus(threadID: String) async throws -> FeatureThread { + let route = try threadRoute(for: threadID) + let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + do { try await refresh(client: route.client) } catch is CancellationError where !Task.isCancelled { + // A live turn advanced the shell during this read. The cache answers only + // while its stream is still authoritative on the current socket. + let connectionID = await route.client.currentConnectionID() + guard connectionID != nil, shellConnectionIDsByEnvironmentID[route.environmentID] == connectionID, + activeEnvironment?.id != route.environmentID || activeStreamIsAuthoritative + else { throw CancellationError() } + } + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, + isKnownClient(route.client, environmentID: route.environmentID, generation: generation) else { + throw CancellationError() + } + guard let thread = shellsByEnvironmentID[route.environmentID]?.threads + .first(where: { $0.id == route.wireID }) else { + throw NativeFeatureClientError.threadNotFound + } + return mapThread(thread, environment: route.client.environment) + } + func cancelTurn(threadID: String) async throws { let route = try threadRoute(for: threadID) let turnID = shellsByEnvironmentID[route.environmentID]?.threads - .first(where: { $0.id == route.wireID })? - .latestTurn? - .turnId - _ = try await route.client.interrupt(threadID: route.wireID, turnID: turnID) - try? await refresh(client: route.client) + .first(where: { $0.id == route.wireID })?.latestTurn?.turnId + try await cancelTurn(threadID: threadID, expectedTurnID: turnID) + } + + func cancelTurn(threadID: String, expectedTurnID: String?) async throws { + let route = try threadRoute(for: threadID) + let generation = environmentGeneration + let turnID = shellsByEnvironmentID[route.environmentID]?.threads + .first(where: { $0.id == route.wireID })?.latestTurn?.turnId + guard turnID == expectedTurnID else { throw FeatureStopTurnChangedError() } + _ = try await route.client.interrupt(threadID: route.wireID, turnID: expectedTurnID) + scheduleAcceptedCommandRefresh( + threadID: route.uiID, client: route.client, generation: generation, refreshDetail: false + ) } func canRewindConversation(threadID: String, messageID: String) -> Bool { @@ -3757,66 +4095,70 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func startPolling(_ activeClient: T3Client) { pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + let streamRetrySleep = aggregateStreamRetrySleep pollingTask = Task { [weak self] in while !Task.isCancelled, - self?.isCurrentSession(client: activeClient, generation: generation) == true + self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true { do { await activeClient.connect() guard - self?.isCurrentSession( - client: activeClient, - generation: generation - ) == true + self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true else { return } - let sequence = self?.latestShell?.snapshotSequence - let events = await activeClient.shellEventBatches(after: sequence, reconnect: false) + let subscription = try await activeClient.shellEventBatchesOnCurrentConnection() + guard self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true + else { return } + if let previous = self?.activeShellConnectionID, previous != subscription.connectionID { + self?.activeShellEpochHasSnapshot = false + self?.activeHTTPAuthorityRevision &+= 1 + self?.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + } + self?.activeShellConnectionID = subscription.connectionID + let events = subscription.events // Re-bind self per event instead of holding it strongly across // the indefinite stream, so the client can deinit mid-stream. - for try await batch in events { + shellEvents: for try await batch in events { guard !Task.isCancelled, let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { break } - self.lastShellEventAt = .now - self.emitConnection(.connected) + let environments = try await self.runtime.environments() + let connectionID = await activeClient.currentConnectionID() + guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), + connectionID == subscription.connectionID, + environments.contains(activeClient.environment) else { break } var deltas: [ShellStreamItem] = [] for item in batch { - switch item { - case let .snapshot(shell): - await self.consume(deltas: deltas, client: activeClient, generation: generation) - deltas.removeAll(keepingCapacity: true) - await self.consume( - shell: shell, - client: activeClient, - generation: generation, - refreshActiveThread: true - ) - case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: - deltas.append(item) - case .refreshRequired: - await self.consume(deltas: deltas, client: activeClient, generation: generation) - deltas.removeAll(keepingCapacity: true) - if let shell = try? await activeClient.shellSnapshot() { - await self.consume( - shell: shell, - client: activeClient, - generation: generation, - refreshActiveThread: true - ) - } - case .synchronized: - break + switch item { + case let .snapshot(shell): + await self.consume(deltas: deltas, client: activeClient, generation: generation) + deltas.removeAll(keepingCapacity: true) + self.acceptActiveShell(shell, client: activeClient, environments: environments, + socketAuthoritative: true, connectHeader: true) + case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: + guard self.activeStreamIsAuthoritative else { + self.activeHTTPAuthorityRevision &+= 1 + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + break shellEvents } + deltas.append(item) + case .refreshRequired: + self.activeStreamIsAuthoritative = false + self.activeHTTPAuthorityRevision &+= 1 + self.emitConnection(.reconnecting, detail: "Refreshing environment data.") + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + break shellEvents + case .synchronized: + break + } } await self.consume(deltas: deltas, client: activeClient, generation: generation) } @@ -3835,16 +4177,21 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard !Task.isCancelled, let self, - self.isCurrentSession(client: activeClient, generation: generation) + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } - self.lastShellEventAt = nil + self.shellConnectionIDsByEnvironmentID[activeClient.environment.id] = nil + if self.activeStreamIsAuthoritative { + self.activeHTTPAuthorityRevision &+= 1 + } + self.activeStreamIsAuthoritative = false + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) self.emitConnection( .reconnecting, detail: "Live updates paused. Refreshing over HTTP." ) - do { try await Task.sleep(for: .milliseconds(250)) } catch { return } + do { try await streamRetrySleep(activeClient.environment.id, .seconds(20)) } catch { return } } } let fallbackPollingInitialDelay = fallbackPollingInitialDelay @@ -3857,59 +4204,31 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } while !Task.isCancelled { guard let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } - let socketIsSynchronized = - await activeClient.liveConnectionActive() - && self.lastShellEventAt != nil - if !socketIsSynchronized { - self.emitConnection( - .reconnecting, - detail: "Live updates reconnecting. Refreshing over HTTP." - ) - do { - let shell = try await activeClient.shellSnapshot() - guard !Task.isCancelled, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { - return - } - await self.consumeFallbackShell( - shell: shell, - client: activeClient, - generation: generation - ) - } catch is CancellationError { - return - } catch { - guard !Task.isCancelled, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { - return - } - if error.isRejectedAuthorization { - self.markActiveEnvironmentNeedsPairing(detail: error.localizedDescription) - return - } - self.emitConnection( - .reconnecting, - detail: "Server unreachable. Retrying automatically." - ) - } - } - do { - try await Task.sleep(for: fallbackPollingInterval) - } catch { - return + if !self.activeStreamIsAuthoritative { + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) } + do { try await Task.sleep(for: fallbackPollingInterval) } + catch { return } + } + } + let reconciliationSleep = shellReconciliationSleep + shellReconciliationTask = Task { [weak self] in + while !Task.isCancelled { + // Ping health cannot prove shell completeness. Keep the faster + // outage fallback independent of this quiet reconciliation. + do { try await reconciliationSleep(activeClient.environment.id, Self.shellReconciliationInterval) } + catch { return } + guard let self, + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) + else { return } + guard self.activeStreamIsAuthoritative else { continue } + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + // One owner coalesces all HTTP requests. The next interval starts + // after this read drains, including failures and supersession. + await self.activeHydrationTask?.value } } configurationTask = Task { [weak self] in @@ -3917,10 +4236,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, for try await event in await activeClient.serverConfigEvents() { guard !Task.isCancelled, let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { break } switch event { @@ -3982,11 +4298,16 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, case .unrelated: continue } - if let shell = self.latestShell { - await self.emitSnapshot( - shell, client: activeClient, expectedGeneration: generation - ) - } + // A failed catalog read skips this publish; later events still apply. + guard let environments = try? await self.runtime.environments() else { continue } + guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), + environments.contains(activeClient.environment), let activeEnvironment = self.activeEnvironment + else { return } + self.publish(self.makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: self.latestSnapshot?.connection.state ?? .reconnecting, + connectionDetail: self.latestSnapshot?.connection.detail + )) } } catch is CancellationError { return @@ -4000,124 +4321,493 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, /// Non-active environments do not hold WebSocket subscriptions. A quiet /// HTTP refresh keeps their home rows and reachability useful without /// multiplying live streams or creating a high-frequency battery cost. - private func startAggregateRefresh(_ activeClient: T3Client) { + private func cancelAggregateRefresh() { aggregateRefreshTask?.cancel() - let generation = environmentGeneration - let refreshID = UUID() - let fastInterval = aggregateRefreshInterval - let idleInterval = aggregateIdleRefreshInterval - let failureInterval = aggregateFailureRefreshInterval - let sleep = aggregateRefreshSleep - let loadEnvironments = aggregateEnvironmentLoader - aggregateRefreshID = refreshID - aggregateRefreshTask = Task { [weak self] in - var nextInterval = fastInterval - var failureBackoffs: [String: Duration] = [:] - while !Task.isCancelled { - let elapsedInterval = nextInterval + aggregatePublishTask?.cancel() + aggregateRefreshWorkers.values.forEach { $0.task.cancel() } + aggregateRefreshWorkers.removeAll() + aggregatePublishTask = nil + aggregateRefreshID = nil + } + + private func startAggregateRefresh(_ activeClient: T3Client, refreshImmediately: Bool = false) { + cancelAggregateRefresh() + guard isForeground else { return } + let context = AggregateRefreshContext(owner: self, activeClient: activeClient, refreshImmediately: refreshImmediately) + aggregateRefreshID = context.refreshID + aggregateRefreshTask = Task { + defer { context.cancelWorkers() } + while context.isCurrent { + var interval = context.fastInterval do { - try await sleep(nextInterval) - } catch { + let environments = try await context.loadEnvironments(context.runtime) + guard context.isCurrent else { return } + interval = context.reconcileWorkers(environments) + } catch is CancellationError where Task.isCancelled { return + } catch { + interval = context.failureInterval } - guard let self, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ), - let activeEnvironment = self.activeEnvironment else { - return + do { try await context.sleep(interval) } catch { return } + } + } + } + + /// A worker retains runtime inputs, never its UI owner across a suspension. + /// This lets deinit cancel the exact outstanding shell/catalogue tasks. + @MainActor + private final class AggregateRefreshContext { + weak var owner: NativeFeatureClient? + let runtime: EnvironmentRuntime + let activeClient: T3Client + let generation: Int + let refreshID = UUID() + let refreshImmediately: Bool + let fastInterval: Duration + let idleInterval: Duration + let failureInterval: Duration + let shellTimeout: TimeInterval + let sleep: @Sendable (Duration) async throws -> Void + let peerSleep: @Sendable (String, Duration) async throws -> Void + let reconciliationSleep: @Sendable (String, Duration) async throws -> Void + let loadEnvironments: @Sendable (EnvironmentRuntime) async throws -> [Environment] + let streamRetrySleep: @Sendable (String, Duration) async throws -> Void + let publishSleep: @Sendable () async throws -> Void + let refreshReceipt: @MainActor @Sendable (NativePassiveShellReceipt) -> Void + var savedEnvironments: [Environment] = [] + var dirtyEnvironmentIDs = Set() + + init(owner: NativeFeatureClient, activeClient: T3Client, refreshImmediately: Bool) { + self.owner = owner + self.refreshImmediately = refreshImmediately + runtime = owner.runtime + self.activeClient = activeClient + generation = owner.environmentGeneration + fastInterval = owner.aggregateRefreshInterval + idleInterval = owner.aggregateIdleRefreshInterval + failureInterval = owner.aggregateFailureRefreshInterval + shellTimeout = owner.environmentShellTimeoutInterval + sleep = owner.aggregateRefreshSleep + peerSleep = owner.aggregatePeerRefreshSleep + reconciliationSleep = owner.shellReconciliationSleep + loadEnvironments = owner.aggregateEnvironmentLoader + streamRetrySleep = owner.aggregateStreamRetrySleep + publishSleep = owner.aggregatePublishSleep + refreshReceipt = owner.aggregateRefreshReceipt + } + + var isCurrent: Bool { + !Task.isCancelled && owner?.isForeground == true && owner?.aggregateRefreshID == refreshID + && owner?.isCurrentSession(client: activeClient, generation: generation) == true + } + + func cancelWorkers() { + guard let owner, owner.aggregateRefreshID == refreshID else { return } + owner.aggregateRefreshWorkers.values.forEach { $0.task.cancel() } + owner.aggregateRefreshWorkers.removeAll() + owner.aggregatePublishTask?.cancel() + owner.aggregatePublishTask = nil + } + + func owns(_ environment: Environment) -> Bool { + isCurrent && owner?.aggregateRefreshWorkers[environment.id]?.environment == environment + && owner?.activeEnvironment?.id != environment.id + } + + func reconcileWorkers(_ environments: [Environment]) -> Duration { + guard let owner, isCurrent else { return idleInterval } + savedEnvironments = environments + let passive = environments.filter { $0.isEnabled && $0.id != owner.activeEnvironment?.id } + let current = Dictionary(uniqueKeysWithValues: passive.map { ($0.id, $0) }) + for (id, worker) in owner.aggregateRefreshWorkers where current[id] != worker.environment { + worker.task.cancel() + owner.aggregateRefreshWorkers[id] = nil + } + // Prune once per topology check, not once per shell result. + owner.reconcileEnvironmentLoads([], savedEnvironments: environments) + if owner.latestSnapshot?.environments != environments.map({ + owner.mapEnvironment($0, activeID: owner.activeEnvironment?.id) + }) { + owner.publishAggregateSnapshot(environments) + } + for environment in passive where owner.aggregateRefreshWorkers[environment.id] == nil { + let state = PassiveShellState() + state.needsHTTP = refreshImmediately || owner.shellsByEnvironmentID[environment.id] == nil + let task = Task { + await withTaskGroup(of: Void.self) { group in + group.addTask { await self.refreshShell(environment, state: state) } + group.addTask { await self.followShell(environment, state: state) } + group.addTask { await self.refreshCatalogue(environment) } + } } - let environments: [Environment] + owner.aggregateRefreshWorkers[environment.id] = (environment, task) + } + if let dirtyID = dirtyEnvironmentIDs.first { schedulePublication(dirtyID) } + return passive.isEmpty ? idleInterval : fastInterval + } + + /// Revalidate membership after suspension, including endpoint edits. + /// Persistence errors retry; removed or disabled peers end their work. + func currentEnvironments(for environment: Environment) async throws -> [Environment]? { + guard isCurrent else { return nil } + let environments = try await runtime.environments() + guard isCurrent, environments.contains(environment), environment.isEnabled, + owner?.activeEnvironment?.id != environment.id else { return nil } + return environments + } + + func refreshShell(_ environment: Environment, state: PassiveShellState) async { + var interval = fastInterval + while owns(environment) { do { - environments = try await loadEnvironments(self.runtime) + let wasLive = state.isLive + if wasLive || !state.needsHTTP { + let delay = interval + // A rejected credential outlives the open socket; stop the quiet cadence. + let reconciles = wasLive && !state.credentialRejected + await withTaskGroup(of: Void.self) { group in + group.addTask { + if reconciles { + try? await self.reconciliationSleep(environment.id, NativeFeatureClient.shellReconciliationInterval) + } else { + try? await self.peerSleep(environment.id, delay) + } + } + group.addTask { await state.waitForRepair(untilLive: !wasLive) } + _ = await group.next() + group.cancelAll() + } + } + guard owns(environment) else { return } + // A newly authoritative snapshot starts the quiet cadence; + // it must not turn a fallback wake-up into a duplicate read. + if !wasLive && state.isLive { continue } + state.needsHTTP = false + guard try await currentEnvironments(for: environment) != nil else { return } + let client = await runtime.client(for: environment) + let epoch = state.epoch + let authority = state.authorityRevision + let connectionID = await client.currentConnectionID() + guard owns(environment), epoch == state.epoch, + authority == state.authorityRevision else { continue } + let shell: OrchestrationShellSnapshot? + var hydrationError: (any Error)? + do { shell = try await client.shellSnapshot(timeoutInterval: shellTimeout) } + catch { shell = nil; hydrationError = error } + defer { refreshReceipt(.httpFinished(environmentID: environment.id)) } + let currentConnection = await client.currentConnectionID() + guard let environments = try await currentEnvironments(for: environment) else { return } + guard owns(environment), epoch == state.epoch, + authority == state.authorityRevision, connectionID == currentConnection else { continue } + // A rejected credential applies even while the old socket stays open. + if let error = hydrationError, error.isRejectedAuthorization { + state.credentialRejected = true + owner?.applyEnvironmentLoad(EnvironmentShellLoad( + environment: environment, client: client, shell: nil, config: nil, + credentialRejected: true, failureDetail: error.localizedDescription + )) + owner?.publishAggregateSnapshot(environments) + interval = .seconds(24 * 60 * 60) + continue + } + // Other HTTP failures say nothing about the connected stream. + if shell == nil && state.isLive { continue } + // A validated HTTP snapshot remains in this socket's cache + // epoch without claiming that the live stream is complete. + if shell != nil { owner?.shellConnectionIDsByEnvironmentID[environment.id] = connectionID } + // The first authoritative response in this subscription epoch + // may reset a cache left over from a restarted server. + interval = applyShell( + shell, client: client, environment: environment, saved: environments, + replacingEpoch: state.cacheEpoch != epoch + ) + if shell != nil { state.cacheEpoch = epoch } } catch is CancellationError where Task.isCancelled { return } catch { - // Persistence can be briefly unavailable while another - // actor atomically replaces the environment document. - // Back off while keeping the loop alive for recovery. - nextInterval = failureInterval - continue + interval = failureInterval } - guard !Task.isCancelled, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + } + } + + func followShell(_ environment: Environment, state: PassiveShellState) async { + while owns(environment) { + do { + guard try await currentEnvironments(for: environment) != nil else { return } + let client = await runtime.client(for: environment) + let subscription = try await client.shellEventsOnCurrentConnection() + guard try await currentEnvironments(for: environment) != nil, + owns(environment) else { return } + if state.connectionID != subscription.connectionID { + let needsRepair = state.connectionID != nil + || owner?.shellsByEnvironmentID[environment.id] == nil + state.connectionID = subscription.connectionID + state.epoch = UUID() + // A fresh bootstrap already read this peer. Its first + // socket attempt need not duplicate that HTTP request. + if needsRepair { state.requestRepair() } + } + let epoch = state.epoch + state.isLive = false + for try await item in subscription.events { + let connectionID = await client.currentConnectionID() + guard owns(environment), epoch == state.epoch, + connectionID == subscription.connectionID else { break } + guard applyStream(item, client: client, environment: environment, state: state) + else { break } + } + } catch is CancellationError where Task.isCancelled { return + } catch { + // HTTP recovery belongs to this peer's single fallback loop. } - let passiveEnvironments = environments.filter { - $0.isEnabled && $0.id != activeEnvironment.id - } - guard !passiveEnvironments.isEmpty else { - nextInterval = idleInterval - continue - } - let passiveIDs = Set(passiveEnvironments.map(\.id)) - failureBackoffs = failureBackoffs.reduce(into: [:]) { result, entry in - guard passiveIDs.contains(entry.key) else { return } - result[entry.key] = max(.zero, entry.value - elapsedInterval) + guard owns(environment) else { return } + state.isLive = false + // A read begun before the stream lost completeness cannot + // repair that gap, even if the socket identity is unchanged. + state.authorityRevision &+= 1 + markStreamPaused(environment, state: state) + state.requestRepair() + do { try await streamRetrySleep(environment.id, failureInterval) } catch { return } + } + } + + func markStreamPaused(_ environment: Environment, state: PassiveShellState) { + guard let owner, owns(environment) else { return } + owner.shellConnectionIDsByEnvironmentID[environment.id] = nil + // A rejected credential still needs pairing; keep Pair again visible. + guard !state.credentialRejected else { return } + owner.environmentConnectionStates[environment.id] = .reconnecting + owner.environmentConnectionDetails[environment.id] = "Live updates paused. Refreshing over HTTP." + schedulePublication(environment.id) + } + + func applyStream( + _ item: ShellStreamItem, client: T3Client, environment: Environment, state: PassiveShellState + ) -> Bool { + guard let owner, owns(environment) else { return false } + let current = owner.shellsByEnvironmentID[environment.id] + let shell: OrchestrationShellSnapshot + switch item { + case let .snapshot(snapshot): + if state.cacheEpoch == state.epoch, let current, + current.snapshotSequence > snapshot.snapshotSequence { + shell = current + } else { + shell = snapshot } - let refreshableEnvironments = passiveEnvironments.filter { - failureBackoffs[$0.id, default: .zero] <= .zero + let remainingIDs = Set(shell.threads.map(\.id)) + for removed in current?.threads ?? [] where !remainingIDs.contains(removed.id) { + owner.clearRemovedThreadDetail(FeatureScopedID.thread( + environmentID: environment.id, wireID: removed.id + )) } - guard !refreshableEnvironments.isEmpty else { - nextInterval = fastInterval - continue + owner.shellConnectionIDsByEnvironmentID[environment.id] = state.connectionID + state.authorityRevision &+= 1 + state.cacheEpoch = state.epoch + let wasLive = state.isLive + state.isLive = true + state.needsHTTP = false + // A rejected peer keeps its back-off even when a new stream seeds it. + if !wasLive && !state.credentialRejected { state.wakeWaiter() } + case .synchronized: + // A completion marker cannot make an arbitrary cached shell authoritative. + return state.isLive + case .refreshRequired: + return false + case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: + guard state.isLive, let current else { return false } + var projects = current.projects + var threads = current.threads + let sequence: Int + switch item { + case let .projectUpserted(nextSequence, project): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + if let index = projects.firstIndex(where: { $0.id == project.id }) { + projects[index] = project + } else { projects.append(project) } + case let .projectRemoved(nextSequence, id): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + projects.removeAll { $0.id == id } + case let .threadUpserted(nextSequence, thread): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + owner.publishThreadReceipt(FeatureScopedID.thread( + environmentID: environment.id, wireID: thread.id + ), source: .shellThreadUpdate) + owner.archivedThreadsByEnvironmentID[environment.id]?.removeAll { + ($0.wireID ?? $0.id) == thread.id + } + if let index = threads.firstIndex(where: { $0.id == thread.id }) { + threads[index] = thread + } else { threads.append(thread) } + case let .threadRemoved(nextSequence, id): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + threads.removeAll { $0.id == id } + owner.clearRemovedThreadDetail(FeatureScopedID.thread( + environmentID: environment.id, wireID: id + )) + default: + return false } - let loads = await self.loadEnvironmentShells(refreshableEnvironments) - guard !Task.isCancelled, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + shell = OrchestrationShellSnapshot( + snapshotSequence: sequence, projects: projects, threads: threads, updatedAt: current.updatedAt + ) + } + let membershipChanged = current?.projects.map(\.id) != shell.projects.map(\.id) + || current?.threads.map(\.id) != shell.threads.map(\.id) + owner.shellsByEnvironmentID[environment.id] = shell + owner.environmentClients[environment.id] = client + // Stream deltas still arrive on the old socket; keep showing the rejection. + if !state.credentialRejected { + owner.environmentConnectionStates[environment.id] = .connected + owner.environmentConnectionDetails[environment.id] = nil + } + if membershipChanged { owner.rebuildEntityIndexes(savedEnvironments) } + owner.synchronizeActiveDetail(with: shell, environment: environment) + refreshReceipt(.shellApplied(environmentID: environment.id, sequence: shell.snapshotSequence)) + schedulePublication(environment.id) + return true + } + + func schedulePublication(_ environmentID: String) { + guard let owner, isCurrent else { return } + dirtyEnvironmentIDs.insert(environmentID) + guard owner.aggregatePublishTask == nil else { return } + owner.aggregatePublishTask = Task { + do { try await self.publishSleep() } catch { return } + let environments: [Environment] + do { environments = try await self.runtime.environments() } + catch { + if self.isCurrent { self.owner?.aggregatePublishTask = nil } return } - let shellsChanged = loads.contains { load in - guard let shell = load.shell else { return false } - return shell != self.shellsByEnvironmentID[load.environment.id] - } - let hasActiveWork = loads.contains { load in - load.shell.map(Self.shellNeedsFrequentAggregateRefresh) == true + guard let owner = self.owner, self.isCurrent else { return } + self.savedEnvironments = environments + owner.reconcileEnvironmentLoads([], savedEnvironments: environments) + let dirty = self.dirtyEnvironmentIDs.filter { + owner.aggregateRefreshWorkers[$0] != nil } - for load in loads { - if load.credentialRejected { - // Only re-pairing changes this. Re-pairing replaces - // the runtime client and restarts this loop. - failureBackoffs[load.environment.id] = .seconds(24 * 60 * 60) - } else if load.shell == nil { - failureBackoffs[load.environment.id] = failureInterval - } else { - failureBackoffs[load.environment.id] = nil + self.dirtyEnvironmentIDs.removeAll() + owner.aggregatePublishTask = nil + guard !dirty.isEmpty else { return } + owner.publishAggregateSnapshot(self.savedEnvironments, changedEnvironmentIDs: dirty) + } + } + + func applyShell( + _ shell: OrchestrationShellSnapshot?, client: T3Client, + environment: Environment, saved: [Environment], replacingEpoch: Bool = false + ) -> Duration { + guard let owner, isCurrent else { return failureInterval } + let previous = owner.shellsByEnvironmentID[environment.id] + let changed = shell.map { $0 != previous } ?? false + let membershipChanged = shell.map { + $0.projects.map(\.id) != previous?.projects.map(\.id) + || $0.threads.map(\.id) != previous?.threads.map(\.id) + } ?? false + if replacingEpoch, let shell { owner.shellsByEnvironmentID[environment.id] = shell } + owner.applyEnvironmentLoad(EnvironmentShellLoad( + environment: environment, client: client, shell: shell, config: nil + )) + if membershipChanged { owner.rebuildEntityIndexes(saved) } + owner.publishAggregateSnapshot(saved, changedEnvironmentIDs: [environment.id]) + guard let shell else { return failureInterval } + return changed || NativeFeatureClient.shellNeedsFrequentAggregateRefresh(shell) + ? fastInterval : idleInterval + } + + func refreshCatalogue(_ environment: Environment) async { + // A rejected peer needs pairing; re-pairing starts a new worker. + while isCurrent && owner?.serverConfigsByEnvironmentID[environment.id] == nil + && owner?.environmentConnectionStates[environment.id] != .needsPairing { + do { + guard try await currentEnvironments(for: environment) != nil else { return } + // Never disconnect the shared client if this peer becomes selected. + let probe = await runtime.ephemeralClient(for: environment) + let config = try? await probe.serverConfig() + await probe.disconnect() + guard let environments = try await currentEnvironments(for: environment) else { return } + // A save or config event that landed during the probe is newer. + guard owner?.serverConfigsByEnvironmentID[environment.id] == nil else { return } + if let config { + applyCatalogue(config, environment: environment, saved: environments) + return } + } catch is CancellationError where Task.isCancelled { + return + } catch { + // Retry a transient environment-document read just like a failed probe. } - self.reconcileEnvironmentLoads(loads, savedEnvironments: environments) - let currentConnection = self.latestSnapshot?.connection - ?? FeatureConnection( - state: .disconnected, - environmentName: activeEnvironment.label, - endpoint: activeEnvironment.httpBaseURL.absoluteString - ) - let snapshot = self.makeSnapshot( - environments: environments, - activeEnvironment: activeEnvironment, - connectionState: currentConnection.state, - connectionDetail: currentConnection.detail - ) - self.publish(snapshot) - if shellsChanged || hasActiveWork { - nextInterval = fastInterval - } else { - nextInterval = idleInterval + do { try await Task.sleep(for: failureInterval) } catch { return } + } + } + + func applyCatalogue(_ config: ServerConfigSnapshot, environment: Environment, saved: [Environment]) { + guard let owner, isCurrent else { return } + owner.setServerConfig(config, environmentID: environment.id) + owner.publishAggregateSnapshot(saved, changedEnvironmentIDs: [environment.id]) + } + } + + @MainActor + private final class PassiveShellState { + var epoch = UUID() + var connectionID: UUID? + var cacheEpoch: UUID? + var authorityRevision = 0 + var isLive = false + var needsHTTP = false + /// Set when HTTP rejects this peer's credential; cleared by a new worker after re-pairing. + var credentialRejected = false + private var waiter: (id: UUID, continuation: CheckedContinuation)? + + func requestRepair() { + // Stream failures must not wake a rejected peer before its back-off. + guard !credentialRejected else { return } + needsHTTP = true + wakeWaiter() + } + + func wakeWaiter() { + waiter?.continuation.resume() + waiter = nil + } + + func waitForRepair(untilLive: Bool = false) async { + guard !needsHTTP, !(untilLive && isLive) else { return } + let id = UUID() + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if Task.isCancelled || needsHTTP || (untilLive && isLive) { continuation.resume() } + else { waiter = (id, continuation) } + } + } onCancel: { + Task { @MainActor [weak self] in + guard self?.waiter?.id == id else { return } + self?.waiter?.continuation.resume() + self?.waiter = nil } } } } + private func publishAggregateSnapshot( + _ environments: [Environment], changedEnvironmentIDs: Set? = nil + ) { + guard let activeEnvironment else { return } + let connection = latestSnapshot?.connection + publish(makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: connection?.state ?? .disconnected, + connectionDetail: connection?.detail, + changedEnvironmentIDs: changedEnvironmentIDs + )) + } + nonisolated private static func shellNeedsFrequentAggregateRefresh( _ shell: OrchestrationShellSnapshot ) -> Bool { @@ -4154,35 +4844,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } - /// HTTP fallback refreshes data while preserving the socket's reconnecting - /// state. The generation travels through the awaited snapshot publish so a - /// task from a previous environment session cannot publish late results. - private func consumeFallbackShell( - shell: OrchestrationShellSnapshot, - client: T3Client, - generation: Int - ) async { - guard isCurrentSession(client: client, generation: generation), - shell.snapshotSequence >= (latestShell?.snapshotSequence ?? .min) else { - return - } - shellPublishTask?.cancel() - shellPublishTask = nil - latestShell = shell - shellsByEnvironmentID[client.environment.id] = shell - await emitSnapshot( - shell, - client: client, - expectedGeneration: generation, - markSourceConnected: false - ) - guard isCurrentSession(client: client, generation: generation), - let threadID = activeThreadID else { - return - } - scheduleDetailRefresh(threadID: threadID, client: client) - } - private func consume(deltas: [ShellStreamItem], client: T3Client, generation: Int) async { guard !deltas.isEmpty, !Task.isCancelled, isCurrentSession(client: client, generation: generation) else { return } @@ -4229,6 +4890,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, changedThreadIDs.insert(FeatureScopedID.thread( environmentID: client.environment.id, wireID: thread.id )) + publishThreadReceipt(FeatureScopedID.thread(environmentID: client.environment.id, wireID: thread.id), source: .shellThreadUpdate) archivedThreadsByEnvironmentID[client.environment.id]?.removeAll { ($0.wireID ?? $0.id) == thread.id } @@ -4244,20 +4906,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, changedThreadIDs.insert(uiThreadID) shouldRefreshArchived = true threads.removeAll { $0.id == threadID } - latestDetails[uiThreadID] = nil - detailRenderCaches[uiThreadID] = nil - detailCacheRecency.removeAll { $0 == uiThreadID } - if activeThreadID == uiThreadID { - resetDetailRefresh() - resetDetailStream() - activeThreadID = nil - activeThreadEnvironmentID = nil - activeRawThread = nil - activeThreadSequence = nil - activeThreadPage = nil - threadHistoryEpoch &+= 1 - pendingOlderThreadPage = nil - } + clearRemovedThreadDetail(uiThreadID) case .snapshot, .synchronized, .refreshRequired: continue } @@ -4283,6 +4932,23 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } + private func clearRemovedThreadDetail(_ uiThreadID: String) { + latestDetails[uiThreadID] = nil + detailRenderCaches[uiThreadID] = nil + detailCacheRecency.removeAll { $0 == uiThreadID } + if activeThreadID == uiThreadID { + resetDetailRefresh() + resetDetailStream() + activeThreadID = nil + activeThreadEnvironmentID = nil + activeRawThread = nil + activeThreadSequence = nil + activeThreadPage = nil + threadHistoryEpoch &+= 1 + pendingOlderThreadPage = nil + } + } + /// Shell streams can emit many metadata updates during one provider turn. /// Home only needs the newest row state, so publish at most four times per /// second while the selected transcript continues on its dedicated stream. @@ -4305,11 +4971,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func scheduleDetailRefresh( threadID: String, client: T3Client, - force: Bool = false + force: Bool = false, + reconcile: Bool = false ) { guard activeThreadID == threadID, activeThreadEnvironmentID == client.environment.id else { return } - guard force || detailStreamTask == nil else { return } + guard force || reconcile || detailStreamTask == nil else { return } if force { detailWasSynchronized = false // This required read owns recovery now. An older fallback must not @@ -4327,11 +4994,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detailRefreshGeneration &+= 1 let generation = detailRefreshGeneration let sessionGeneration = environmentGeneration + let streamGeneration = detailStreamGeneration detailRefreshTask = Task { [weak self] in do { // Shell updates can be coalesced. A required replacement cannot // apply more thread events until its snapshot arrives. - if !force { + if !force && !reconcile { try await Task.sleep(for: .milliseconds(250)) } } catch { @@ -4347,11 +5015,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, generation: sessionGeneration ) { do { - try await self.refreshThread(id: threadID, client: client) + try await self.refreshThread( + id: threadID, client: client, + expectedStreamGeneration: reconcile ? streamGeneration : nil, + reconcile: reconcile + ) } catch is CancellationError { // Closing a thread cancels its read without changing its status. } catch { - if !Task.isCancelled, + if !reconcile, !Task.isCancelled, self.detailRefreshGeneration == generation, self.activeThreadID == threadID, self.activeRawThread == nil || self.detailStreamTask == nil { @@ -4376,6 +5048,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, id: route.uiID, state: warmConnectionID != nil ? .live : .catchingUp )) ensureDetailCatchUpFallback(route, generation: streamGeneration) + startSelectedThreadReconciliation(route, generation: streamGeneration) let retryDelay = threadRetryDelay detailStreamTask = Task { [weak self] in var failedAttempts = 0 @@ -4500,6 +5173,60 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, continuation.yield(.threadSync(id: route.uiID, state: .failed(message))) } + /// The selected transcript gets a bounded HTTP check even when its socket + /// remains healthy but subscription delivery stops. Other threads are untouched. + private func startSelectedThreadReconciliation(_ route: NativeThreadRoute, generation: Int) { + stopSelectedThreadReconciliation() + guard isForeground else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() + let sessionGeneration = environmentGeneration + let sleep = selectedThreadReconciliationSleep + selectedThreadReconciliationTask = Task { [weak self] in + while !Task.isCancelled { + let remaining = self.map { owner in + max(.zero, .seconds(30) - (owner.selectedThreadLastProgressAt ?? owner.selectedThreadReconciliationNow()) + .duration(to: owner.selectedThreadReconciliationNow())) + } ?? .seconds(30) + do { try await sleep(remaining) } catch { return } + guard !Task.isCancelled, let self, self.isForeground, + self.environmentGeneration == sessionGeneration, + self.isCurrentDetail(route, generation: generation) else { return } + if let progress = self.selectedThreadLastProgressAt, + progress.duration(to: self.selectedThreadReconciliationNow()) < .seconds(30) { + self.selectedThreadReconciliationReceipt(.deferred(threadID: route.uiID)) + continue + } + guard self.detailRefreshTask == nil, self.detailCatchUpTask == nil, + self.acceptedCommandRefreshes[route.uiID] == nil, + self.activeRawThread != nil, + self.activeThreadPage?.isLoading != true || self.pendingOlderThreadPage != nil else { + self.selectedThreadLastProgressAt = self.selectedThreadReconciliationNow() + self.selectedThreadReconciliationReceipt(.deferred(threadID: route.uiID)) + continue + } + self.scheduleDetailRefresh(threadID: route.uiID, client: route.client, reconcile: true) + self.selectedThreadReconciliationRefreshGeneration = self.detailRefreshGeneration + let refresh = self.detailRefreshTask + await refresh?.value + guard !Task.isCancelled, self.isForeground, + self.environmentGeneration == sessionGeneration, + self.isCurrentDetail(route, generation: generation) else { return } + self.selectedThreadReconciliationRefreshGeneration = nil + self.selectedThreadLastProgressAt = self.selectedThreadReconciliationNow() + self.selectedThreadReconciliationReceipt(.finished(threadID: route.uiID)) + } + } + } + + private func stopSelectedThreadReconciliation() { + selectedThreadReconciliationTask?.cancel() + selectedThreadReconciliationTask = nil + if selectedThreadReconciliationRefreshGeneration == detailRefreshGeneration { + resetDetailRefresh() + } + selectedThreadReconciliationRefreshGeneration = nil + } + private func isCurrentDetail(_ route: NativeThreadRoute, generation: Int) -> Bool { detailStreamGeneration == generation && activeThreadID == route.uiID @@ -4597,6 +5324,16 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } + private var threadReceiptGate = NativeThreadReceiptGate() + + private func publishThreadReceipt(_ threadID: String, source: FeatureThreadReceipt.Source) { + guard let receipt = threadReceiptGate.receive( + threadID: threadID, selectedThreadID: activeThreadID, + generation: detailStreamGeneration, now: Date(), source: source + ) else { return } + continuation.yield(.threadReceipt(receipt)) + } + private func consumeDetailStreamItem( _ item: ThreadStreamItem, route: NativeThreadRoute, @@ -4616,6 +5353,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, subscriptionEpoch < requiredEpoch { return } guard snapshot.snapshotSequence >= (activeThreadSequence ?? 0), activeRawThread == nil || snapshot.snapshotSequence > (activeThreadSequence ?? 0) else { return } + guard snapshot.thread.id == route.wireID else { return } + publishThreadReceipt(route.uiID, source: .detailSnapshot) + selectedThreadLastProgressAt = selectedThreadReconciliationNow() beginWarmReplayIfNeeded(route) resetDetailRefresh() detailSnapshotRequiredAfterEpoch = nil @@ -4627,6 +5367,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, scheduleRawDetailPublish(route: route, mutation: .full) if detailCompletionReceived { markDetailSynchronized(route) } case let .event(event): + if case .object = event, + let type = event["type"]?.stringValue, !type.isEmpty, + let occurredAt = event["occurredAt"]?.stringValue, + NativeTimestampParser.parse(occurredAt) != nil, + event["payload"]?["threadId"]?.stringValue == route.wireID, + case let .number(value) = event["sequence"], + let sequence = Int(exactly: value), sequence > (activeThreadSequence ?? 0) { + publishThreadReceipt(route.uiID, source: .detailEvent) + } guard let current = activeRawThread else { // Do not apply later events to a snapshot that missed earlier // ones. It must cover every event skipped while replacing it. @@ -4642,7 +5391,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, scheduleDetailRefresh(threadID: route.uiID, client: route.client, force: true) return } - let reduction = NativeThreadDetailReducer.apply(event, to: current) + let reduction = NativeThreadDetailReducer.apply( + event, to: current, afterSequence: activeThreadSequence ?? 0 + ) if reduction.sequence < 0 { threadHistoryEpoch &+= 1 detailSnapshotRequiredAfterEpoch = threadHistoryEpoch @@ -4653,6 +5404,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, return } guard reduction.sequence > (activeThreadSequence ?? 0) else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() beginWarmReplayIfNeeded(route) switch reduction.result { case let .updated(thread): @@ -4685,8 +5437,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pendingDetailRenderMutations.formUnion(mutation) guard detailPublishTask == nil else { return } let streamGeneration = detailStreamGeneration + let sleep = detailPublicationSleep detailPublishTask = Task { [weak self] in - try? await Task.sleep(for: .milliseconds(80)) + do { try await sleep() } catch { return } guard let self else { return } guard !Task.isCancelled, self.detailStreamGeneration == streamGeneration, @@ -4752,6 +5505,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func resetDetailStream() { + stopSelectedThreadReconciliation() detailStreamGeneration &+= 1 detailCompletionReceived = false detailWasSynchronized = false @@ -4849,6 +5603,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) { let savedIDs = Set(savedEnvironments.map(\.id)) environmentClients = environmentClients.filter { savedIDs.contains($0.key) } + shellConnectionIDsByEnvironmentID = shellConnectionIDsByEnvironmentID.filter { savedIDs.contains($0.key) } shellsByEnvironmentID = shellsByEnvironmentID.filter { savedIDs.contains($0.key) } shellProjectionCache = shellProjectionCache.filter { savedIDs.contains($0.key) } serverConfigsByEnvironmentID = serverConfigsByEnvironmentID.filter { @@ -4870,31 +5625,33 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, savedIDs.contains($0.key) } - for load in loads { - environmentClients[load.environment.id] = load.client - if let config = load.config { - setServerConfig(config, environmentID: load.environment.id) - if load.environment.id == activeEnvironment?.id { - latestServerConfig = config - } + for load in loads { applyEnvironmentLoad(load) } + rebuildEntityIndexes(savedEnvironments) + } + + private func applyEnvironmentLoad(_ load: EnvironmentShellLoad) { + environmentClients[load.environment.id] = load.client + if let config = load.config { + setServerConfig(config, environmentID: load.environment.id) + if load.environment.id == activeEnvironment?.id { + latestServerConfig = config } - if let shell = load.shell { - if shell.snapshotSequence - >= (shellsByEnvironmentID[load.environment.id]?.snapshotSequence ?? .min) { - shellsByEnvironmentID[load.environment.id] = shell - } - environmentConnectionStates[load.environment.id] = .connected - environmentConnectionDetails[load.environment.id] = nil - } else if load.credentialRejected && load.environment.kind != .managedDPoP { - environmentConnectionStates[load.environment.id] = .needsPairing - environmentConnectionDetails[load.environment.id] = Self.needsPairingDetail - } else { - environmentConnectionStates[load.environment.id] = .disconnected - environmentConnectionDetails[load.environment.id] = - load.failureDetail ?? "That server is currently unreachable." + } + if let shell = load.shell { + if shell.snapshotSequence + >= (shellsByEnvironmentID[load.environment.id]?.snapshotSequence ?? .min) { + shellsByEnvironmentID[load.environment.id] = shell } + environmentConnectionStates[load.environment.id] = .connected + environmentConnectionDetails[load.environment.id] = nil + } else if load.credentialRejected && load.environment.kind != .managedDPoP { + environmentConnectionStates[load.environment.id] = .needsPairing + environmentConnectionDetails[load.environment.id] = Self.needsPairingDetail + } else { + environmentConnectionStates[load.environment.id] = .disconnected + environmentConnectionDetails[load.environment.id] = + load.failureDetail ?? "That server is currently unreachable." } - rebuildEntityIndexes(savedEnvironments) } static let needsPairingDetail = "This computer no longer accepts the saved pairing. Pair again." @@ -4906,10 +5663,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() + // Rejected credentials cannot hydrate; stop the quiet shell reads too. + shellReconciliationTask?.cancel() pollingTask = nil fallbackPollingTask = nil configurationTask = nil - lastShellEventAt = nil + shellReconciliationTask = nil + activeStreamIsAuthoritative = false if activeEnvironment?.kind == .managedDPoP { emitConnection(.disconnected, detail: detail) } else { @@ -5042,42 +5802,60 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, indexedProvisionalRoutes = provisionalThreadRoutes } + /// A missing expectation means an ordinary publication. A present value + /// retains even an absent source cache or socket as an exact expectation. + private struct RefreshPublicationGuard { + let bootstrapID: UUID + let sourceSequence: Int? + let connectionID: UUID? + let activeAuthorityRevision: Int? + } + private func refresh(client: T3Client, includeArchived: Bool = false) async throws { let environment = client.environment let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + let sourceSequence = shellsByEnvironmentID[environment.id]?.snapshotSequence + let authorityRevision = activeHTTPAuthorityRevision + // Archive RPC can establish the first socket. Bind the subsequent shell + // read to that socket instead of treating its creation as replacement. + let archivedShell = includeArchived ? try? await client.archivedShellSnapshot() : nil + let connectionID = await client.currentConnectionID() + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID else { throw CancellationError() } let shell = try await client.shellSnapshot() - guard isKnownClient(client, environmentID: environment.id, generation: generation) else { + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, + isKnownClient(client, environmentID: environment.id, generation: generation) else { throw CancellationError() } - guard shell.snapshotSequence - >= (shellsByEnvironmentID[environment.id]?.snapshotSequence ?? .min) else { - return - } - shellsByEnvironmentID[environment.id] = shell - if activeEnvironment?.id == environment.id { - latestShell = shell - } - if includeArchived, - let archivedShell = try? await client.archivedShellSnapshot(), - isKnownClient(client, environmentID: environment.id, generation: generation) { - archivedThreadsByEnvironmentID[environment.id] = archivedShell.threads.map { - mapThread($0, environment: environment) - } - archivedShellThreadsByEnvironmentID[environment.id] = Dictionary( - uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } - ) + let currentConnectionID = await client.currentConnectionID() + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, + isKnownClient(client, environmentID: environment.id, generation: generation), + connectionID == currentConnectionID, + sourceSequence == shellsByEnvironmentID[environment.id]?.snapshotSequence, + activeEnvironment?.id != environment.id || authorityRevision == activeHTTPAuthorityRevision else { + throw CancellationError() } - await emitSnapshot(shell, client: client, expectedGeneration: generation) + guard shell.snapshotSequence + >= (shellsByEnvironmentID[environment.id]?.snapshotSequence ?? .min) else { return } + await emitSnapshot( + shell, client: client, expectedGeneration: generation, + refreshGuard: RefreshPublicationGuard( + bootstrapID: bootstrapID, sourceSequence: sourceSequence, connectionID: connectionID, + activeAuthorityRevision: activeEnvironment?.id == environment.id ? authorityRevision : nil + ), archivedShell: archivedShell + ) } private func scheduleArchivedRefresh(client: T3Client, environment: Environment) { archivedRefreshTask?.cancel() let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID archivedRefreshTask = Task { [weak self] in - guard let self, - let archivedShell = try? await client.archivedShellSnapshot(), + guard let self else { return } + defer { self.aggregateRefreshReceipt(.archiveFinished(environmentID: environment.id)) } + guard let archivedShell = try? await client.archivedShellSnapshot(), !Task.isCancelled, - self.isCurrentSession(client: client, generation: generation) else { + self.isCurrentForeground(client, generation: generation, bootstrapID: bootstrapID) else { return } self.archivedThreadsByEnvironmentID[environment.id] = archivedShell.threads.map { @@ -5087,7 +5865,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } ) if let shell = self.latestShell { - await self.emitSnapshot(shell, client: client, expectedGeneration: generation) + await self.emitSnapshot(shell, client: client, expectedGeneration: generation, markSourceConnected: false) } } } @@ -5103,7 +5881,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func refreshThread( - id: String, client: T3Client, expectedStreamGeneration: Int? = nil + id: String, client: T3Client, expectedStreamGeneration: Int? = nil, reconcile: Bool = false, + isSuperseded: (() -> Bool)? = nil ) async throws { let route = try threadRoute(for: id) guard route.client === client else { @@ -5115,16 +5894,100 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let supportsPagination = serverConfigsByEnvironmentID[ environment.id ]?.threadSnapshotPagination == true - let snapshot = try await client.threadSnapshot( + let loadedUsers = reconcile ? activeRawThread?.messages.filter { $0.role == "user" } ?? [] : [] + let loadedAnchor = loadedUsers.first?.id + let pendingHistory = reconcile ? pendingOlderThreadPage : nil + if let pendingHistory { + guard pendingHistory.epoch == historyEpoch, pendingHistory.threadID == route.uiID, + pendingHistory.environmentID == route.environmentID, + (pendingHistory.snapshot.page?.threadSequence ?? 0) >= (activeThreadSequence ?? 0) else { return } + } + let loadedIDs = Set(loadedUsers.map(\.id)) + // Pending rows determine the requested extent only. The published data + // always comes from a new authoritative snapshot, never a cached union. + let pendingUsers = pendingHistory?.snapshot.thread.messages.filter { + $0.role == "user" && !loadedIDs.contains($0.id) + } ?? [] + let retainedUsers = pendingUsers + loadedUsers + let retainedIDs = Set(retainedUsers.map(\.id)) + let retainedPage = activeThreadPage + let retainedThreads = reconcile ? [activeRawThread, pendingHistory?.snapshot.thread].compactMap { $0 } : [] + let retainedMessageIDs = Set(retainedThreads.flatMap { $0.messages.map(\.id) }) + let retainedActivityIDs = Set(retainedThreads.flatMap { $0.activities.map(\.id) }) + let retainedCheckpointTurns = Set(retainedThreads.flatMap { $0.checkpoints.map(\.turnId) }) + let turnLimit = max(Self.initialThreadUserTurnLimit, retainedUsers.count) + + func ownsReconciliationExtent() -> Bool { + !reconcile || ( + pendingOlderThreadPage == pendingHistory + && threadHistoryEpoch == historyEpoch + && activeThreadPage == retainedPage + && (activeThreadPage?.isLoading != true || pendingHistory != nil) + && loadedAnchor == activeRawThread?.messages.first(where: { $0.role == "user" })?.id + ) + } + func coversRetainedHistory(_ thread: OrchestrationThread) -> Bool { + // A server page can hit its raw-turn cap before reaching even one + // user row. Preserve every loaded collection, not just user anchors. + retainedMessageIDs.isSubset(of: Set(thread.messages.map(\.id))) + && retainedActivityIDs.isSubset(of: Set(thread.activities.map(\.id))) + && retainedCheckpointTurns.isSubset(of: Set(thread.checkpoints.map(\.turnId))) + } + func additionalTailUsers(in thread: OrchestrationThread) -> Int? { + let users = thread.messages.filter { $0.role == "user" } + guard let lastKnown = users.lastIndex(where: { retainedIDs.contains($0.id) }) else { return nil } + return users.distance(from: users.index(after: lastKnown), to: users.endIndex) + } + var snapshot = try await client.threadSnapshot( id: route.wireID, - turnLimit: supportsPagination ? Self.initialThreadUserTurnLimit : nil, + turnLimit: supportsPagination ? turnLimit : nil, timeoutInterval: threadSnapshotTimeoutInterval ) + guard !Task.isCancelled, + expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true, + isKnownClient(client, environmentID: environment.id, generation: generation) else { + throw CancellationError() + } + guard ownsReconciliationExtent() else { return } + // A later delivered cursor does not prove that every earlier row arrived. + // Independent HTTP repair must compare content, even at an equal watermark. + if reconcile, pendingHistory == nil, let current = activeRawThread, + snapshot.snapshotSequence >= (activeThreadSequence ?? 0), + snapshot.thread == current { + return + } + if reconcile, snapshot.page?.hasMore == true, + !coversRetainedHistory(snapshot.thread) { + if let added = additionalTailUsers(in: snapshot.thread), added > 0 { + // One extra user turn needs one extra slot, not an automatic + // page of older history. Do not keep expanding a moving head. + snapshot = try await client.threadSnapshot( + id: route.wireID, turnLimit: turnLimit + added, + timeoutInterval: threadSnapshotTimeoutInterval + ) + guard !Task.isCancelled, ownsReconciliationExtent(), + expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { return } + guard additionalTailUsers(in: snapshot.thread) == added else { return } + } + if snapshot.page?.hasMore == true, + !coversRetainedHistory(snapshot.thread) { + // Missing retained rows or a raw-turn cap get one full read. Never + // resurrect old rows by unioning them into a rewound thread. + snapshot = try await client.threadSnapshot( + id: route.wireID, timeoutInterval: threadSnapshotTimeoutInterval + ) + } + } guard !Task.isCancelled, isKnownClient(client, environmentID: environment.id, generation: generation), expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { throw CancellationError() } + // A newer accepted command makes this snapshot stale; its owner reads again. + if isSuperseded?() == true { return } + guard ownsReconciliationExtent() else { return } + if let pendingWatermark = pendingHistory?.snapshot.page?.threadSequence, + snapshot.snapshotSequence < pendingWatermark { return } if activeThreadID == route.uiID { if activeRawThread == nil, historyEpoch != threadHistoryEpoch { return @@ -5167,7 +6030,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, wasSynchronized: false, connectionID: nil ) - if activeThreadID == route.uiID, + if !reconcile, activeThreadID == route.uiID, detailCompletionReceived || serverConfigsByEnvironmentID[environment.id]?.threadResumeCompletionMarker != true { markDetailSynchronized(route) @@ -5180,16 +6043,32 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, _ shell: OrchestrationShellSnapshot, client sourceClient: T3Client, expectedGeneration: Int, + refreshGuard: RefreshPublicationGuard? = nil, + archivedShell: OrchestrationShellSnapshot? = nil, markSourceConnected: Bool = true ) async { let sourceEnvironment = sourceClient.environment guard !Task.isCancelled, + refreshGuard.map({ + $0.bootstrapID == foregroundBootstrapID + && $0.sourceSequence == shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence + && ($0.activeAuthorityRevision.map { $0 == activeHTTPAuthorityRevision } ?? true) + }) ?? true, let environment = activeEnvironment, isKnownClient( sourceClient, environmentID: sourceEnvironment.id, generation: expectedGeneration ) else { return } let environments = (try? await runtime.environments()) ?? [environment] + // The metadata await can outlive the socket too. Read its identity last, + // then validate all expectations together before any cache mutation. + let currentConnectionID = refreshGuard == nil ? nil : await sourceClient.currentConnectionID() guard !Task.isCancelled, + refreshGuard.map({ $0.connectionID == currentConnectionID }) ?? true, + refreshGuard.map({ + $0.bootstrapID == foregroundBootstrapID + && $0.sourceSequence == shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence + && ($0.activeAuthorityRevision.map { $0 == activeHTTPAuthorityRevision } ?? true) + }) ?? true, isKnownClient( sourceClient, environmentID: sourceEnvironment.id, generation: expectedGeneration ), @@ -5199,6 +6078,17 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, >= (shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence ?? .min) else { return } + if let archivedShell { + archivedThreadsByEnvironmentID[sourceEnvironment.id] = archivedShell.threads.map { + mapThread($0, environment: sourceEnvironment) + } + archivedShellThreadsByEnvironmentID[sourceEnvironment.id] = Dictionary( + uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } + ) + } + if let refreshGuard { + shellConnectionIDsByEnvironmentID[sourceEnvironment.id] = refreshGuard.connectionID + } shellsByEnvironmentID[sourceEnvironment.id] = shell if markSourceConnected { environmentConnectionStates[sourceEnvironment.id] = .connected @@ -5206,6 +6096,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } if sourceEnvironment.id == environment.id { latestShell = shell + if markSourceConnected { + activeHasHydrated = true + activeShellEpochHasSnapshot = true + activeHTTPAuthorityRevision &+= 1 + } } rebuildEntityIndexes(environments) synchronizeActiveDetail( @@ -5215,8 +6110,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let connectionState: FeatureConnection.State let connectionDetail: String? if sourceEnvironment.id == environment.id, markSourceConnected { - connectionState = .connected - connectionDetail = nil + connectionState = activeStreamIsAuthoritative ? .connected : .reconnecting + connectionDetail = activeStreamIsAuthoritative ? nil + : latestSnapshot?.connection.detail ?? "Live updates reconnecting. Refreshing over HTTP." } else { connectionState = latestSnapshot?.connection.state ?? environmentConnectionStates[environment.id] @@ -5241,10 +6137,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard activeThreadEnvironmentID == environment.id, let threadID = activeThreadID, let wireID = threadWireIDs[threadID], - let shellThread = shell.threads.first(where: { $0.id == wireID }), - var detail = latestDetails[threadID] else { + let shellThread = shell.threads.first(where: { $0.id == wireID }) else { return } + repairMissingCompletedTurn(shellThread, sequence: shell.snapshotSequence, threadID: threadID) + guard var detail = latestDetails[threadID] else { return } let backgroundLiveness = shellThread.backgroundLiveness let backgroundWorkIsActive = backgroundLiveness == .working @@ -5255,22 +6152,26 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detail.thread.supportsTitleRegeneration = capabilities?.threadTitleRegeneration detail.thread.supportsPullRequestLinking = capabilities?.threadPullRequestLinking detail.thread.supportsMultiplePullRequests = capabilities?.threadPullRequests - let sessionIsLive = shellThread.session?.status == "starting" - || shellThread.session?.status == "running" - detail.thread.state = Self.resolveThreadState( - latestTurn: shellThread.latestTurn, - session: shellThread.session, - hasApprovals: !detail.approvals.isEmpty, - hasUserInput: !detail.userInputs.isEmpty, - backgroundLiveness: backgroundLiveness - ) - detail.thread.workingStartedAt = workingStartedAt( - latestTurn: shellThread.latestTurn, - session: shellThread.session, - backgroundWorkIsActive: backgroundWorkIsActive, - fallbackUpdatedAt: shellThread.updatedAt - ) - if shell.snapshotSequence >= (activeThreadSequence ?? .min) { + let shellMetadataIsCurrent = shell.snapshotSequence >= (activeThreadSequence ?? .min) + let latestTurn = shellMetadataIsCurrent ? shellThread.latestTurn : activeRawThread?.latestTurn + let session = shellMetadataIsCurrent ? shellThread.session : activeRawThread?.session + let sessionIsLive = session?.status == "starting" || session?.status == "running" + if shellMetadataIsCurrent || activeRawThread != nil { + detail.thread.state = Self.resolveThreadState( + latestTurn: latestTurn, + session: session, + hasApprovals: !detail.approvals.isEmpty, + hasUserInput: !detail.userInputs.isEmpty, + backgroundLiveness: backgroundLiveness + ) + detail.thread.workingStartedAt = workingStartedAt( + latestTurn: latestTurn, + session: session, + backgroundWorkIsActive: backgroundWorkIsActive, + fallbackUpdatedAt: shellThread.updatedAt + ) + } + if shellMetadataIsCurrent { applyShellMetadataAuthority(from: shellThread, to: &detail.thread) if let compaction = detailRenderCaches[threadID]?.compaction { detail.isCompacting = compaction.isActive( @@ -5288,6 +6189,44 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, publish(detail, threadID: threadID, renderCacheIsSource: true) } + private func repairMissingCompletedTurn( + _ shellThread: OrchestrationThreadShell, sequence: Int, threadID: String + ) { + guard let rawThread = activeRawThread, + sequence >= (activeThreadSequence ?? .min), + let completed = shellThread.latestTurn, completed.state == "completed", + let route = try? threadRoute(for: threadID) else { return } + if let messageID = completed.assistantMessageId { + if let message = rawThread.messages.first(where: { $0.id == messageID }), + !message.streaming { + flushDetailPublish(route) + return + } + } else if rawThread.latestTurn?.turnId == completed.turnId, + rawThread.latestTurn?.state == completed.state, + rawThread.latestTurn?.completedAt == completed.completedAt, + rawThread.messages.contains(where: { + $0.role == "assistant" && $0.turnId == completed.turnId && !$0.streaming + }), + !rawThread.messages.contains(where: { + $0.role == "assistant" && $0.turnId == completed.turnId && $0.streaming + }) { + flushDetailPublish(route) + return + } + + // Equal-cursor completion may flush final content, but only a newer + // shell can prove that detail is missing and require HTTP repair. + guard sequence > (activeThreadSequence ?? .min) else { return } + + // Keep rendered partial text while requiring a snapshot that includes + // the shell's completion. Later detail events advance this same floor. + flushDetailPublish(route) + activeRawThread = nil + activeThreadSequence = sequence + scheduleDetailRefresh(threadID: threadID, client: route.client, force: true) + } + /// Thread-only shell changes stay granular so Home does not replace and /// diff the aggregate snapshot for every active turn update. Structural /// changes retain the canonical snapshot event as a safe fallback. @@ -5405,23 +6344,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, return nil } - var changedIDs = Set(mutations.messages.map(\.id)) - for activity in mutations.activities { - if activity.kind == "user-input.answer-submitted" { - changedIDs.formUnion(NativeQuestionAnswerHistory.messages( - activity, createdAt: parseDate(activity.createdAt) - ).map(\.id)) - } - if NativeActivityNotice.accepts(activity) { - changedIDs.insert("activity-\(activity.id)") - } else if NativeWorkLogAccumulator.accepts(activity) { - changedIDs.insert("work-log-\(activity.turnId ?? "unscoped")") - } - } - guard let cache = detailRenderCaches[next.thread.id] else { return nil } + let changedIDs = cache.transcript.changedIDs let changedMessages = changedIDs.compactMap { id in - cache.mergedIndexByID[id].map { cache.mergedMessages[$0] } + cache.transcript.indexByID[id].map { cache.transcript.messages[$0] } } let appendedCount = next.messages.count - previous.messages.count let appendedMessageIDs = appendedCount == 0 @@ -5510,14 +6436,29 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environments: [Environment], activeEnvironment: Environment, connectionState: FeatureConnection.State, - connectionDetail: String? = nil + connectionDetail: String? = nil, + changedEnvironmentIDs: Set? = nil ) -> FeatureSnapshot { let enabledEnvironments = environments.filter(\.isEnabled) let enabledIDs = Set(enabledEnvironments.map(\.id)) shellProjectionCache = shellProjectionCache.filter { enabledIDs.contains($0.key) } var threads: [FeatureThread] = [] var projects: [FeatureProject] = [] + let previousThreads = changedEnvironmentIDs == nil ? [:] + : Dictionary(grouping: latestSnapshot?.threads ?? [], by: \.environmentID) + let previousProjects = changedEnvironmentIDs == nil ? [:] + : Dictionary(grouping: latestSnapshot?.projects ?? [], by: \.environmentID) + let previousEnvironments = Dictionary(uniqueKeysWithValues: + (latestSnapshot?.environments ?? []).map { ($0.id, $0) } + ) for environment in enabledEnvironments { + if let changedEnvironmentIDs, !changedEnvironmentIDs.contains(environment.id), + previousEnvironments[environment.id] + == mapEnvironment(environment, activeID: activeEnvironment.id) { + threads.append(contentsOf: previousThreads[environment.id] ?? []) + projects.append(contentsOf: previousProjects[environment.id] ?? []) + continue + } // Take ownership while updating so the cache does not copy its // retained arrays when one row changes. var projection = shellProjectionCache.removeValue(forKey: environment.id) @@ -5597,6 +6538,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let providersByEnvironment = enabledEnvironments.reduce( into: [String: [FeatureProvider]]() ) { catalogues, environment in + if let changedEnvironmentIDs, !changedEnvironmentIDs.contains(environment.id), + let previous = latestSnapshot?.providersByEnvironment?[environment.id] { + catalogues[environment.id] = previous + return + } guard let shell = shellsByEnvironmentID[environment.id] else { return } catalogues[environment.id] = mapProviders( environmentID: environment.id, @@ -5780,6 +6726,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, backgroundWorkIsActive: backgroundWorkIsActive, fallbackUpdatedAt: thread.updatedAt ), + latestTurnID: thread.latestTurn?.turnId, + latestTurnState: thread.latestTurn?.state, latestTurnCompletedAt: thread.latestTurn?.completedAt.flatMap(parseValidDate), settlementFacts: settlementFacts( override: thread.settledOverride, @@ -5871,6 +6819,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, backgroundWorkIsActive: backgroundWorkIsActive, fallbackUpdatedAt: thread.updatedAt ), + latestTurnID: thread.latestTurn?.turnId, + latestTurnState: thread.latestTurn?.state, latestTurnCompletedAt: thread.latestTurn?.completedAt.flatMap(parseValidDate), settlementFacts: settlementFacts( override: thread.settledOverride, @@ -5907,38 +6857,30 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, cache.compaction.apply(raw, createdAt: parseDate(raw.createdAt)) } resetPendingRequests(thread, environment: environment, cache: cache) - let notices = thread.activities.compactMap { activity in - cache.compaction.apply(activity) - return NativeActivityNotice.message(activity, createdAt: parseDate(activity.createdAt)) - } + for activity in thread.activities { cache.compaction.apply(activity) } let sessionIsLive = thread.session?.status == "starting" || thread.session?.status == "running" - let activityMessages = (notices + collapsedWorkLogs( - thread.activities, + cache.transcript = NativeTranscriptTimeline( + messages: thread.messages.compactMap { cache.messagesByID[$0.id] }, + activities: thread.activities, sessionIsLive: sessionIsLive - )) - .sorted { $0.createdAt < $1.createdAt } - seedWorkLogs(thread.activities, sessionIsLive: sessionIsLive, cache: cache) + ) cache.subagents.reset(with: thread.activities) - let messages = thread.messages.compactMap { cache.messagesByID[$0.id] } - cache.mergedMessages = (messages + activityMessages) - .sorted { $0.createdAt < $1.createdAt } - rebuildMergedIndexes(cache) cache.isInitialized = true } else if let mutations { - for message in mutations.messages { - let mapped = mapMessage(message, environmentID: environment.id) - cache.messagesByID[message.id] = mapped - cache.compaction.apply(message, createdAt: mapped.createdAt) - upsertMergedMessage(mapped, cache: cache) - } - for activity in mutations.activities { - applyActivityMutation( - activity, - threadID: threadID, - environment: environment, - cache: cache - ) + cache.transcript.changedIDs.removeAll(keepingCapacity: true) + for mutation in mutations.ordered { + switch mutation { + case let .message(message): + let mapped = mapMessage(message, environmentID: environment.id) + cache.messagesByID[message.id] = mapped + cache.compaction.apply(message, createdAt: mapped.createdAt) + cache.transcript.append(mapped) + case let .activity(activity): + applyActivityMutation(activity, threadID: threadID, environment: environment, cache: cache) + case .full, .metadata, .none: + break + } } } else { assertionFailure("Initialized detail caches require an incremental mutation") @@ -5952,14 +6894,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let backgroundWorkIsActive = backgroundLiveness == .working let sessionIsLive = thread.session?.status == "starting" || thread.session?.status == "running" - if !sessionIsLive { - for (groupID, var accumulator) in cache.workLogsByGroupID - where accumulator.hasActiveWork { - accumulator.clearActiveWork() - cache.workLogsByGroupID[groupID] = accumulator - upsertMergedMessage(accumulator.message(groupID: groupID), cache: cache) - } - } + if !sessionIsLive { cache.transcript.finishActiveWork() } mappedThread.state = Self.resolveThreadState( latestTurn: thread.latestTurn, session: thread.session, @@ -5976,7 +6911,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } return FeatureThreadDetail( thread: mappedThread, - messages: cache.mergedMessages, + messages: cache.transcript.messages, approvals: cache.approvals, userInputs: cache.userInputs, page: page, @@ -6097,7 +7032,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environmentID: route.environmentID ) } - cache.mergedMessages = mergedMessages + cache.transcript.messages = mergedMessages rebuildMergedIndexes(cache) } @@ -6122,14 +7057,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let messages = thread.messages.map { mapMessage($0, environmentID: environmentID) } - let workIsLive = thread.session?.status == "starting" - || thread.session?.status == "running" - || backgroundLiveness(threadID: thread.id, environmentID: environmentID) == .working - let activities = thread.activities.compactMap { - NativeActivityNotice.message($0, createdAt: parseDate($0.createdAt)) - } - + collapsedWorkLogs(thread.activities, sessionIsLive: workIsLive) - return (messages + activities).sorted { $0.createdAt < $1.createdAt } + // Older pages contain earlier turn windows, but carry current session + // metadata. That metadata must not revive their historical tool rows. + return NativeTranscriptTimeline( + messages: messages, activities: thread.activities, sessionIsLive: false + ).messages } private func mergingOlderHistory( @@ -6181,31 +7113,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func rebuildMergedIndexes(_ cache: NativeDetailRenderCache) { - cache.mergedIndexByID = cache.mergedMessages.enumerated().reduce(into: [:]) { - $0[$1.element.id] = $1.offset - } - } - - /// Known stream events are chronological, so new render entities land at - /// the tail and existing streaming/work-log entities patch in constant time. - private func upsertMergedMessage( - _ message: FeatureMessage, - cache: NativeDetailRenderCache - ) { - if let index = cache.mergedIndexByID[message.id] { - cache.mergedMessages[index] = message - return - } - if let last = cache.mergedMessages.last, last.createdAt > message.createdAt { - // Out-of-order events are rare; preserve correctness while keeping - // the normal append path independent of transcript size. - cache.mergedMessages.append(message) - cache.mergedMessages.sort { $0.createdAt < $1.createdAt } - rebuildMergedIndexes(cache) - return - } - cache.mergedIndexByID[message.id] = cache.mergedMessages.count - cache.mergedMessages.append(message) + cache.transcript.rebuildIndexes() } private func applyActivityMutation( @@ -6228,27 +7136,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environment: environment, cache: cache ) - if let notice = NativeActivityNotice.message(activity, createdAt: parseDate(activity.createdAt)) { - upsertMergedMessage(notice, cache: cache) - } - for answer in NativeQuestionAnswerHistory.messages(activity, createdAt: parseDate(activity.createdAt)) { - upsertMergedMessage(answer, cache: cache) - } - guard NativeWorkLogAccumulator.accepts(activity), - cache.workLogActivityIDs.insert(activity.id).inserted else { - return - } - let groupID = activity.turnId ?? "unscoped" - var accumulator = cache.workLogsByGroupID[groupID] ?? NativeWorkLogAccumulator() - accumulator.append( - activity, - preview: previewText(activity.payload["detail"]?.stringValue), - createdAt: parseDate(activity.createdAt) - ) - cache.workLogsByGroupID[groupID] = accumulator - guard accumulator.hasContent else { return } - let message = accumulator.message(groupID: groupID) - upsertMergedMessage(message, cache: cache) + cache.transcript.append(activity) } /// Decorate-sort so each timestamp is parsed once (via the memoized date @@ -6270,32 +7158,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, return decorated.map(\.activity) } - private func seedWorkLogs( - _ activities: [OrchestrationActivity], - sessionIsLive: Bool, - cache: NativeDetailRenderCache - ) { - cache.workLogsByGroupID.removeAll(keepingCapacity: true) - cache.workLogActivityIDs.removeAll(keepingCapacity: true) - for activity in sortedByCreation(activities) - where NativeWorkLogAccumulator.accepts(activity) { - cache.workLogActivityIDs.insert(activity.id) - let groupID = activity.turnId ?? "unscoped" - var accumulator = cache.workLogsByGroupID[groupID] ?? NativeWorkLogAccumulator() - accumulator.append( - activity, - preview: previewText(activity.payload["detail"]?.stringValue), - createdAt: parseDate(activity.createdAt) - ) - cache.workLogsByGroupID[groupID] = accumulator - } - if !sessionIsLive { - for groupID in cache.workLogsByGroupID.keys { - cache.workLogsByGroupID[groupID]?.clearActiveWork() - } - } - } - private func applyApprovalActivity( _ activity: OrchestrationActivity, threadID: String, @@ -6418,32 +7280,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) } - /// Lifecycle updates can number in the thousands on a long turn. Keep the - /// primary transcript message-sized while preserving a bounded, expandable - /// summary for each turn. - private func collapsedWorkLogs( - _ activities: [OrchestrationActivity], - sessionIsLive: Bool - ) -> [FeatureMessage] { - let groups = Dictionary(grouping: sortedByCreation(activities).filter { - NativeWorkLogAccumulator.accepts($0) - }) { activity in - activity.turnId ?? "unscoped" - } - return groups.compactMap { groupID, group in - var accumulator = NativeWorkLogAccumulator() - for activity in group { - accumulator.append( - activity, - preview: previewText(activity.payload["detail"]?.stringValue), - createdAt: parseDate(activity.createdAt) - ) - } - if !sessionIsLive { accumulator.clearActiveWork() } - return accumulator.hasContent ? accumulator.message(groupID: groupID) : nil - } - } - /// Snapshot replay and live updates share terminal request rules. Request IDs /// are unique, so a late requested activity must not reopen a resolved request. private func resetPendingRequests( @@ -6696,6 +7532,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, providerCatalogCache[environmentID] = nil } serverConfigsByEnvironmentID[environmentID] = config + aggregateRefreshReceipt(.configurationApplied(environmentID: environmentID)) } private func mapProviders( @@ -7343,8 +8180,7 @@ enum NativeDetailRenderMutation: Equatable { struct NativeDetailRenderMutations { private(set) var hasUpdates = false private(set) var requiresFullRebuild = false - private(set) var messages: [OrchestrationMessage] = [] - private(set) var activities: [OrchestrationActivity] = [] + private(set) var ordered: [NativeDetailRenderMutation] = [] mutating func formUnion(_ mutation: NativeDetailRenderMutation) { if mutation != .none { hasUpdates = true } @@ -7352,33 +8188,24 @@ struct NativeDetailRenderMutations { switch mutation { case .full: requiresFullRebuild = true - messages.removeAll(keepingCapacity: true) - activities.removeAll(keepingCapacity: true) + ordered.removeAll(keepingCapacity: true) case let .message(message): - if let index = messages.firstIndex(where: { $0.id == message.id }) { - messages[index] = message - } else { - messages.append(message) - } - case let .activity(activity): - if let index = activities.firstIndex(where: { $0.id == activity.id }) { - activities[index] = activity - } else { - activities.append(activity) - } + if case let .message(previous) = ordered.last, previous.id == message.id { + ordered[ordered.count - 1] = mutation + } else { ordered.append(mutation) } + case .activity: + ordered.append(mutation) case .metadata, .none: break } } } +@MainActor private final class NativeDetailRenderCache { var isInitialized = false var messagesByID: [String: FeatureMessage] = [:] - var mergedMessages: [FeatureMessage] = [] - var mergedIndexByID: [String: Int] = [:] - var workLogsByGroupID: [String: NativeWorkLogAccumulator] = [:] - var workLogActivityIDs: Set = [] + var transcript = NativeTranscriptTimeline() var approvals: [FeatureApproval] = [] var userInputs: [FeatureUserInput] = [] var closedApprovalRequestIDs: Set = [] @@ -7545,136 +8372,6 @@ enum NativeSharedPreferenceChange { } } -struct NativeWorkLogAccumulator { - private static let terminalKinds = Set([ - "tool.completed", "task.completed", - ]) - private static let activeKinds = Set(["tool.started", "tool.updated"]) - private static let imageExtensions = Set([ - "avif", "bmp", "gif", "heic", "heif", "jpeg", "jpg", "png", "tif", "tiff", "webp", - ]) - - private(set) var count = 0 - private var visibleLines: [String] = [] - private var createdAt = Date.distantPast - private var activeEntries: [String: String] = [:] - private var activeOrder: [String] = [] - private var imagePaths: [String] = [] - private var toolPresentation: ToolActivityPresentation? - private var activePresentations: [String: ToolActivityPresentation] = [:] - - var hasActiveWork: Bool { !activeEntries.isEmpty } - var hasContent: Bool { count > 0 || hasActiveWork || !imagePaths.isEmpty } - - static func accepts(_ activity: OrchestrationActivity) -> Bool { - guard activeKinds.contains(activity.kind) - || (activity.tone != "error" && terminalKinds.contains(activity.kind)) else { - return false - } - if NativeActivityFilters.isPlanBoundaryTool(activity) { return false } - if NativeActivityFilters.isAgentInternal(activity) { return false } - return true - } - - mutating func append( - _ activity: OrchestrationActivity, - preview: String?, - createdAt: Date - ) { - if count == 0 && activeEntries.isEmpty { - self.createdAt = createdAt - } - let key = Self.lifecycleKey(activity) - toolPresentation = ToolActivityPresentation(payload: activity.payload) ?? activePresentations[key] - let label = activity.payload["title"]?.stringValue ?? activity.summary - let lifecycleStatus = activity.payload["status"]?.stringValue - let isTerminalUpdate = activity.kind == "tool.updated" - && lifecycleStatus.map { $0 != "inProgress" && $0 != "in_progress" } == true - if Self.activeKinds.contains(activity.kind) && !isTerminalUpdate - && activity.tone != "error" { - activeEntries[key] = label - activePresentations[key] = toolPresentation - activeOrder.removeAll { $0 == key } - activeOrder.append(key) - } else { - activeEntries[key] = nil - activePresentations[key] = nil - activeOrder.removeAll { $0 == key } - guard activity.tone != "error" else { return } - count += 1 - visibleLines.append("• \(preview ?? activity.summary)") - if visibleLines.count > 40 { - visibleLines.removeFirst(visibleLines.count - 40) - } - } - if let path = Self.viewedImagePath(activity), !imagePaths.contains(path) { - imagePaths.append(path) - if imagePaths.count > 8 { imagePaths.removeFirst(imagePaths.count - 8) } - } - } - - mutating func clearActiveWork() { - activeEntries.removeAll(keepingCapacity: true) - activePresentations.removeAll(keepingCapacity: true) - activeOrder.removeAll(keepingCapacity: true) - } - - func message(groupID: String) -> FeatureMessage { - var lines: [String] = [] - if count > visibleLines.count { - lines.append("\(count - visibleLines.count) earlier updates hidden") - } - lines.append(contentsOf: visibleLines) - var message = FeatureMessage( - id: "work-log-\(groupID)", - role: .tool, - text: lines.joined(separator: "\n"), - createdAt: createdAt, - state: .complete, - toolName: "Work log · \(count)", - workLogImagePaths: imagePaths.isEmpty ? nil : imagePaths, - activeWorkLabel: activeOrder.last.flatMap { activeEntries[$0] } - ) - message.toolPresentation = activeOrder.last.flatMap { activePresentations[$0] } ?? toolPresentation - return message - } - - private static func lifecycleKey(_ activity: OrchestrationActivity) -> String { - if let id = activity.payload["toolCallId"]?.stringValue - ?? activity.payload["data"]?["toolCallId"]?.stringValue { - return "id:\(id)" - } - let itemType = activity.payload["itemType"]?.stringValue ?? "" - let title = activity.payload["title"]?.stringValue ?? activity.summary - let detail = activity.payload["detail"]?.stringValue ?? "" - return "fallback:\([itemType, title, detail].map(normalizedLifecycleText).joined(separator: "|"))" - } - - private static func normalizedLifecycleText(_ value: String) -> String { - value.trimmingCharacters(in: .whitespacesAndNewlines) - .lowercased() - .replacingOccurrences( - of: #"\s+(complete|completed)$"#, - with: "", - options: .regularExpression - ) - } - - private static func viewedImagePath(_ activity: OrchestrationActivity) -> String? { - let itemType = normalizedLifecycleText(activity.payload["itemType"]?.stringValue ?? "") - let title = normalizedLifecycleText(activity.payload["title"]?.stringValue ?? activity.summary) - let qualifies = activity.payload["requestKind"]?.stringValue == "file-read" - || itemType == "image_view" - || (itemType == "dynamic_tool_call" && title == "read file") - guard qualifies, - let detail = activity.payload["detail"]?.stringValue, - !detail.contains("\n"), !detail.contains("\r") else { return nil } - let path = detail.trimmingCharacters(in: .whitespacesAndNewlines) - guard let ext = path.split(separator: ".").last?.lowercased(), - imageExtensions.contains(String(ext)) else { return nil } - return path - } -} enum NativeThreadDetailReductionResult: Equatable { case updated(OrchestrationThread) @@ -7704,7 +8401,8 @@ struct NativeThreadDetailReduction: Equatable { enum NativeThreadDetailReducer { static func apply( _ event: JSONValue, - to thread: OrchestrationThread + to thread: OrchestrationThread, + afterSequence: Int? = nil ) -> NativeThreadDetailReduction { guard case let .object(object) = event, let type = object["type"]?.stringValue, @@ -7719,6 +8417,13 @@ enum NativeThreadDetailReducer { ) } + // Validate ownership and the common envelope before skipping replayed events. + if let afterSequence, sequence >= 0, sequence <= afterSequence { + return NativeThreadDetailReduction( + sequence: sequence, result: .unchanged, renderMutation: .none + ) + } + let result: NativeThreadDetailReductionResult var renderMutation = NativeDetailRenderMutation.metadata switch type { @@ -8338,7 +9043,7 @@ private struct NativeProjectRoute { let client: T3Client } -private struct PendingOlderThreadPage { +private struct PendingOlderThreadPage: Equatable { let snapshot: OrchestrationThreadDetailSnapshot let epoch: Int let threadID: String @@ -8489,3 +9194,34 @@ private enum NativeFeatureClientError: LocalizedError { } } } + +/// Deterministic receipts at the passive transport/publication boundary. +enum NativePassiveShellReceipt: Sendable { + case shellApplied(environmentID: String, sequence: Int) + case httpFinished(environmentID: String) + case configurationApplied(environmentID: String) + case archiveFinished(environmentID: String) +} + +/// Receipt display has second precision. Collapse only updates invisible at that precision. +struct NativeThreadReceiptGate { + private var lastThreadID: String? + private var lastGeneration: Int? + private var lastSecond: TimeInterval? + private var lastSource: FeatureThreadReceipt.Source? + + mutating func receive( + threadID: String, selectedThreadID: String?, generation: Int, + now: Date, source: FeatureThreadReceipt.Source + ) -> FeatureThreadReceipt? { + guard threadID == selectedThreadID else { return nil } + let second = floor(now.timeIntervalSince1970) + guard lastThreadID != threadID || lastGeneration != generation + || lastSecond != second || lastSource != source else { return nil } + lastThreadID = threadID + lastGeneration = generation + lastSecond = second + lastSource = source + return FeatureThreadReceipt(threadID: threadID, receivedAt: now, source: source) + } +} diff --git a/apps/swift-ios/Core/NativeTranscriptTimeline.swift b/apps/swift-ios/Core/NativeTranscriptTimeline.swift new file mode 100644 index 000000000000..c6300ae6f987 --- /dev/null +++ b/apps/swift-ios/Core/NativeTranscriptTimeline.swift @@ -0,0 +1,250 @@ +import Foundation + +/// Tool updates may replace only the current row for the same explicit call. +/// Text, notices and other calls close that row; later updates start a continuation. +@MainActor +struct NativeTranscriptTimeline { + var messages: [FeatureMessage] = [] + var indexByID: [String: Int] = [:] + var changedIDs: Set = [] + private var activityIDs: Set = [] + private var tail: (id: String, call: String?, turn: String?, log: NativeWorkLogAccumulator)? + + init() {} + + init(messages: [FeatureMessage], activities: [OrchestrationActivity], sessionIsLive: Bool) { + self.init() + // Messages have no event sequence. Exact cross-kind timestamp ties are + // ambiguous in snapshots: keep message wire order, then activity sequence. + // Live batches instead append in received event order, including ties. + let sortedActivities = activities.enumerated().map { + (index: $0.offset, date: NativeTimestampParser.parse($0.element.createdAt) ?? .distantPast, + activity: $0.element) + }.sorted { lhs, rhs in + if lhs.date != rhs.date { return lhs.date < rhs.date } + let a = lhs.activity.sequence ?? Int.max + let b = rhs.activity.sequence ?? Int.max + return a != b ? a < b : lhs.index < rhs.index + } + var events: [(date: Date, message: FeatureMessage?, activity: OrchestrationActivity?)] = + messages.map { ($0.createdAt, $0, nil) } + events += sortedActivities.map { + ($0.date, nil, $0.activity) + } + for event in events.enumerated().sorted(by: { + $0.element.date != $1.element.date + ? $0.element.date < $1.element.date : $0.offset < $1.offset + }) { + if let message = event.element.message { append(message) } + if let activity = event.element.activity { append(activity) } + } + if !sessionIsLive { finishActiveWork() } + } + + mutating func append(_ message: FeatureMessage) { + closeTail() + upsert(message) + } + + mutating func append(_ activity: OrchestrationActivity) { + guard activityIDs.insert(activity.id).inserted else { return } + let date = NativeTimestampParser.parse(activity.createdAt) ?? .distantPast + if let notice = NativeActivityNotice.message(activity, createdAt: date) { + append(notice) + return + } + let answers = NativeQuestionAnswerHistory.messages(activity, createdAt: date) + if !answers.isEmpty { + for answer in answers { append(answer) } + return + } + guard NativeWorkLogAccumulator.accepts(activity) else { return } + let call = activity.payload["toolCallId"]?.stringValue + ?? activity.payload["data"]?["toolCallId"]?.stringValue + let canCoalesce = call?.isEmpty == false && tail?.call == call + && tail?.turn == activity.turnId && messages.last?.id == tail?.id + && date >= (messages.last?.createdAt ?? .distantPast) + if !canCoalesce { + closeTail() + tail = ("work-log-\(activity.id)", call, activity.turnId, NativeWorkLogAccumulator()) + } + guard var current = tail else { return } + let detail = activity.payload["detail"]?.stringValue + let compact = detail?.split(whereSeparator: \.isWhitespace).joined(separator: " ") + let preview = compact.flatMap { $0.isEmpty ? nil : String($0.prefix(160)) } + current.log.append(activity, preview: preview, createdAt: date) + tail = current + if current.log.hasContent { upsert(current.log.message(groupID: String(current.id.dropFirst(9)))) } + } + + mutating func finishActiveWork() { closeTail() } + + mutating func rebuildIndexes() { + indexByID = messages.enumerated().reduce(into: [:]) { $0[$1.element.id] = $1.offset } + } + + private mutating func closeTail() { + guard var current = tail else { return } + current.log.freeze() + if current.log.hasContent { + upsert(current.log.message(groupID: String(current.id.dropFirst(9)))) + } + tail = nil + } + + private mutating func upsert(_ message: FeatureMessage) { + changedIDs.insert(message.id) + if let index = indexByID[message.id] { + messages[index] = message + } else if let last = messages.last, last.createdAt > message.createdAt { + messages.append(message) + messages = messages.enumerated().sorted { + $0.element.createdAt != $1.element.createdAt + ? $0.element.createdAt < $1.element.createdAt : $0.offset < $1.offset + }.map(\.element) + rebuildIndexes() + } else { + indexByID[message.id] = messages.count + messages.append(message) + } + } +} + +struct NativeWorkLogAccumulator { + private static let terminalKinds = Set([ + "tool.completed", "task.completed", + ]) + private static let activeKinds = Set(["tool.started", "tool.updated"]) + private static let imageExtensions = Set([ + "avif", "bmp", "gif", "heic", "heif", "jpeg", "jpg", "png", "tif", "tiff", "webp", + ]) + + private(set) var count = 0 + private var visibleLines: [String] = [] + private var createdAt = Date.distantPast + private var title = "Tool activity" + private var activeEntries: [String: String] = [:] + private var activeOrder: [String] = [] + private var imagePaths: [String] = [] + private var toolPresentation: ToolActivityPresentation? + private var activePresentations: [String: ToolActivityPresentation] = [:] + + var hasActiveWork: Bool { !activeEntries.isEmpty } + var hasContent: Bool { count > 0 || hasActiveWork || !visibleLines.isEmpty || !imagePaths.isEmpty } + + static func accepts(_ activity: OrchestrationActivity) -> Bool { + guard activeKinds.contains(activity.kind) + || (activity.tone != "error" && terminalKinds.contains(activity.kind)) else { + return false + } + if NativeActivityFilters.isPlanBoundaryTool(activity) { return false } + if NativeActivityFilters.isAgentInternal(activity) { return false } + return true + } + + mutating func append( + _ activity: OrchestrationActivity, + preview: String?, + createdAt: Date + ) { + self.createdAt = createdAt + let key = Self.lifecycleKey(activity) + toolPresentation = ToolActivityPresentation(payload: activity.payload) ?? activePresentations[key] + let label = activity.payload["title"]?.stringValue ?? activity.summary + title = label + let lifecycleStatus = activity.payload["status"]?.stringValue + let isTerminalUpdate = activity.kind == "tool.updated" + && lifecycleStatus.map { $0 != "inProgress" && $0 != "in_progress" } == true + if Self.activeKinds.contains(activity.kind) && !isTerminalUpdate + && activity.tone != "error" { + activeEntries[key] = label + activePresentations[key] = toolPresentation + activeOrder.removeAll { $0 == key } + activeOrder.append(key) + } else { + activeEntries[key] = nil + activePresentations[key] = nil + activeOrder.removeAll { $0 == key } + guard activity.tone != "error" else { return } + count += 1 + visibleLines.append("• \(preview ?? activity.summary)") + if visibleLines.count > 40 { + visibleLines.removeFirst(visibleLines.count - 40) + } + } + if let path = Self.viewedImagePath(activity), !imagePaths.contains(path) { + imagePaths.append(path) + if imagePaths.count > 8 { imagePaths.removeFirst(imagePaths.count - 8) } + } + } + + /// A closed row records its last observation, rather than continuing to look live. + mutating func freeze() { + for key in activeOrder { + if let label = activeEntries[key] { visibleLines.append("• \(label)") } + } + clearActiveWork() + } + + mutating func clearActiveWork() { + activeEntries.removeAll(keepingCapacity: true) + activePresentations.removeAll(keepingCapacity: true) + activeOrder.removeAll(keepingCapacity: true) + } + + func message(groupID: String) -> FeatureMessage { + var lines: [String] = [] + if count > visibleLines.count { + lines.append("\(count - visibleLines.count) earlier updates hidden") + } + lines.append(contentsOf: visibleLines) + var message = FeatureMessage( + id: "work-log-\(groupID)", + role: .tool, + text: lines.joined(separator: "\n"), + createdAt: createdAt, + state: .complete, + toolName: title, + workLogImagePaths: imagePaths.isEmpty ? nil : imagePaths, + activeWorkLabel: activeOrder.last.flatMap { activeEntries[$0] } + ) + message.toolPresentation = activeOrder.last.flatMap { activePresentations[$0] } ?? toolPresentation + return message + } + + private static func lifecycleKey(_ activity: OrchestrationActivity) -> String { + if let id = activity.payload["toolCallId"]?.stringValue + ?? activity.payload["data"]?["toolCallId"]?.stringValue { + return "id:\(id)" + } + let itemType = activity.payload["itemType"]?.stringValue ?? "" + let title = activity.payload["title"]?.stringValue ?? activity.summary + let detail = activity.payload["detail"]?.stringValue ?? "" + return "fallback:\([itemType, title, detail].map(normalizedLifecycleText).joined(separator: "|"))" + } + + private static func normalizedLifecycleText(_ value: String) -> String { + value.trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + .replacingOccurrences( + of: #"\s+(complete|completed)$"#, + with: "", + options: .regularExpression + ) + } + + private static func viewedImagePath(_ activity: OrchestrationActivity) -> String? { + let itemType = normalizedLifecycleText(activity.payload["itemType"]?.stringValue ?? "") + let title = normalizedLifecycleText(activity.payload["title"]?.stringValue ?? activity.summary) + let qualifies = activity.payload["requestKind"]?.stringValue == "file-read" + || itemType == "image_view" + || (itemType == "dynamic_tool_call" && title == "read file") + guard qualifies, + let detail = activity.payload["detail"]?.stringValue, + !detail.contains("\n"), !detail.contains("\r") else { return nil } + let path = detail.trimmingCharacters(in: .whitespacesAndNewlines) + guard let ext = path.split(separator: ".").last?.lowercased(), + imageExtensions.contains(String(ext)) else { return nil } + return path + } +} diff --git a/apps/swift-ios/Core/T3Client.swift b/apps/swift-ios/Core/T3Client.swift index 0c8659ac4e58..902c5fa081e5 100644 --- a/apps/swift-ios/Core/T3Client.swift +++ b/apps/swift-ios/Core/T3Client.swift @@ -638,6 +638,29 @@ public actor T3Client { ) } + /// Passive shells start with a full snapshot on each socket subscription. + /// Registering its identity atomically prevents late socket results from + /// acquiring authority over a replacement connection. + public func shellEventsOnCurrentConnection() async throws -> ( + events: AsyncThrowingStream, connectionID: UUID + ) { + try await rpc.subscribeOnCurrentConnection( + RPCMethod.subscribeShell.rawValue, + payload: .object(["requestCompletionMarker": .bool(true)]), + as: ShellStreamItem.self + ) + } + + public func shellEventBatchesOnCurrentConnection() async throws -> ( + events: AsyncThrowingStream<[ShellStreamItem], Error>, connectionID: UUID + ) { + try await rpc.subscribeBatchesOnCurrentConnection( + RPCMethod.subscribeShell.rawValue, + payload: .object(["requestCompletionMarker": .bool(true)]), + as: ShellStreamItem.self + ) + } + public func threadEventBatches( threadID: String, after sequence: Int? = nil, diff --git a/apps/swift-ios/Features/Chat/FeatureComposerView.swift b/apps/swift-ios/Features/Chat/FeatureComposerView.swift index ba17e50d2652..4eb804683e56 100644 --- a/apps/swift-ios/Features/Chat/FeatureComposerView.swift +++ b/apps/swift-ios/Features/Chat/FeatureComposerView.swift @@ -70,6 +70,8 @@ struct FeatureComposerView: View { private let materializesDefaultSelection: Bool private let isSending: Bool private let isWorking: Bool + private let stopPhase: FeatureStopPhase? + private let onRetryStop: (() -> Void)? @Binding private var focused: Bool private let contextUsage: Double? private let forceExpanded: Bool @@ -100,6 +102,8 @@ struct FeatureComposerView: View { materializesDefaultSelection: Bool = true, isSending: Bool, isWorking: Bool, + stopPhase: FeatureStopPhase? = nil, + onRetryStop: (() -> Void)? = nil, focused: Binding, onSend: @escaping () -> Void, onStop: @escaping () -> Void, @@ -141,6 +145,8 @@ struct FeatureComposerView: View { self.materializesDefaultSelection = materializesDefaultSelection self.isSending = isSending self.isWorking = isWorking + self.stopPhase = stopPhase + self.onRetryStop = onRetryStop _focused = focused self.onSend = onSend self.onStop = onStop @@ -288,12 +294,28 @@ struct FeatureComposerView: View { private var composerSurfaceContent: some View { VStack(spacing: 0) { + if let stopPhase { + HStack { + Text(stopIsUnconfirmed + ? "Stop unconfirmed. Waiting for the thread’s status." + : "Stopping… Waiting for the agent to finish.") + if stopPhase == .unconfirmed, let onRetryStop { + Button("Retry stop", action: onRetryStop) + .disabled(!environmentIsConnected) + } + } + .font(T3Typography.supporting) + .foregroundStyle(T3Colors.textSecondary) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .accessibilityIdentifier("thread-stop-status") + } if let approval = pendingApprovals.first, let onApprovalDecision { FeatureComposerApprovalPanel( approval: approval, position: 1, total: pendingApprovals.count, - isResponding: resolvingRequestIDs.contains(approval.id), + isResponding: resolvingRequestIDs.contains(approval.id) || stopPhase != nil, onDecision: { decision in onApprovalDecision(approval.id, decision) }, @@ -310,6 +332,8 @@ struct FeatureComposerView: View { environmentID: environmentID, attachmentPreferences: attachmentPreferences ) + .disabled(stopPhase != nil) + .opacity(stopPhase != nil ? 0.56 : 1) } else if isExpanded { expandedComposer } else { @@ -724,12 +748,18 @@ struct FeatureComposerView: View { } private var submitSymbol: String { - if isSending { return "ellipsis" } + if isSending || (showsStop && stopPhase != nil) { return "ellipsis" } return showsStop ? "stop.fill" : "arrow.up" } + private var stopIsUnconfirmed: Bool { + stopPhase != nil && (stopPhase == .unconfirmed || !environmentIsConnected) + } + private var submitAccessibilityLabel: String { if isSending { return "Sending message" } + if showsStop, stopIsUnconfirmed { return "Stop outcome unconfirmed" } + if showsStop, stopPhase != nil { return "Stopping agent" } if showsStop { return "Stop agent" } return isWorking ? "Queue message" : "Send message" } @@ -754,7 +784,7 @@ struct FeatureComposerView: View { } private var submitDisabled: Bool { - isSending || (!showsStop && !canSend) + isSending || (showsStop && stopPhase != nil) || (!showsStop && !canSend) } private var textIsEmpty: Bool { diff --git a/apps/swift-ios/Features/Chat/FeatureMessageAgeView.swift b/apps/swift-ios/Features/Chat/FeatureMessageAgeView.swift new file mode 100644 index 000000000000..9d611363260d --- /dev/null +++ b/apps/swift-ios/Features/Chat/FeatureMessageAgeView.swift @@ -0,0 +1,51 @@ +import SwiftUI + +struct FeatureMessageAge { + let sentAt: Date + + static func lastMessageDate(in messages: [FeatureMessage]) -> Date? { + messages.last { + ($0.role == .user || $0.role == .assistant) && $0.state != .queued + }?.createdAt + } + + func text(at now: Date) -> String { + let seconds = max(0, Int(now.timeIntervalSince(sentAt))) + if seconds < 60 { return "\(seconds)s" } + if seconds < 3_600 { return "\(seconds / 60)m" } + if seconds < 86_400 { return "\(seconds / 3_600)h" } + return "\(seconds / 86_400)d" + } +} + +/// Message age belongs to the activity row, not connection or snapshot status. +struct FeatureMessageAgeView: View { + let sentAt: Date + + var body: some View { + TimelineView(MessageAgeSchedule(sentAt: sentAt)) { context in + Text("Last message · \(FeatureMessageAge(sentAt: sentAt).text(at: context.date)) ago") + .font(T3Typography.supporting) + .foregroundStyle(T3Colors.textTertiary) + } + .accessibilityIdentifier("thread-last-message-age") + } +} + +/// Wake only at the next displayed second, minute, hour, or day boundary. +private struct MessageAgeSchedule: TimelineSchedule { + let sentAt: Date + + func entries(from startDate: Date, mode: TimelineScheduleMode) -> AnySequence { + AnySequence { + var next = startDate + return AnyIterator { + let date = next + let age = max(0, date.timeIntervalSince(sentAt)) + let unit: TimeInterval = age < 60 ? 1 : age < 3_600 ? 60 : age < 86_400 ? 3_600 : 86_400 + next = sentAt.addingTimeInterval((floor(age / unit) + 1) * unit) + return date + } + } + } +} diff --git a/apps/swift-ios/Features/Chat/ThreadDetailView.swift b/apps/swift-ios/Features/Chat/ThreadDetailView.swift index 6483b15c0ef4..fe7ab74f710c 100644 --- a/apps/swift-ios/Features/Chat/ThreadDetailView.swift +++ b/apps/swift-ios/Features/Chat/ThreadDetailView.swift @@ -14,6 +14,7 @@ public struct ThreadDetailView: View { let submitMessage: (FeatureMessageSubmission) async -> Bool let onNavigateBack: () -> Void private let draftStore: FeatureComposerDraftStore + private let managesThreadPresentation: Bool @State private var draft = "" @State private var composerContext: OrchestrationMessageContext? @@ -52,20 +53,22 @@ public struct ThreadDetailView: View { thread: FeatureThread, submitMessage: @escaping (FeatureMessageSubmission) async -> Bool, onNavigateBack: @escaping () -> Void = {}, - draftStore: FeatureComposerDraftStore = .shared + draftStore: FeatureComposerDraftStore = .shared, + managesThreadPresentation: Bool = true ) { self.model = model self.thread = thread self.submitMessage = submitMessage self.onNavigateBack = onNavigateBack self.draftStore = draftStore + self.managesThreadPresentation = managesThreadPresentation } private var threadContent: some View { Group { if let detail { timeline(detail) - } else if isLoading { + } else if isOpening { FeatureThreadOpeningView() } else { ContentUnavailableView { @@ -102,16 +105,27 @@ public struct ThreadDetailView: View { } } .task(id: thread.id) { + guard managesThreadPresentation else { return } isLoading = true - _ = await model.detail(for: thread.id, force: true) - isLoading = false + await model.runThreadPresentation(id: thread.id) { + isLoading = false + } } .task(id: thread.id) { // A cached thread can already show its composer while the server // is catching up. Local drafts must not wait for that request. - await model.checkRewindRecovery(for: currentThread) - guard !didRestoreDraft else { return } - await restoreDraft(from: composerDraft, key: draftKey) + // When a thread is reopened in the compact split view, SwiftUI can + // report this view as disappeared right after it appears while it + // stays on screen. That cancels this task and it never re-runs, so + // the composer would stay busy. Finish the restore outside the + // view's task lifetime; each thread has its own view identity, so + // a late restore cannot reach another thread's composer. + let restore = Task { @MainActor in + await model.checkRewindRecovery(for: currentThread) + guard !didRestoreDraft else { return } + await restoreDraft(from: composerDraft, key: draftKey) + } + await restore.value } .task(id: pullRequestObservationID) { await observeThreadPullRequest() @@ -133,7 +147,7 @@ public struct ThreadDetailView: View { .onChange(of: threadConnectionState) { _, state in if state == .connected, case .failed = model.detailLoadStates[thread.id], - !isLoading { + !isOpening { reloadThread() } } @@ -143,7 +157,6 @@ public struct ThreadDetailView: View { } } .onDisappear { - model.releaseThread(thread.id) persistDraftBeforeLeaving() } .sheet(item: $toolSurface) { surface in @@ -277,6 +290,12 @@ public struct ThreadDetailView: View { } } + private var isOpening: Bool { + if managesThreadPresentation { return isLoading } + if case .failed = model.detailLoadStates[thread.id] { return false } + return detail == nil || model.detailLoadStates[thread.id] == .loading + } + private var detail: FeatureThreadDetail? { model.details[thread.id] } @@ -683,10 +702,8 @@ public struct ThreadDetailView: View { } private func reloadThread() { - isLoading = true - Task { - _ = await model.detail(for: thread.id, force: true, fresh: true) - isLoading = false + if model.refreshThreadPresentation(id: thread.id, onLoaded: { isLoading = false }) { + isLoading = true } } @@ -699,7 +716,7 @@ public struct ThreadDetailView: View { ThreadRefreshPresentation.resolve( loadState: model.detailLoadStates[thread.id], connectionState: threadConnectionState, - isOpening: isLoading, + isOpening: isOpening, syncState: model.threadSyncStates[thread.id] ) } @@ -850,10 +867,12 @@ public struct ThreadDetailView: View { isSending: isSending || isRewinding || !didRestoreDraft, isWorking: detail.thread.state == .working || detail.thread.state == .queued || isCompacting, + stopPhase: model.stopPhase(threadID: thread.id), + onRetryStop: { model.retryCancelTurn(threadID: thread.id) }, focused: $composerFocused, onSend: send, onStop: { - Task { await model.cancelTurn(threadID: thread.id) } + model.requestCancelTurn(threadID: thread.id) }, pendingApprovals: detail.approvals, pendingUserInputs: detail.userInputs, @@ -957,12 +976,10 @@ public struct ThreadDetailView: View { private func timelineMessages(_ messages: [FeatureMessage]) -> [FeatureMessage] { guard !feedbackMessages.isEmpty else { return messages } - return (messages + feedbackMessages).sorted { - if $0.createdAt == $1.createdAt { - return $0.id < $1.id - } - return $0.createdAt < $1.createdAt - } + return (messages + feedbackMessages).enumerated().sorted { + $0.element.createdAt != $1.element.createdAt + ? $0.element.createdAt < $1.element.createdAt : $0.offset < $1.offset + }.map(\.element) } private var markdownImageContext: MarkdownImageContext? { @@ -1777,6 +1794,7 @@ private struct FeatureTranscriptCollectionView: UIViewRepresentable { private var currentDetailRevision: UInt64? private var currentDynamicTypeSize: DynamicTypeSize? private var currentCodeSizeSteps = 0 + private var currentLastMessageAt: Date? private var currentIsWorking = false private var currentIsCompacting = false private var currentActiveSubagentCount = 0 @@ -1828,6 +1846,7 @@ private struct FeatureTranscriptCollectionView: UIViewRepresentable { if messageID == FeatureTranscriptCollectionView.workingIndicatorID { cell.contentConfiguration = UIHostingConfiguration { FeatureThreadWorkingIndicator( + lastMessageAt: self?.currentLastMessageAt, isCompacting: self?.currentIsCompacting == true, activeSubagentCount: self?.currentActiveSubagentCount ?? 0, backgroundWorkIsActive: self?.currentBackgroundWorkIsActive == true, @@ -1908,7 +1927,9 @@ private struct FeatureTranscriptCollectionView: UIViewRepresentable { || currentCodeSizeSteps != codeSizeSteps let revisionChanged = currentDetailRevision != renderUpdate?.revision let workingChanged = currentIsWorking != isWorking - let workingDetailChanged = currentIsCompacting != isCompacting + let lastMessageAt = FeatureMessageAge.lastMessageDate(in: messages) + let workingDetailChanged = currentLastMessageAt != lastMessageAt + || currentIsCompacting != isCompacting || currentActiveSubagentCount != activeSubagentCount || currentBackgroundWorkIsActive != backgroundWorkIsActive || currentIsMonitoring != isMonitoring @@ -1934,6 +1955,7 @@ private struct FeatureTranscriptCollectionView: UIViewRepresentable { currentDetailRevision = renderUpdate?.revision currentDynamicTypeSize = dynamicTypeSize currentCodeSizeSteps = codeSizeSteps + currentLastMessageAt = lastMessageAt currentIsWorking = isWorking currentIsCompacting = isCompacting currentActiveSubagentCount = activeSubagentCount @@ -2312,6 +2334,7 @@ private struct FeatureLoadEarlierTurnsButton: View { } private struct FeatureThreadWorkingIndicator: View { + let lastMessageAt: Date? let isCompacting: Bool let activeSubagentCount: Int let backgroundWorkIsActive: Bool @@ -2348,7 +2371,9 @@ private struct FeatureThreadWorkingIndicator: View { Text(title) .font(T3Typography.supportingStrong) .foregroundStyle(T3Colors.statusRunning) - if let detail { + if let lastMessageAt { + FeatureMessageAgeView(sentAt: lastMessageAt) + } else if let detail { Text(detail) .font(T3Typography.supporting) .foregroundStyle(T3Colors.textTertiary) @@ -2358,7 +2383,7 @@ private struct FeatureThreadWorkingIndicator: View { .frame(maxWidth: .infinity, alignment: .leading) .padding(.vertical, 4) .accessibilityElement(children: .combine) - .accessibilityLabel(detail.map { "\(title). \($0)." } ?? "\(title).") + } } @@ -3078,28 +3103,36 @@ struct FeatureMessageView: View { HStack(alignment: .top, spacing: 8) { Image(systemName: "exclamationmark.triangle") .foregroundStyle(T3Colors.warning) - VStack(alignment: .leading, spacing: 5) { - Text(message.text) - .foregroundStyle(T3Colors.textPrimary) - .textSelection(.enabled) - Text(message.createdAt, format: .dateTime.month(.abbreviated).day().hour().minute()) - .foregroundStyle(T3Colors.textSecondary) - } + Text(message.text) + .foregroundStyle(T3Colors.textPrimary) + .textSelection(.enabled) + Spacer(minLength: 8) + FeatureActivityTimestamp(date: message.createdAt) } .font(T3Typography.supporting) .frame(maxWidth: .infinity, alignment: .leading) .accessibilityElement(children: .combine) } else if message.toolName == "context-compaction" { - Label(message.text, systemImage: "arrow.down.right.and.arrow.up.left") + HStack(alignment: .top, spacing: 8) { + Label(message.text, systemImage: "arrow.down.right.and.arrow.up.left") + Spacer(minLength: 8) + FeatureActivityTimestamp(date: message.createdAt) + } .font(T3Typography.supporting) .foregroundStyle(T3Colors.textSecondary) .frame(maxWidth: .infinity, alignment: .center) .padding(.vertical, 4) + .accessibilityElement(children: .combine) } else { - Text(message.text) + HStack(alignment: .top, spacing: 8) { + Text(message.text) + Spacer(minLength: 8) + FeatureActivityTimestamp(date: message.createdAt) + } .font(T3Typography.supporting) .foregroundStyle(T3Colors.textSecondary) .frame(maxWidth: .infinity, alignment: .center) + .accessibilityElement(children: .combine) } } @@ -3114,6 +3147,19 @@ struct FeatureMessageView: View { } } +private struct FeatureActivityTimestamp: View { + let date: Date + + var body: some View { + Text(date, format: Date.FormatStyle(date: .numeric, time: .standard)) + .font(T3Typography.supporting.monospacedDigit()) + .foregroundStyle(T3Colors.textSecondary) + .multilineTextAlignment(.trailing) + .fixedSize(horizontal: false, vertical: true) + .accessibilityLabel(date.formatted(date: .complete, time: .standard)) + } +} + private struct FeatureWorkLogView: View { let message: FeatureMessage let imageContext: MarkdownImageContext? @@ -3144,6 +3190,7 @@ private struct FeatureWorkLogView: View { Spacer(minLength: 8) Image(systemName: isExpanded ? "chevron.down" : "chevron.right") .font(.caption.weight(.semibold)) + FeatureActivityTimestamp(date: message.createdAt) } .font(T3Typography.tool.weight(.medium)) .foregroundStyle(T3Colors.textSecondary) diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index cb91af596570..a8fc101e2660 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -23,11 +23,108 @@ enum FeatureThreadLoadState: Equatable { case failed(String) } +enum FeatureStopPhase: Equatable { + case requesting + case awaitingOutcome + case unconfirmed +} + @MainActor @Observable public final class FeatureRootModel { private static let maximumRetainedThreadDetails = 6 + private struct StopKey: Hashable { + let threadID: String + let turnID: String? + } + private struct StopRequest { + let token: UUID + let key: StopKey + var canObserveOutcome: Bool + var phase: FeatureStopPhase + } + private var stopRequests: [String: StopRequest] = [:] + + func stopPhase(threadID: String) -> FeatureStopPhase? { + let key = stopKey(threadID: threadID) + guard let request = stopRequests[threadID], request.key == key else { return nil } + return request.phase + } + + private func stopKey(threadID: String) -> StopKey { + let thread = snapshot.threads.first { $0.id == threadID } ?? details[threadID]?.thread + return thread.map { StopKey(threadID: threadID, turnID: $0.latestTurnID) } + ?? stopRequests[threadID]?.key ?? StopKey(threadID: threadID, turnID: nil) + } + + /// Claims the request on the tap's actor turn, before dispatching async work. + func requestCancelTurn(threadID: String) { + let key = stopKey(threadID: threadID) + guard let token = claimStop(key) else { return } + Task { await executeStop(key, token: token) } + } + + private func claimStop(_ key: StopKey) -> UUID? { + guard stopRequests[key.threadID]?.key != key else { return nil } + let token = UUID() + let thread = snapshot.threads.first { $0.id == key.threadID } ?? details[key.threadID]?.thread + let canObserveOutcome = key.turnID != nil && thread?.latestTurnState == "running" + stopRequests[key.threadID] = StopRequest( + token: token, key: key, canObserveOutcome: canObserveOutcome, phase: .requesting + ) + return token + } + + /// A failed transport response may have followed acceptance. Reconcile before retrying. + @discardableResult + func retryCancelTurn(threadID: String) -> Task? { + let key = stopKey(threadID: threadID) + guard let previous = stopRequests[key.threadID], previous.key == key, + previous.phase == .unconfirmed else { return nil } + let token = UUID() + // Retry must retain the running-turn evidence captured by this same request. + let canObserveOutcome = previous.canObserveOutcome + stopRequests[key.threadID] = StopRequest( + token: token, key: key, canObserveOutcome: canObserveOutcome, phase: .requesting + ) + return Task { + do { + let status = try await client.stopStatus(threadID: threadID) + guard stopRequests[key.threadID]?.token == token else { return } + upsert(status) + guard stopRequests[key.threadID]?.token == token else { return } + guard status.latestTurnID == key.turnID else { + stopRequests[key.threadID]?.phase = .unconfirmed + return + } + guard status.latestTurnID != nil, status.latestTurnState == "running" else { + stopRequests[key.threadID]?.phase = .unconfirmed + return + } + stopRequests[key.threadID]?.canObserveOutcome = true + await executeStop(key, token: token) + } catch { + guard stopRequests[key.threadID]?.token == token else { return } + stopRequests[key.threadID]?.phase = .unconfirmed + errorMessage = error.localizedDescription + } + } + } + + private func reconcileStop(_ thread: FeatureThread) { + guard let turnID = thread.latestTurnID, + thread.latestTurnState == "completed" || thread.latestTurnState == "interrupted" + || thread.latestTurnState == "error" else { return } + // Approval and input presentation can hide a session that is still running. + guard let sessionStatus = thread.settlementFacts?.sessionStatus, + ["ready", "interrupted", "stopped", "error"].contains(sessionStatus), + thread.state != .working && thread.state != .queued, + let request = stopRequests[thread.id], request.canObserveOutcome, + request.key.turnID == turnID else { return } + stopRequests.removeValue(forKey: thread.id) + } + private struct PendingSettlementMutation { let id: UUID let settled: Bool @@ -54,6 +151,7 @@ public final class FeatureRootModel { public private(set) var details: [String: FeatureThreadDetail] = [:] private(set) var detailLoadStates: [String: FeatureThreadLoadState] = [:] private(set) var threadSyncStates: [String: FeatureThreadSyncState] = [:] + private(set) var selectedThreadReceipt: FeatureThreadReceipt? private var backgroundedAt: Date? /// Advances when a Home shelf or order input changes (see `HomeOrderKey`) /// or when projects, environments, providers, or preferences change. @@ -90,11 +188,19 @@ public final class FeatureRootModel { draftStore: draftStore ) private var pendingSubmissionsByID: [String: FeatureQueuedSubmission] = [:] + /// Accepted messages not yet present in a server transcript, by message ID. + private var deliveredAwaitingDetail: [String: (threadID: String, message: FeatureMessage)] = [:] + /// Message IDs in each thread's latest server transcript. + private var serverMessageIDs: [String: Set] = [:] private var activeSubmissionCounts: [String: Int] = [:] private var pendingThreadsByID: [String: FeatureThread] = [:] private var pendingSettlementMutations: [String: PendingSettlementMutation] = [:] private var pendingCompletionSubmissionIDs: Set = [] private var pendingDiscardSubmissionIDs: Set = [] + @ObservationIgnored private var activeDetailPresentation: ( + threadID: String, owner: UUID, lifetime: AsyncStream.Continuation, + refreshTask: Task? + )? private var detailRecency: [String] = [] private var detailLoadGeneration: UInt64 = 0 private var detailLoadRevisions: [String: UInt64] = [:] @@ -149,6 +255,7 @@ public final class FeatureRootModel { func applicationDidEnterBackground(at date: Date = .now) { backgroundedAt = date + client.suspendForBackground() } func applicationDidBecomeActive(at date: Date = .now) async { @@ -721,6 +828,56 @@ public final class FeatureRootModel { } } + /// Holds selected-thread transport for the owning presentation task, including after loading. + func runThreadPresentation(id: String, onLoaded: @MainActor () -> Void = {}) async { + guard !Task.isCancelled else { return } + if let previous = activeDetailPresentation { + releaseThread(previous.threadID) + } + selectedThreadReceipt = nil + let owner = UUID() + let lifetime = AsyncStream.makeStream() + activeDetailPresentation = (id, owner, lifetime.continuation, nil) + await withTaskCancellationHandler { + defer { releaseThreadPresentation(id: id, owner: owner) } + _ = await detail(for: id, force: true) + guard !Task.isCancelled, + activeDetailPresentation?.owner == owner else { return } + onLoaded() + for await _ in lifetime.stream {} + } onCancel: { + // Cancellation must release even when the detail read is still suspended. + Task { @MainActor [weak self] in + self?.releaseThreadPresentation(id: id, owner: owner) + } + } + } + + /// Retries belong to the current presentation and cannot outlive a close or replacement. + @discardableResult + func refreshThreadPresentation( + id: String, fresh: Bool = true, onLoaded: @escaping @MainActor () -> Void = {} + ) -> Bool { + guard let presentation = activeDetailPresentation, presentation.threadID == id else { return false } + presentation.refreshTask?.cancel() + let owner = presentation.owner + activeDetailPresentation?.refreshTask = Task { [weak self] in + guard let self, !Task.isCancelled, + self.activeDetailPresentation?.owner == owner else { return } + _ = await self.detail(for: id, force: true, fresh: fresh) + guard !Task.isCancelled, + self.activeDetailPresentation?.owner == owner else { return } + onLoaded() + } + return true + } + + private func releaseThreadPresentation(id: String, owner: UUID) { + guard activeDetailPresentation?.threadID == id, + activeDetailPresentation?.owner == owner else { return } + releaseThread(id) + } + public func detail(for id: String, force: Bool = false, fresh: Bool = false) async -> FeatureThreadDetail? { if !force, let cached = details[id] { return cached @@ -799,6 +956,12 @@ public final class FeatureRootModel { /// Ends any selected-thread transport work when its detail view closes. public func releaseThread(_ id: String) { + if selectedThreadReceipt?.threadID == id { selectedThreadReceipt = nil } + if let presentation = activeDetailPresentation, presentation.threadID == id { + activeDetailPresentation = nil + presentation.refreshTask?.cancel() + presentation.lifetime.finish() + } client.releaseThread(id: id) markDetailRecentlyUsed(id) evictOldThreadDetailsIfNeeded() @@ -1007,28 +1170,59 @@ public final class FeatureRootModel { } public func cancelTurn(threadID: String) async { + let key = stopKey(threadID: threadID) + guard let token = claimStop(key) else { return } + await executeStop(key, token: token) + } + + private func executeStop(_ key: StopKey, token: UUID) async { + guard stopRequests[key.threadID]?.token == token else { return } + do { + let sentRemotely = try await cancelClaimedTurn(key) + guard stopRequests[key.threadID]?.token == token else { return } + if !sentRemotely { + stopRequests.removeValue(forKey: key.threadID) + } else { + let phase: FeatureStopPhase = stopRequests[key.threadID]?.canObserveOutcome == true + ? .awaitingOutcome : .unconfirmed + stopRequests[key.threadID]?.phase = phase + } + } catch { + guard stopRequests[key.threadID]?.token == token else { return } + if error is FeatureStopTurnChangedError { + stopRequests.removeValue(forKey: key.threadID) + } else { + stopRequests[key.threadID]?.phase = .unconfirmed + } + errorMessage = error.localizedDescription + } + } + + private func cancelClaimedTurn(_ key: StopKey) async throws -> Bool { + let threadID = key.threadID if pendingSubmissionsByID.values.contains(where: { $0.threadID == threadID && $0.creation != nil }) { await stopOutboxDrain() + defer { scheduleOutboxDrain() } let queued = pendingSubmissionsByID.values.filter { $0.threadID == threadID } + var discardedAll = true for submission in queued { if !(await discardQueuedSubmission(submission)) { + discardedAll = false scheduleOutboxRetry() } } + guard discardedAll else { throw FeatureStopStatusUnavailableError() } if pendingThreadsByID[threadID] == nil, snapshot.threads.contains(where: { $0.id == threadID }) { - await perform { - try await client.cancelTurn(threadID: threadID) - } + try await client.cancelTurn(threadID: threadID) + return true } - scheduleOutboxDrain() - return - } - await perform { - try await client.cancelTurn(threadID: threadID) + return false } + try await client.cancelTurn(threadID: threadID, expectedTurnID: key.turnID) + return true } public func resolveApproval(_ id: String, decision: FeatureApprovalDecision) async { @@ -1229,6 +1423,8 @@ public final class FeatureRootModel { private func apply(_ event: FeatureEvent) { switch event { + case let .threadReceipt(receipt): + recordThreadReceipt(receipt) case let .snapshot(value): install(value) case let .connection(value, environmentID): @@ -1276,7 +1472,15 @@ public final class FeatureRootModel { } } + func recordThreadReceipt(_ receipt: FeatureThreadReceipt) { + guard activeDetailPresentation?.threadID == receipt.threadID else { return } + guard selectedThreadReceipt.map({ receipt.receivedAt >= $0.receivedAt }) ?? true else { return } + // Receipt-only updates must not invalidate the transcript or Home projections. + selectedThreadReceipt = receipt + } + private func upsert(_ thread: FeatureThread) { + reconcileStop(thread) let thread = retainingPendingSettlement(in: thread) discardStalePullRequest(for: thread) var metadataChanged = false @@ -1317,6 +1521,8 @@ public final class FeatureRootModel { } private func removeThread(id: String) { + forgetDeliveredMessages(threadID: id) + stopRequests = stopRequests.filter { $0.key != id } guard let index = snapshot.threads.firstIndex(where: { $0.id == id }) else { return } let projectID = snapshot.threads[index].projectID snapshot.threads.remove(at: index) @@ -1340,6 +1546,7 @@ public final class FeatureRootModel { pendingRewindRecoveryIDs.remove(thread.id) rewindErrors[thread.id] = nil } + for thread in value.threads { reconcileStop(thread) } if settingsWriteTask != nil { // A shell refresh can still contain the settings from before a // queued write. Keep both the visible choice and its rollback point. @@ -1449,15 +1656,18 @@ public final class FeatureRootModel { } } + /// `isServerTranscript` is false for details built locally, such as a + /// restored outbox or a pending creation; they cannot confirm delivery. private func store( _ incoming: FeatureThreadDetail, - invalidatesInFlightLoad: Bool = true + invalidatesInFlightLoad: Bool = true, + isServerTranscript: Bool = true ) { var incoming = retainingLocalAttachmentPreviews(in: incoming) incoming.thread = retainingPendingSettlement(in: incoming.thread) let id = incoming.thread.id acknowledgeDeliveredMessages(incoming.messages) - let prepared = addingPendingMessages(to: incoming) + let prepared = addingPendingMessages(to: incoming, isServerTranscript: isServerTranscript) let next = details[id].map { current in FeatureThreadDetail( thread: prepared.thread, @@ -1484,7 +1694,7 @@ public final class FeatureRootModel { incoming.thread = retainingPendingSettlement(in: incoming.thread) let id = incoming.thread.id acknowledgeDeliveredMessages(incoming.messages) - let next = addingPendingMessages(to: incoming) + let next = addingPendingMessages(to: incoming, isServerTranscript: true) details[id] = next markDetailRecentlyUsed(id) bumpDetailLoadRevision(id: id) @@ -1523,7 +1733,19 @@ public final class FeatureRootModel { return true } - private func removeDetail(id: String) { + private func forgetDeliveredMessages(threadID: String) { + deliveredAwaitingDetail = deliveredAwaitingDetail.filter { $0.value.threadID != threadID } + serverMessageIDs[threadID] = nil + } + + private func removeDetail(id: String, forgettingDeliveredMessages: Bool = true) { + if forgettingDeliveredMessages { + forgetDeliveredMessages(threadID: id) + } else { + // The next server transcript records these again; keep the cache bounded. + serverMessageIDs[id] = nil + } + if selectedThreadReceipt?.threadID == id { selectedThreadReceipt = nil } if details.removeValue(forKey: id) != nil { detailRecency.removeAll { $0 == id } } @@ -1536,6 +1758,9 @@ public final class FeatureRootModel { } private func clearDetails() { + deliveredAwaitingDetail.removeAll() + serverMessageIDs.removeAll() + selectedThreadReceipt = nil detailLoadGeneration &+= 1 detailLoadRevisions.removeAll() storedDetailLoadRequestRevisions.removeAll() @@ -1571,7 +1796,8 @@ public final class FeatureRootModel { while details.count > Self.maximumRetainedThreadDetails, let candidate = detailRecency.first(where: { !protected.contains($0) }) { detailRecency.removeAll { $0 == candidate } - removeDetail(id: candidate) + // Eviction only drops the cache; accepted messages must survive a reopen. + removeDetail(id: candidate, forgettingDeliveredMessages: false) } } @@ -1614,7 +1840,7 @@ public final class FeatureRootModel { if snapshot.threads.contains(where: { $0.id == submission.threadID }) { pendingSubmissionsByID[submission.id] = submission if let detail = details[submission.threadID] { - store(addingPendingMessages(to: detail)) + store(detail, isServerTranscript: false) } continue } @@ -1645,7 +1871,7 @@ public final class FeatureRootModel { } pendingSubmissionsByID[submission.id] = submission if let detail = details[submission.threadID] { - store(addingPendingMessages(to: detail)) + store(detail, isServerTranscript: false) } } } @@ -1708,7 +1934,7 @@ public final class FeatureRootModel { store(FeatureThreadDetail( thread: thread, messages: [queuedMessage(for: submission)] - )) + ), isServerTranscript: false) } private func provider(id: String?, environmentID: String) -> FeatureProvider? { @@ -1737,16 +1963,30 @@ public final class FeatureRootModel { ) } - private func addingPendingMessages(to incoming: FeatureThreadDetail) -> FeatureThreadDetail { + private func addingPendingMessages( + to incoming: FeatureThreadDetail, + isServerTranscript: Bool + ) -> FeatureThreadDetail { + let existing = Set(incoming.messages.map(\.id)) + // A delivered message stays visible until a server transcript includes it. + // Its refresh no longer blocks the send, so an older read can arrive first. + if isServerTranscript { + serverMessageIDs[incoming.thread.id] = existing + for (id, delivered) in deliveredAwaitingDetail + where delivered.threadID == incoming.thread.id && existing.contains(id) { + deliveredAwaitingDetail.removeValue(forKey: id) + } + } + let delivered = deliveredAwaitingDetail.values + .filter { $0.threadID == incoming.thread.id && !existing.contains($0.message.id) } + .map(\.message) let queued = pendingSubmissionsByID.values - .filter { $0.threadID == incoming.thread.id } - .sorted { $0.identity.createdAt < $1.identity.createdAt } - guard !queued.isEmpty else { return incoming } + .filter { $0.threadID == incoming.thread.id && !existing.contains($0.identity.messageID) } + .map(queuedMessage(for:)) + guard !queued.isEmpty || !delivered.isEmpty else { return incoming } var result = incoming - let existing = Set(result.messages.map(\.id)) - result.messages.append(contentsOf: queued.lazy - .filter { !existing.contains($0.identity.messageID) } - .map(queuedMessage(for:))) + // A newer send can be delivered while an older one waits to retry; keep send order. + result.messages.append(contentsOf: (delivered + queued).sorted { $0.createdAt < $1.createdAt }) return result } @@ -1823,6 +2063,12 @@ public final class FeatureRootModel { } pendingCompletionSubmissionIDs.remove(submission.id) pendingSubmissionsByID.removeValue(forKey: submission.id) + // A transcript that already includes it confirmed delivery; do not retain it. + if serverMessageIDs[submission.threadID]?.contains(submission.identity.messageID) != true { + var message = queuedMessage(for: submission) + message.state = .complete + deliveredAwaitingDetail[submission.identity.messageID] = (submission.threadID, message) + } setAttachmentOutboxOwnership(false, for: submission) pendingThreadsByID.removeValue(forKey: submission.threadID) markQueuedMessageDelivered(submission) diff --git a/apps/swift-ios/Features/Shared/FeatureClient.swift b/apps/swift-ios/Features/Shared/FeatureClient.swift index 5b791cca92cf..8c873ca06dce 100644 --- a/apps/swift-ios/Features/Shared/FeatureClient.swift +++ b/apps/swift-ios/Features/Shared/FeatureClient.swift @@ -10,6 +10,7 @@ public protocol FeatureClient: AnyObject { func backgroundSnapshot() async throws -> FeatureSnapshot func events() -> AsyncStream func resumeAfterBackground(reconnect: Bool) async + func suspendForBackground() func preuploadAttachment( _ attachment: FeatureUploadAttachment, @@ -90,6 +91,8 @@ public protocol FeatureClient: AnyObject { threadID: String, messageID: String, prepareRecovery: @MainActor (FeatureRevertedMessage) async throws -> Void ) async throws + func stopStatus(threadID: String) async throws -> FeatureThread + func cancelTurn(threadID: String, expectedTurnID: String?) async throws func resolveApproval(id: String, decision: FeatureApprovalDecision) async throws func resolveUserInput( id: String, answers: [String: FeatureInputAnswer], @@ -252,6 +255,14 @@ public extension FeatureClient { throw FeatureCapabilityUnavailable("Conversation rewind") } + func stopStatus(threadID: String) async throws -> FeatureThread { + throw FeatureStopStatusUnavailableError() + } + + func cancelTurn(threadID: String, expectedTurnID: String?) async throws { + try await cancelTurn(threadID: threadID) + } + func serverPreferences(environmentID: String) async throws -> ServerSettingsSnapshot { throw FeatureCapabilityUnavailable("Server preferences") } @@ -314,6 +325,8 @@ public extension FeatureClient { func resumeAfterBackground(reconnect: Bool) async {} + func suspendForBackground() {} + func preuploadAttachment( _ attachment: FeatureUploadAttachment, environmentID: String @@ -570,3 +583,13 @@ public extension FeatureClient { throw FeatureCapabilityUnavailable("Terminal") } } + +public struct FeatureStopTurnChangedError: LocalizedError { + public init() {} + public var errorDescription: String? { "The active turn changed. Check the thread before stopping again." } +} + +public struct FeatureStopStatusUnavailableError: LocalizedError { + public init() {} + public var errorDescription: String? { "Could not confirm the thread’s stop status." } +} diff --git a/apps/swift-ios/Features/Shared/FeatureModels.swift b/apps/swift-ios/Features/Shared/FeatureModels.swift index 930ce76dfe65..af7cfcce862f 100644 --- a/apps/swift-ios/Features/Shared/FeatureModels.swift +++ b/apps/swift-ios/Features/Shared/FeatureModels.swift @@ -311,6 +311,8 @@ public struct FeatureThread: Identifiable, Sendable, Equatable, Hashable, Codabl public var isRegeneratingTitle: Bool public var attentionAt: Date? public var workingStartedAt: Date? + public var latestTurnID: String? + public var latestTurnState: String? public var latestTurnCompletedAt: Date? public var settlementFacts: FeatureThreadSettlementFacts? public var runtimeMode: FeatureRuntimeMode @@ -359,6 +361,8 @@ public struct FeatureThread: Identifiable, Sendable, Equatable, Hashable, Codabl isRegeneratingTitle: Bool = false, attentionAt: Date? = nil, workingStartedAt: Date? = nil, + latestTurnID: String? = nil, + latestTurnState: String? = nil, latestTurnCompletedAt: Date? = nil, settlementFacts: FeatureThreadSettlementFacts? = nil, runtimeMode: FeatureRuntimeMode = .fullAccess, @@ -406,6 +410,8 @@ public struct FeatureThread: Identifiable, Sendable, Equatable, Hashable, Codabl self.isRegeneratingTitle = isRegeneratingTitle self.attentionAt = attentionAt self.workingStartedAt = workingStartedAt + self.latestTurnID = latestTurnID + self.latestTurnState = latestTurnState self.latestTurnCompletedAt = latestTurnCompletedAt self.settlementFacts = settlementFacts self.runtimeMode = runtimeMode @@ -1394,6 +1400,7 @@ public enum FeatureThreadSyncState: Sendable, Equatable { } public enum FeatureEvent: Sendable { + case threadReceipt(FeatureThreadReceipt) case snapshot(FeatureSnapshot) /// Active environment connection change. `environmentID` lets the model /// patch that environment's row without installing a whole snapshot. diff --git a/apps/swift-ios/Features/Shared/FeatureThreadReceipt.swift b/apps/swift-ios/Features/Shared/FeatureThreadReceipt.swift new file mode 100644 index 000000000000..b2142a59bbd0 --- /dev/null +++ b/apps/swift-ios/Features/Shared/FeatureThreadReceipt.swift @@ -0,0 +1,27 @@ +import Foundation + +/// Records receipt on this device, separately from a message’s server timestamp. +public struct FeatureThreadReceipt: Sendable, Equatable { + public enum Source: Sendable, Equatable { + case detailEvent + case detailSnapshot + case shellThreadUpdate + } + + public let threadID: String + public let receivedAt: Date + public let source: Source + + public init(threadID: String, receivedAt: Date, source: Source) { + self.threadID = threadID + self.receivedAt = receivedAt + self.source = source + } + + func formattedTimestamp(locale: Locale = .current, timeZone: TimeZone = .current) -> String { + receivedAt.formatted(Date.FormatStyle( + date: .numeric, time: .standard, locale: locale, timeZone: timeZone + )) + } + +} diff --git a/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift b/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift new file mode 100644 index 000000000000..70c524635174 --- /dev/null +++ b/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift @@ -0,0 +1,25 @@ +import Foundation + +struct WorkspaceThreadSelection: Equatable { + private(set) var selectedID: String? + private(set) var lastOpenedID: String? + + var highlightedID: String? { selectedID ?? lastOpenedID } + + mutating func open(_ id: String) { + selectedID = id + lastOpenedID = id + } + + func presentedID(isCompact: Bool, showsDetailColumn: Bool) -> String? { + isCompact && !showsDetailColumn ? nil : selectedID + } + + mutating func reconcilePresentation(isCompact: Bool, showsDetailColumn: Bool) { + if isCompact && !showsDetailColumn { close() } + } + + mutating func close() { + selectedID = nil + } +} diff --git a/apps/swift-ios/Features/Workspace/WorkspaceView.swift b/apps/swift-ios/Features/Workspace/WorkspaceView.swift index 2a0d3a448149..9e13c977dda4 100644 --- a/apps/swift-ios/Features/Workspace/WorkspaceView.swift +++ b/apps/swift-ios/Features/Workspace/WorkspaceView.swift @@ -18,24 +18,9 @@ struct FeatureWorkspaceNavigationRequest: Equatable, Sendable { } } -struct WorkspaceThreadSelection: Equatable { - private(set) var selectedID: String? - private(set) var lastOpenedID: String? - - var highlightedID: String? { selectedID ?? lastOpenedID } - - mutating func open(_ id: String) { - selectedID = id - lastOpenedID = id - } - - mutating func close() { - selectedID = nil - } -} - public struct WorkspaceView: View { @SwiftUI.Environment(\.dynamicTypeSize) private var dynamicTypeSize + @SwiftUI.Environment(\.horizontalSizeClass) private var horizontalSizeClass @Bindable var model: FeatureRootModel private let navigationRequest: FeatureWorkspaceNavigationRequest? @@ -81,8 +66,8 @@ public struct WorkspaceView: View { self.submitMessage = { submission in await model.sendMessage(submission) } } - public var body: some View { - NavigationSplitView(preferredCompactColumn: $preferredCompactColumn) { + private var workspaceNavigation: some View { + NavigationSplitView(preferredCompactColumn: compactColumnBinding) { sidebar .navigationSplitViewColumnWidth( min: T3Metrics.minimumSidebarWidth, @@ -93,6 +78,17 @@ public struct WorkspaceView: View { detail } .navigationSplitViewStyle(.balanced) + .task(id: presentedThreadID) { [id = presentedThreadID] in + guard let id else { return } + await model.runThreadPresentation(id: id) + } + .onChange(of: horizontalSizeClass) { _, _ in + reconcileThreadPresentation() + } + } + + public var body: some View { + workspaceNavigation .sheet(isPresented: $showingNewTask) { NewThreadView( model: model, @@ -297,7 +293,8 @@ public struct WorkspaceView: View { model: model, thread: thread, submitMessage: submitMessage, - onNavigateBack: closeSelectedThread + onNavigateBack: closeSelectedThread, + managesThreadPresentation: false ) .id(id) } else { @@ -600,6 +597,31 @@ public struct WorkspaceView: View { return model.snapshot.projects.contains { $0.id == selectedProjectID } } + private var compactColumnBinding: Binding { + Binding( + get: { preferredCompactColumn }, + set: { column in + preferredCompactColumn = column + reconcileThreadPresentation() + } + ) + } + + private var presentedThreadID: String? { + guard let id = threadSelection.presentedID( + isCompact: horizontalSizeClass == .compact, + showsDetailColumn: preferredCompactColumn == .detail + ), model.snapshot.threads.contains(where: { $0.id == id }) else { return nil } + return id + } + + private func reconcileThreadPresentation() { + threadSelection.reconcilePresentation( + isCompact: horizontalSizeClass == .compact, + showsDetailColumn: preferredCompactColumn == .detail + ) + } + private func openThread(_ id: String) { threadSelection.open(id) preferredCompactColumn = .detail diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureMessageAgeTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureMessageAgeTests.swift new file mode 100644 index 000000000000..338396984e38 --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/FeatureMessageAgeTests.swift @@ -0,0 +1,26 @@ +import Foundation +import Testing +@testable import T3Code + +struct FeatureMessageAgeTests { + @Test func elapsedUnitsAndFutureClockSkew() { + let date = Date(timeIntervalSince1970: 1_000) + let age = FeatureMessageAge(sentAt: date) + for (seconds, expected) in [(-8, "0s"), (8, "8s"), (59, "59s"), (60, "1m"), + (3_599, "59m"), (3_600, "1h"), (86_400, "1d")] { + #expect(age.text(at: date.addingTimeInterval(Double(seconds))) == expected) + } + } + + @Test func usesConversationMessageNotToolStatusOrQueuedDraft() { + let first = FeatureMessage(id: "user", role: .user, text: "Check this", createdAt: Date(timeIntervalSince1970: 10)) + let reply = FeatureMessage(id: "reply", role: .assistant, text: "Checking", createdAt: Date(timeIntervalSince1970: 20)) + let tool = FeatureMessage(id: "tool", role: .tool, text: "Read file", createdAt: Date(timeIntervalSince1970: 30)) + let status = FeatureMessage(id: "status", role: .system, text: "Connected", createdAt: Date(timeIntervalSince1970: 40)) + let queued = FeatureMessage(id: "draft", role: .user, text: "Next", createdAt: Date(timeIntervalSince1970: 50), state: .queued) + #expect(FeatureMessageAge.lastMessageDate(in: []) == nil) + #expect(FeatureMessageAge.lastMessageDate(in: [tool, status, queued]) == nil) + #expect(FeatureMessageAge.lastMessageDate(in: [first, tool]) == first.createdAt) + #expect(FeatureMessageAge.lastMessageDate(in: [first, reply, tool, status, queued]) == reply.createdAt) + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 241354476b03..9a613778ab20 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -9,6 +9,223 @@ import XCTest @MainActor @Suite("Feature root model") struct FeatureRootModelTests { + @Test + func receiptBelongsToCurrentPresentationWithoutInvalidatingTranscript() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + var loads = loaded.stream.makeAsyncIterator() + let first = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield("a") } + } + #expect(await loads.next() == "a") + let detailRevision = model.detailRevision + let homeRevision = model.homePresentationRevision + let receipt = FeatureThreadReceipt(threadID: "a", receivedAt: Date(timeIntervalSince1970: 100), source: .detailEvent) + model.recordThreadReceipt(receipt) + #expect(model.selectedThreadReceipt == receipt) + #expect(model.detailRevision == detailRevision) + #expect(model.homePresentationRevision == homeRevision) + model.recordThreadReceipt(.init(threadID: "b", receivedAt: .now, source: .shellThreadUpdate)) + #expect(model.selectedThreadReceipt == receipt) + + let second = Task { + await model.runThreadPresentation(id: "b") { loaded.continuation.yield("b") } + } + #expect(await loads.next() == "b") + await first.value + #expect(model.selectedThreadReceipt == nil) + model.recordThreadReceipt(receipt) + #expect(model.selectedThreadReceipt == nil) + model.recordThreadReceipt(.init(threadID: "b", receivedAt: .now, source: .detailSnapshot)) + #expect(model.selectedThreadReceipt?.source == .detailSnapshot) + second.cancel() + await second.value + #expect(model.selectedThreadReceipt == nil) + model.recordThreadReceipt(.init(threadID: "b", receivedAt: .now, source: .detailEvent)) + #expect(model.selectedThreadReceipt == nil) + loaded.continuation.finish() + } + + @Test(arguments: [FeatureThreadReceipt.Source.detailEvent, .shellThreadUpdate]) + func olderReceiptCannotReplaceNewerReceipt(source: FeatureThreadReceipt.Source) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + let newest = FeatureThreadReceipt(threadID: "a", receivedAt: Date(timeIntervalSince1970: 100), source: source) + model.recordThreadReceipt(newest) + model.recordThreadReceipt(.init(threadID: "a", receivedAt: Date(timeIntervalSince1970: 99), source: .detailSnapshot)) + #expect(model.selectedThreadReceipt == newest) + let next = FeatureThreadReceipt(threadID: "a", receivedAt: Date(timeIntervalSince1970: 101), source: .shellThreadUpdate) + model.recordThreadReceipt(next) + #expect(model.selectedThreadReceipt == next) + presentation.cancel() + await presentation.value + loaded.continuation.finish() + } + + @Test(arguments: [["a", "b", "a"], ["a", "a"]]) + func presentationTaskReplacementAndCloseReleaseExactlyOnce(threadIDs: [String]) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + var loads = loaded.stream.makeAsyncIterator() + var previous: Task? + for id in threadIDs { + let next = Task { + await model.runThreadPresentation(id: id) { loaded.continuation.yield(id) } + } + #expect(await loads.next() == id) + // Replacement finishes the old lifetime even before SwiftUI cancels its task. + await previous?.value + previous?.cancel() + previous = next + } + #expect(client.loadedThreadIDs == threadIDs) + #expect(client.releasedThreadIDs == Array(threadIDs.dropLast())) + previous?.cancel() + await previous?.value + #expect(client.releasedThreadIDs == threadIDs) + loaded.continuation.finish() + } + + @Test + func cancelledPresentationTaskCannotReplaceVisibleThread() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let current = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + let cancelled = Task { @MainActor in + withUnsafeCurrentTask { $0?.cancel() } + await model.runThreadPresentation(id: "a") + } + await cancelled.value + #expect(client.loadedThreadIDs == ["a"]) + #expect(client.releasedThreadIDs.isEmpty) + current.cancel() + await current.value + #expect(client.releasedThreadIDs == ["a"]) + loaded.continuation.finish() + } + + @Test(arguments: [false, true]) + func cancellationReleasesSuspendedPresentationBeforeItsLoadReturns(cancelBeforeReplacement: Bool) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let started = AsyncStream.makeStream() + let released = AsyncStream.makeStream() + let loaded = AsyncStream.makeStream() + client.onReleaseThread = { released.continuation.yield($0) } + var pendingLoad: CheckedContinuation? + client.loadThreadHandler = { _ in + try await withCheckedThrowingContinuation { continuation in + pendingLoad = continuation + started.continuation.yield(()) + } + } + var oldLoaded = false + let old = Task { await model.runThreadPresentation(id: "a") { oldLoaded = true } } + var starts = started.stream.makeAsyncIterator() + await starts.next() + var releases = released.stream.makeAsyncIterator() + if cancelBeforeReplacement { + old.cancel() + #expect(await releases.next() == "a") + } + #expect(pendingLoad != nil) + #expect(!oldLoaded) + + client.loadThreadHandler = nil + let replacement = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + if !cancelBeforeReplacement { + // The old cancellation handler runs only after the same-ID replacement owns transport. + old.cancel() + #expect(await releases.next() == "a") + } + pendingLoad?.resume(returning: FeatureThreadDetail(thread: client.createdThread)) + pendingLoad = nil + await old.value + #expect(!oldLoaded) + #expect(client.releasedThreadIDs == ["a"]) + replacement.cancel() + await replacement.value + #expect(client.releasedThreadIDs == ["a", "a"]) + #expect(client.loadedThreadIDs == ["a", "a"]) + started.continuation.finish() + released.continuation.finish() + loaded.continuation.finish() + } + + @Test + func presentationCloseStillReleasesAfterConnectionReload() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + await model.reloadAfterConnection() + presentation.cancel() + await presentation.value + #expect(client.releasedThreadIDs == ["a"]) + loaded.continuation.finish() + } + + @Test + func presentationCloseCancelsRetryAndRejectsLateRetry() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let started = AsyncStream.makeStream() + let cancelled = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + var pendingRetry: CheckedContinuation? + client.loadThreadHandler = { _ in + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + pendingRetry = continuation + started.continuation.yield(()) + } + } onCancel: { + cancelled.continuation.yield(()) + } + } + #expect(model.refreshThreadPresentation(id: "a")) + var retries = started.stream.makeAsyncIterator() + await retries.next() + presentation.cancel() + await presentation.value + var cancellations = cancelled.stream.makeAsyncIterator() + await cancellations.next() + #expect(client.releasedThreadIDs == ["a"]) + #expect(!model.refreshThreadPresentation(id: "a")) + pendingRetry?.resume(throwing: CancellationError()) + pendingRetry = nil + #expect(client.loadedThreadIDs == ["a", "a"]) + loaded.continuation.finish() + started.continuation.finish() + cancelled.continuation.finish() + } + @Test func rewindLocksSendingAndSavesRecoveredInputAfterLeavingTheThread() async throws { let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) @@ -287,6 +504,7 @@ struct FeatureRootModelTests { await model.applicationDidBecomeActive(at: start.addingTimeInterval(10)) await model.applicationDidBecomeActive(at: start.addingTimeInterval(11)) #expect(client.foregroundReconnects == [false, true]) + #expect(client.backgroundSuspends == 2) } @Test @@ -1683,6 +1901,157 @@ struct FeatureRootModelTests { #expect(model.details[thread.id]?.messages.last?.state == .complete) } + @Test + func deliveredMessageSurvivesAnOlderDetailUntilTheServerIncludesIt() async throws { + let client = FeatureClientStub() + let thread = FeatureThread( + id: "thread-1", + projectID: "project-1", + environmentID: "environment-1", + title: "Thread" + ) + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", + name: "Studio", + endpoint: "https://studio.example", + isActive: true, + connectionState: .connected + ), + ], + threads: [thread] + ) + client.threadDetail = FeatureThreadDetail(thread: thread) + let model = testRootModel(client: client) + await model.reload() + _ = await model.detail(for: thread.id) + + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + // Its detail refresh no longer blocks the send, so an older read can land first. + _ = await model.detail(for: thread.id, force: true) + let delivered = try #require(model.details[thread.id]?.messages.last) + #expect(delivered.text == "ship it") + #expect(delivered.state == .complete) + + client.threadDetail = FeatureThreadDetail( + thread: thread, + messages: [FeatureMessage(id: delivered.id, role: .user, text: "ship it")] + ) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.filter { $0.id == delivered.id }.count == 1) + } + + private func deliveredMessageModel() async -> (FeatureClientStub, FeatureRootModel, FeatureThread) { + let client = FeatureClientStub() + let thread = FeatureThread( + id: "thread-1", projectID: "project-1", environmentID: "environment-1", title: "Thread" + ) + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", name: "Studio", endpoint: "https://studio.example", + isActive: true, connectionState: .connected + ), + ], + threads: [thread] + ) + client.threadDetail = FeatureThreadDetail(thread: thread) + let model = testRootModel(client: client) + await model.reload() + _ = await model.detail(for: thread.id) + return (client, model, thread) + } + + @Test + func confirmedDeliveredMessageDoesNotReturnAfterARewind() async throws { + let (client, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + client.threadDetail = FeatureThreadDetail( + thread: thread, messages: [FeatureMessage(id: id, role: .user, text: "ship it")] + ) + _ = await model.detail(for: thread.id, force: true) + // A rewind removes the message from the server transcript. + client.threadDetail = FeatureThreadDetail(thread: thread) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) + } + + @Test + func clearedDetailsDropRetainedDeliveredMessages() async throws { + let (_, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + await model.reloadAfterConnection() + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) + } + + @Test + func evictedDetailKeepsItsDeliveredMessageOnReopen() async throws { + let (client, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + client.loadThreadHandler = { id in + FeatureThreadDetail(thread: FeatureThread( + id: id, projectID: "project-1", environmentID: "environment-1", title: id + )) + } + // Opening and closing other threads evicts this detail from the bounded cache. + for index in 0..<6 { _ = await model.detail(for: "other-\(index)") } + model.releaseThread("other-5") + #expect(model.details[thread.id] == nil) + // The server transcript has not caught up yet. + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) + } + + @Test + func localCopiesKeepSendOrderWhenANewerSendIsDeliveredFirst() async throws { + let (client, model, thread) = await deliveredMessageModel() + client.sendMessageError = URLError(.notConnectedToInternet) + _ = await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "first", selection: nil)) + client.sendMessageError = nil + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "second", selection: nil))) + // An older read that includes neither message. + _ = await model.detail(for: thread.id, force: true) + let texts = model.details[thread.id]?.messages.map(\.text) ?? [] + #expect(texts.filter { $0 == "first" || $0 == "second" } == ["first", "second"]) + } + + @Test + func startedTaskKeepsItsPromptUntilTheServerIncludesIt() async throws { + let client = FeatureClientStub() + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", name: "Studio", endpoint: "https://studio.example", + isActive: true, connectionState: .connected + ), + ], + projects: [.init(id: "project-1", environmentID: "environment-1", name: "Native", path: "/native")] + ) + let model = testRootModel(client: client) + await model.reload() + // The server creates the thread under the ID the app queued it with. + client.beforeStartTask = { @MainActor in + if let pending = model.snapshot.threads.first { client.createdThread = pending } + } + let thread = try #require(await model.startTask(NewTaskRequest( + projectID: "project-1", prompt: "Ship it", selection: nil, + runtimeMode: .fullAccess, interactionMode: .standard + ))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + // An older read that does not include the accepted prompt yet. + client.threadDetail = FeatureThreadDetail(thread: thread) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) + } + @Test func sendPreservesTheThreadAutomaticPermission() async { let client = FeatureClientStub() @@ -3772,7 +4141,17 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { try await prepareRecovery(.init(message: message, attachments: rewindAttachments)) _ = try await rewindHandler(threadID, messageID) } + + var loadedThreadIDs: [String] = [] + var releasedThreadIDs: [String] = [] + var onReleaseThread: ((String) -> Void)? + func releaseThread(id: String) { + releasedThreadIDs.append(id) + onReleaseThread?(id) + } var foregroundReconnects: [Bool] = [] + var backgroundSuspends = 0 + func suspendForBackground() { backgroundSuspends += 1 } func resumeAfterBackground(reconnect: Bool) async { foregroundReconnects.append(reconnect) } private let eventStream: AsyncStream private let eventContinuation: AsyncStream.Continuation @@ -3800,6 +4179,8 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { var sentRuntimeModes: [FeatureRuntimeMode] = [] var setRuntimeModeCalls: [FeatureRuntimeMode] = [] var cancelTurnCallCount = 0 + var cancelTurnHandler: (() async throws -> Void)? + var stopStatusHandler: (() async throws -> FeatureThread)? var signOutCallCount = 0 var startTaskError: (any Error)? var sendMessageError: (any Error)? @@ -3950,6 +4331,7 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { } func loadThread(id: String, fresh: Bool) async throws -> FeatureThreadDetail { + loadedThreadIDs.append(id) if let loadThreadError { throw loadThreadError } @@ -3974,7 +4356,7 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { selection: FeatureSelection?, runtimeMode: FeatureRuntimeMode, attachments _: [FeatureUploadAttachment], - identity _: FeatureSubmissionIdentity, + identity: FeatureSubmissionIdentity, context: OrchestrationMessageContext? = nil ) async throws { sentRuntimeModes.append(runtimeMode) @@ -3986,6 +4368,11 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { func cancelTurn(threadID: String) async throws { cancelTurnCallCount += 1 + try await cancelTurnHandler?() + } + func stopStatus(threadID: String) async throws -> FeatureThread { + if let stopStatusHandler { return try await stopStatusHandler() } + return snapshot.threads.first { $0.id == threadID } ?? createdThread } func setThreadSettled(id: String, settled: Bool) async throws { await beforeSettlementReturn?() @@ -4050,3 +4437,308 @@ private final class FeatureSettingsSaveGate { firstRelease = nil } } + +extension FeatureRootModelTests { + @Test("A failed Stop resumes queued work for other threads", .timeLimit(.minutes(1))) + func failedStopRestartsOtherQueuedSubmissions() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + let outbox = FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")) + let identity = FeatureSubmissionIdentity(threadID: "creating", createdAt: Date(timeIntervalSince1970: 100)) + let creatingID = FeatureScopedID.thread(environmentID: "one", wireID: "creating") + let other = FeatureThread(id: "other", projectID: "project", environmentID: "one", title: "Other thread") + try await outbox.enqueue(.init(environmentID: "one", identity: identity, threadID: creatingID, + text: "Create", selection: nil, runtimeMode: .fullAccess, interactionMode: .standard, attachments: [], + creation: .init(projectID: "project", projectName: "Project", workspaceMode: .local, + branch: nil, worktreePath: nil, startFromOrigin: false))) + try await outbox.enqueue(.init(environmentID: "one", + identity: .init(threadID: "other", createdAt: Date(timeIntervalSince1970: 101)), threadID: other.id, + text: "Send the other queued message", selection: nil, runtimeMode: .fullAccess, + interactionMode: .standard, attachments: [])) + let client = FeatureClientStub() + client.snapshot = FeatureSnapshot(connection: .init(state: .connected), + environments: [.init(id: "one", name: "One", endpoint: "https://one.example", isActive: true, connectionState: .connected)], + projects: [.init(id: "project", environmentID: "one", name: "Project", path: "/project")], threads: [other]) + let model = FeatureRootModel(client: client, outboxStore: outbox) + let entered = AsyncStream.makeStream() + let sent = AsyncStream.makeStream() + let gate = StopDrainCancellationGate() + client.beforeStartTask = { + var acknowledged = try #require(model.snapshot.threads.first { $0.id == creatingID }) + acknowledged.state = .working + await withCheckedContinuation { continuation in + withObservationTracking { _ = model.snapshot.threads.first { $0.id == creatingID }?.state } + onChange: { continuation.resume() } + client.emit(.thread(acknowledged)) + } + entered.continuation.yield(()) + await gate.waitForCancellation() + } + client.cancelTurnHandler = { throw URLError(.networkConnectionLost) } + client.beforeSendMessage = { sent.continuation.yield(()) } + let run = Task { await model.start() } + var starts = entered.stream.makeAsyncIterator() + await starts.next() + await model.cancelTurn(threadID: creatingID) + var sends = sent.stream.makeAsyncIterator() + await sends.next() + #expect(client.cancelTurnCallCount == 1) + #expect(client.sentText == "Send the other queued message") + #expect(model.stopPhase(threadID: creatingID) == .unconfirmed) + client.finishEvents() + await run.value + await model.disconnect() + } +} + +@MainActor +private final class StopDrainCancellationGate { + private var pending: CheckedContinuation? + func waitForCancellation() async { + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if Task.isCancelled { continuation.resume() } + else { pending = continuation } + } + } onCancel: { + Task { @MainActor in + self.pending?.resume() + self.pending = nil + } + } + } +} + +extension FeatureRootModelTests { + @Test func duplicateStopWhileRequestHeld() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let started = AsyncStream.makeStream() + var held: CheckedContinuation? + client.cancelTurnHandler = { + if held == nil { + await withCheckedContinuation { held = $0; started.continuation.yield(()) } + } + } + let first = Task { await model.cancelTurn(threadID: "created") } + var starts = started.stream.makeAsyncIterator() + await starts.next() + await model.cancelTurn(threadID: "created") + #expect(client.cancelTurnCallCount == 1) + held?.resume() + await first.value + } + + @Test func stopClaimsSynchronouslyAndAcknowledgementIsNotTerminal() async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running")] + let model = FeatureRootModel(client: client) + await model.reload() + let called = AsyncStream.makeStream() + client.cancelTurnHandler = { called.continuation.yield(()) } + model.requestCancelTurn(threadID: "created") + #expect(model.stopPhase(threadID: "created") == .requesting) + model.requestCancelTurn(threadID: "created") + var calls = called.stream.makeAsyncIterator() + await calls.next() + #expect(client.cancelTurnCallCount == 1) + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + await model.reload() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + client.snapshot.threads[0].latestTurnState = "completed" + client.snapshot.threads[0].state = .completed + client.snapshot.threads[0].settlementFacts = .init(sessionStatus: "ready") + await model.reload() + #expect(model.stopPhase(threadID: "created") == nil) + } + + @Test func stopSurvivesCloseReopenAndMissingReconnectSeed() async { + let client = FeatureClientStub() + let thread = FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running") + client.snapshot.threads = [thread] + client.createdThread = thread + let model = FeatureRootModel(client: client) + await model.reload() + await model.cancelTurn(threadID: "created") + let loaded = AsyncStream.makeStream() + var loads = loaded.stream.makeAsyncIterator() + for _ in 0..<2 { + let presentation = Task { await model.runThreadPresentation(id: "created") { loaded.continuation.yield(()) } } + await loads.next() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + presentation.cancel() + await presentation.value + } + client.snapshot.threads = [] + await model.reloadAfterConnection() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + client.snapshot.threads = [thread] + await model.reload() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + #expect(client.cancelTurnCallCount == 1) + } + + @Test(arguments: [false, true]) + func lateOldStopResponseCannotClearNewTurn(fails: Bool) async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running")] + let model = FeatureRootModel(client: client) + await model.reload() + let started = AsyncStream.makeStream() + var held: CheckedContinuation? + client.cancelTurnHandler = { try await withCheckedThrowingContinuation { held = $0; started.continuation.yield(()) } } + let old = Task { await model.cancelTurn(threadID: "created") } + var starts = started.stream.makeAsyncIterator() + await starts.next() + client.snapshot.threads[0].latestTurnID = "b" + await model.reload() + #expect(model.stopPhase(threadID: "created") == nil) + client.cancelTurnHandler = nil + await model.cancelTurn(threadID: "created") + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + if fails { held?.resume(throwing: URLError(.networkConnectionLost)) } + else { held?.resume() } + await old.value + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + #expect(model.errorMessage == nil) + client.snapshot.threads[0].latestTurnID = "a" + client.snapshot.threads[0].latestTurnState = "completed" + client.snapshot.threads[0].state = .completed + client.snapshot.threads[0].settlementFacts = .init(sessionStatus: "ready") + await model.reload() + client.snapshot.threads[0].latestTurnID = "b" + client.snapshot.threads[0].latestTurnState = "running" + client.snapshot.threads[0].state = .working + await model.reload() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + } + + @Test func uncertainFailureRequiresStatusReceiptBeforeRetry() async { + let client = FeatureClientStub() + let thread = FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running") + client.snapshot.threads = [thread] + let model = FeatureRootModel(client: client) + await model.reload() + client.cancelTurnHandler = { throw URLError(.networkConnectionLost) } + await model.cancelTurn(threadID: "created") + #expect(model.stopPhase(threadID: "created") == .unconfirmed) + let started = AsyncStream.makeStream() + let retried = AsyncStream.makeStream() + var held: CheckedContinuation? + client.stopStatusHandler = { try await withCheckedThrowingContinuation { held = $0; started.continuation.yield(()) } } + client.cancelTurnHandler = { retried.continuation.yield(()) } + model.retryCancelTurn(threadID: "created") + var starts = started.stream.makeAsyncIterator() + await starts.next() + #expect(client.cancelTurnCallCount == 1) + #expect(model.stopPhase(threadID: "created") == .requesting) + held?.resume(returning: thread) + var retries = retried.stream.makeAsyncIterator() + await retries.next() + #expect(client.cancelTurnCallCount == 2) + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + } + + @Test(arguments: [true, false]) + func compactionAndUnknownIdentityStayExplicitlyUnconfirmed(hasIdentity: Bool) async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: hasIdentity ? "a" : nil, latestTurnState: hasIdentity ? "completed" : nil)] + let model = FeatureRootModel(client: client) + await model.reload() + await model.cancelTurn(threadID: "created") + #expect(model.stopPhase(threadID: "created") == .unconfirmed) + await model.reload() + #expect(model.stopPhase(threadID: "created") == .unconfirmed) + } + + @Test(arguments: [false, true]) + func retryPreservesRunningEvidenceAfterTerminalWhileBusy(statusStillBusy: Bool) async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running")] + let model = FeatureRootModel(client: client) + await model.reload() + client.cancelTurnHandler = { throw URLError(.networkConnectionLost) } + await model.cancelTurn(threadID: "created") + #expect(model.stopPhase(threadID: "created") == .unconfirmed) + + client.snapshot.threads[0].latestTurnState = "completed" + await model.reload() + #expect(model.stopPhase(threadID: "created") == .unconfirmed) + var status = client.snapshot.threads[0] + status.state = statusStillBusy ? .working : .completed + status.settlementFacts = .init(sessionStatus: "ready") + client.stopStatusHandler = { status } + await model.retryCancelTurn(threadID: "created")?.value + #expect(client.cancelTurnCallCount == 1) + if statusStillBusy { + #expect(model.stopPhase(threadID: "created") == .unconfirmed) + client.snapshot.threads[0].state = .completed + client.snapshot.threads[0].settlementFacts = .init(sessionStatus: "ready") + await model.reload() + } + #expect(model.stopPhase(threadID: "created") == nil) + let rejectedRetry = model.retryCancelTurn(threadID: "created") + #expect(rejectedRetry == nil) + await rejectedRetry?.value + #expect(client.cancelTurnCallCount == 1) + } + + @Test func matchingTerminalWhileStillWorkingDoesNotClearStop() async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running")] + let model = FeatureRootModel(client: client) + await model.reload() + await model.cancelTurn(threadID: "created") + client.snapshot.threads[0].latestTurnState = "completed" + await model.reload() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + client.snapshot.threads[0].state = .completed + client.snapshot.threads[0].settlementFacts = .init(sessionStatus: "ready") + await model.reload() + #expect(model.stopPhase(threadID: "created") == nil) + } + + @Test(arguments: ["completed", "different-turn", "unknown"]) + func retryReconciliationDoesNotInterruptTerminalDifferentOrUnknownTurn(statusKind: String) async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread(id: "created", projectID: "p", title: "Work", state: .working, latestTurnID: "a", latestTurnState: "running")] + let model = FeatureRootModel(client: client) + await model.reload() + client.cancelTurnHandler = { throw URLError(.networkConnectionLost) } + await model.cancelTurn(threadID: "created") + var status = client.snapshot.threads[0] + if statusKind == "completed" { + status.latestTurnState = "completed" + status.state = .completed + status.settlementFacts = .init(sessionStatus: "ready") + } + if statusKind == "different-turn" { status.latestTurnID = "b" } + if statusKind == "unknown" { status.latestTurnState = nil } + client.stopStatusHandler = { status } + await model.retryCancelTurn(threadID: "created")?.value + #expect(client.cancelTurnCallCount == 1) + if statusKind == "completed" { #expect(model.stopPhase(threadID: "created") == nil) } + } + + @Test(arguments: [nil, "missing-status", "unknown", "idle", "starting", "running"] as [String?]) + func stopRetainsFeedbackWithoutKnownInactiveSession(sessionStatus: String?) async { + let client = FeatureClientStub() + client.snapshot.threads = [FeatureThread( + id: "created", projectID: "p", title: "Work", state: .waitingForApproval, + latestTurnID: "a", latestTurnState: "running", + settlementFacts: .init(sessionStatus: "running") + )] + let model = FeatureRootModel(client: client) + await model.reload() + await model.cancelTurn(threadID: "created") + client.snapshot.threads[0].latestTurnState = "interrupted" + client.snapshot.threads[0].settlementFacts = sessionStatus.map { + .init(sessionStatus: $0 == "missing-status" ? nil : $0) + } + await model.reload() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + client.snapshot.threads[0].state = .waitingForInput + await model.reload() + #expect(model.stopPhase(threadID: "created") == .awaitingOutcome) + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureThreadReceiptTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureThreadReceiptTests.swift new file mode 100644 index 000000000000..4e7f51aedd61 --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/FeatureThreadReceiptTests.swift @@ -0,0 +1,17 @@ +import Foundation +import Testing +@testable import T3Code + +struct FeatureThreadReceiptTests { + @Test func timestampIncludesSecondsAndUsesSelectedTimeZone() throws { + let utc = try #require(TimeZone(secondsFromGMT: 0)) + let sydney = try #require(TimeZone(identifier: "Australia/Sydney")) + let first = FeatureThreadReceipt(threadID: "a", receivedAt: Date(timeIntervalSince1970: 1788877928), source: .detailEvent) + let next = FeatureThreadReceipt(threadID: "a", receivedAt: first.receivedAt.addingTimeInterval(1), source: .detailEvent) + let locale = Locale(identifier: "en_GB") + let text = first.formattedTimestamp(locale: locale, timeZone: utc) + #expect(text.contains("14:32:08")) + #expect(next.formattedTimestamp(locale: locale, timeZone: utc).contains("14:32:09")) + #expect(first.formattedTimestamp(locale: locale, timeZone: sydney).contains("0:32:08")) + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index cc188815bb14..4b2d324787a9 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -1,4 +1,6 @@ import Foundation +import Darwin +import Observation import Testing import XCTest @testable import T3Code @@ -24,7 +26,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { projectID: "project-shared", threadID: "thread-shared", title: "Remote files", workspaceRoot: #"C:\work\project"#, worktreePath: #"D:\worktrees\feature"# ), host: "two.example") - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let root = try await fixture.client.listFiles(threadID: thread.id, path: nil) @@ -56,7 +58,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { rpcConnectionWaitTimeout: .seconds(2) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "one" }) let root = try await fixture.client.listFiles(threadID: thread.id, path: nil) @@ -87,17 +89,19 @@ final class NativeMultiEnvironmentTests: XCTestCase { ])]] ) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) XCTAssertEqual(project.defaultSelection?.modelID, environmentDefault.model) _ = try await fixture.client.createThread(projectID: project.id, title: "Task", selection: nil) - let creates = await fixture.transport.dispatchRecords().filter { $0.command["type"] == .string("thread.create") } - XCTAssertEqual(creates.last?.command["modelSelection"], try JSONValue.encode(environmentDefault)) + // The configured peer keeps its socket, so the command crosses it rather than HTTP. + let creates = await server.dispatchedCommands(host: "two.example").filter { $0["type"] == .string("thread.create") } + XCTAssertEqual(creates.last?["modelSelection"], try JSONValue.encode(environmentDefault)) await fixture.client.disconnect() } @@ -117,12 +121,13 @@ final class NativeMultiEnvironmentTests: XCTestCase { ]] ) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) XCTAssertEqual(project.defaultSelection?.modelID, environmentDefault.model) XCTAssertEqual(project.supportsProjectSettingsOverrides, true) @@ -139,7 +144,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .responseStreamingMode, value: .string("turn") )) - let updated = try await fixture.client.initialSnapshot() + let updated = try await fixture.hydratedSnapshot() let updatedProject = try XCTUnwrap(updated.projects.first { $0.id == project.id }) XCTAssertEqual(updatedProject.defaultSelection?.modelID, projectDefault.model) XCTAssertEqual(updatedProject.defaultWorkspaceMode, .worktree) @@ -162,12 +167,12 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .defaultModelSelection, value: .null )) - let noDefault = try await fixture.client.initialSnapshot() + let noDefault = try await fixture.hydratedSnapshot() XCTAssertNil(noDefault.projects.first { $0.id == project.id }?.defaultSelection) try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .defaultModelSelection, value: nil )) - let inherited = try await fixture.client.initialSnapshot() + let inherited = try await fixture.hydratedSnapshot() XCTAssertEqual(inherited.projects.first { $0.id == project.id }?.defaultSelection?.modelID, environmentDefault.model) await fixture.client.disconnect() @@ -194,7 +199,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { snapshotSequence: source.snapshotSequence, projects: [wireProject], threads: source.threads, updatedAt: source.updatedAt ), host: "two.example") - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) let preferences = try await fixture.client.projectPreferences(projectID: project.id) XCTAssertEqual(preferences.environment.defaultModelSelection?.model, "environment-model") @@ -212,7 +217,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) do { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( @@ -325,7 +330,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(Set(snapshot.projects.map(\.environmentID)), ["one", "two"]) XCTAssertEqual(Set(snapshot.threads.compactMap(\.wireID)), ["thread-one", "thread-two"]) @@ -383,19 +388,19 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveProviderRefreshKeepsActiveThreadsAndAcceptsTheirNextSequence() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let providers = try await fixture.client.refreshProviders(environmentID: "two") XCTAssertEqual(providers.map(\.id), ["codex-two.example"]) - let refreshed = try await fixture.client.initialSnapshot() + let refreshed = try await fixture.hydratedSnapshot() XCTAssertEqual( refreshed.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -421,7 +426,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let updated = try await fixture.client.initialSnapshot() + let updated = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(updated.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID)), ["thread-one", "thread-new"] @@ -435,20 +440,20 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveEnvironmentSettingsDoNotReplaceActiveThreads() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() try await fixture.client.updateServerPreferences( environmentID: "two", change: .environmentIcon("mac-mini") ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -465,6 +470,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testMachineModelDefaultsRefreshCachedProjectsWithoutChangingOtherEnvironments() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), @@ -479,7 +485,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { snapshotSequence: source.snapshotSequence, projects: [project], threads: source.threads, updatedAt: source.updatedAt ), host: "two.example") - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() XCTAssertNil(initial.projects.first { $0.environmentID == "two" }?.defaultSelection) let localDefault = initial.projects.first { $0.environmentID == "one" }?.defaultSelection @@ -488,13 +494,13 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateServerPreferences(environmentID: "two", change: .sharedPreferences(.object([ "defaultModelSelection": try JSONValue.encode(selection), ]))) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.projects.first { $0.environmentID == "two" }?.defaultSelection?.modelID, model) XCTAssertEqual(snapshot.projects.first { $0.environmentID == "one" }?.defaultSelection, localDefault) } await fixture.transport.setShell(source, host: "two.example") - let overridden = try await fixture.client.initialSnapshot() + let overridden = try await fixture.hydratedSnapshot() XCTAssertEqual(overridden.projects.first { $0.environmentID == "two" }?.defaultSelection?.modelID, "gpt-5.6-sol") await fixture.client.disconnect() } @@ -502,20 +508,20 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testSharedSettingsFanOutDoesNotReplaceActiveThreads() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() try await fixture.client.updateServerPreferences( environmentID: "one", change: .defaultThreadEnvMode(.worktree) ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -532,13 +538,14 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testRestartPreferenceOnlyReachesComputersThatSupportIt() async throws { let server = MultiEnvironmentConfigurationServer(restartSupportHosts: ["one.example"]) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.preferencesByEnvironment?["one"]?.continueThreadsAfterServerUpdate, false) XCTAssertNil(snapshot.preferencesByEnvironment?["two"]?.continueThreadsAfterServerUpdate) @@ -583,7 +590,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap( snapshot.threads.first(where: { $0.wireID == "thread-one" }) ) @@ -618,7 +625,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let detail = try await fixture.client.loadThread(id: thread.id) @@ -650,7 +657,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let detail = try await fixture.client.loadThread(id: thread.id) @@ -696,7 +703,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) XCTAssertTrue(thread.isRegeneratingTitle) @@ -724,7 +731,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture(repositoryIdentity: identity) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let groups = DailyUXCreationContext.projectGroups(in: snapshot) XCTAssertEqual(Set(snapshot.projects.compactMap(\.repositoryIdentity?.canonicalKey)), [ @@ -739,15 +746,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await fixture.transport.setReachable(false, host: "two.example") - let passiveFailure = try await fixture.client.initialSnapshot() + let passiveFailure = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(passiveFailure.threads.compactMap(\.wireID)), ["thread-one", "thread-two"] ) - XCTAssertEqual(passiveFailure.connection.state, .connected) + XCTAssertEqual(passiveFailure.environments.first { $0.id == "one" }?.connectionState, .connected) XCTAssertEqual( passiveFailure.environments.first(where: { $0.id == "two" })?.connectionState, .disconnected @@ -756,7 +763,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { await fixture.transport.setReachable(false, host: "one.example") await fixture.transport.setReachable(true, host: "two.example") - let activeFailure = try await fixture.client.initialSnapshot() + let activeFailure = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(activeFailure.threads.compactMap(\.wireID)), ["thread-one", "thread-two"] @@ -773,7 +780,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testCachedShellRowsApplySettlementAndRemoveDeletedRoutes() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() let original = try XCTUnwrap(initial.threads.first { $0.environmentID == "two" }) let updated = multiEnvironmentShell( projectID: "project-two", threadID: "thread-two", title: "Remote work", @@ -782,7 +789,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { settledOverride: "settled", settledAt: "2026-07-31T12:01:00.000Z" ) await fixture.transport.setShell(updated, host: "two.example") - let refreshed = try await fixture.client.initialSnapshot() + let refreshed = try await fixture.hydratedSnapshot() let settled = try XCTUnwrap(refreshed.threads.first { $0.id == original.id }) XCTAssertEqual(settled.updatedAt, original.updatedAt) XCTAssertTrue(settled.isSettled) @@ -796,7 +803,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "two.example" ) - let removed = try await fixture.client.initialSnapshot() + let removed = try await fixture.hydratedSnapshot() XCTAssertFalse(removed.threads.contains { $0.id == original.id }) XCTAssertEqual(removed.projects.first { $0.environmentID == "two" }?.threadCount, 0) do { @@ -811,7 +818,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testOlderHTTPSnapshotCannotReplaceNewerEnvironmentState() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let newer = multiEnvironmentShell( projectID: "project-one", @@ -827,7 +834,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let older = multiEnvironmentShell( projectID: "project-one", @@ -844,7 +851,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.first(where: { $0.environmentID == "one" })?.title, @@ -856,7 +863,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testThreadCreationCannotReplaceNewerEnvironmentStateWithAnOlderShell() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let newer = multiEnvironmentShell( projectID: "project-one", @@ -872,7 +879,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - let current = try await fixture.client.initialSnapshot() + let current = try await fixture.hydratedSnapshot() let project = try XCTUnwrap( current.projects.first(where: { $0.environmentID == "one" }) ) @@ -897,7 +904,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { title: "Another task", selection: nil ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.first(where: { $0.wireID == "thread-one" })?.title, @@ -969,7 +976,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await loader.waitForCallCount(2) let retryCallCount = await loader.callCount @@ -990,7 +997,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { _ = try await fixture.client.initialSnapshot() await loader.waitForCallCount(1) - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await loader.waitForFirstLoadCancellation() await loader.waitForCallCount(2) @@ -1003,7 +1010,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture(duplicateIDs: true) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.projects.count, 2) XCTAssertEqual(snapshot.threads.count, 2) XCTAssertEqual(Set(snapshot.projects.map(\.id)).count, 2) @@ -1026,7 +1033,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let remoteProject = try XCTUnwrap( snapshot.projects.first(where: { $0.environmentID == "two" }) ) @@ -1059,7 +1066,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveCreateRecoversACommittedThreadAfterItsReplyIsLost() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap( snapshot.projects.first(where: { $0.environmentID == "two" }) ) @@ -1084,32 +1091,17 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testUnarchiveImmediatelyRestoresLiveThreadWhenRefreshIsUnavailable() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap( initial.threads.first(where: { $0.environmentID == "one" }) ) - let events = fixture.client.events() - var iterator = events.makeAsyncIterator() + let recorder = try XCTUnwrap(fixture.recorder) await fixture.transport.setShellReadsEnabled(false, host: "one.example") - try await fixture.client.setThreadArchived(id: thread.id, archived: true) - while let event = await iterator.next() { - if case let .thread(candidate) = event, - candidate.id == thread.id, - candidate.isArchived { - break - } - } + _ = try await recorder.wait { $0.threads.contains { $0.id == thread.id && $0.isArchived } } try await fixture.client.setThreadArchived(id: thread.id, archived: false) - var restored: FeatureThread? - while let event = await iterator.next() { - if case let .thread(candidate) = event, - candidate.id == thread.id, - !candidate.isArchived { - restored = candidate - break - } - } + let restoredSnapshot = try await recorder.wait { $0.threads.contains { $0.id == thread.id && !$0.isArchived } } + let restored = restoredSnapshot.threads.first { $0.id == thread.id } XCTAssertEqual(restored?.id, thread.id) XCTAssertEqual(restored?.isArchived, false) @@ -1122,7 +1114,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { fallbackPollingInterval: .seconds(2) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let current = multiEnvironmentShell( projectID: "project-one", threadID: "thread-one", @@ -1148,23 +1140,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - let events = fixture.client.events() - var iterator = events.makeAsyncIterator() - var refreshed: FeatureSnapshot? - while let event = await iterator.next() { - if case let .snapshot(snapshot) = event, - snapshot.projects.contains(where: { $0.wireID == addedProject.id }) { - refreshed = snapshot - break - } - } - - XCTAssertEqual(refreshed?.connection.state, .reconnecting) + let recorder = try XCTUnwrap(fixture.recorder) + let refreshed = try await recorder.wait { $0.projects.contains { $0.wireID == addedProject.id } } + XCTAssertEqual(refreshed.connection.state, .reconnecting) await fixture.client.disconnect() } fileprivate static func makeFixture( duplicateIDs: Bool = false, + requireConfiguration: Bool = false, passiveSequence: Int = 1, includeThirdEnvironment: Bool = false, repositoryIdentity: RepositoryIdentity? = nil, @@ -1173,12 +1157,25 @@ final class NativeMultiEnvironmentTests: XCTestCase { rpcConnectionWaitTimeout: Duration = .milliseconds(5), fallbackPollingInitialDelay: Duration = .seconds(3), fallbackPollingInterval: Duration = .seconds(2), + shellReconciliationSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, aggregateRefreshInterval: Duration = NativeFeatureClient.defaultAggregateRefreshInterval, aggregateIdleRefreshInterval: Duration = NativeFeatureClient.defaultAggregateIdleRefreshInterval, aggregateFailureRefreshInterval: Duration = NativeFeatureClient.defaultAggregateFailureRefreshInterval, aggregateRefreshSleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }, + aggregatePeerRefreshSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(250)) + }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, aggregateEnvironmentLoader: @escaping @Sendable (EnvironmentRuntime) async throws -> [Environment] = { try await $0.environments() } @@ -1273,20 +1270,30 @@ final class NativeMultiEnvironmentTests: XCTestCase { let settings = UserDefaults( suiteName: "t3-native-multi-\(UUID().uuidString)" )! + let receipts = PassiveLiveReceipts() return MultiEnvironmentFixture( directory: directory, transport: transport, + runtime: runtime, client: NativeFeatureClient( runtime: runtime, settingsStore: settings, fallbackPollingInitialDelay: fallbackPollingInitialDelay, fallbackPollingInterval: fallbackPollingInterval, + shellReconciliationSleep: shellReconciliationSleep, aggregateRefreshInterval: aggregateRefreshInterval, aggregateIdleRefreshInterval: aggregateIdleRefreshInterval, aggregateFailureRefreshInterval: aggregateFailureRefreshInterval, aggregateRefreshSleep: aggregateRefreshSleep, + aggregatePeerRefreshSleep: aggregatePeerRefreshSleep, + aggregateStreamRetrySleep: aggregateStreamRetrySleep, + aggregatePublishSleep: aggregatePublishSleep, + aggregateRefreshReceipt: { receipt in + receipts.record(receipt) + aggregateRefreshReceipt(receipt) + }, aggregateEnvironmentLoader: aggregateEnvironmentLoader - ) + ), receipts: receipts, requireConfiguration: requireConfiguration ) } } @@ -1369,11 +1376,13 @@ struct NativeGitHubRoutingTests { private func configuredFixture(_ server: MultiEnvironmentConfigurationServer, includeThird: Bool = false) async throws -> MultiEnvironmentFixture { let fixture = try await NativeMultiEnvironmentTests.makeFixture( + requireConfiguration: true, includeThirdEnvironment: includeThird, pullRequestsAvailable: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(2) ) - _ = try await fixture.client.initialSnapshot() + // Peers hydrate independently; routing reads each peer's configuration. + _ = try await fixture.hydratedSnapshot() for id in includeThird ? ["one", "two", "three"] : ["one", "two"] { try await fixture.client.setGitHubRoutingPermission(environmentID: id, permission: .readWrite) } @@ -1391,19 +1400,17 @@ struct NativePassiveThreadRefreshTests { func passiveThreadEventsArriveWithinFiveSecondsAndStayFastAfterChanges() async throws { let refreshSleep = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { _, interval in + try await refreshSleep.sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() - let thread = try #require( - initial.threads.first(where: { $0.environmentID == "two" }) - ) + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") let updatedTitle = "Passive work updated automatically" let eventProbe = ThreadTitleEventProbe( events: fixture.client.events(), - threadID: thread.id, + threadID: threadID, title: updatedTitle ) eventProbe.start() @@ -1433,8 +1440,8 @@ struct NativePassiveThreadRefreshTests { func passiveRefreshUsesTenSecondsWhenWorkIsUnchanged() async throws { let refreshSleep = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { _, interval in + try await refreshSleep.sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } @@ -1448,52 +1455,199 @@ struct NativePassiveThreadRefreshTests { await fixture.client.disconnect() } - @Test("A failed passive environment backs off without slowing an active peer") + @Test("A failed passive environment backs off without slowing an active peer", .timeLimit(.minutes(1))) func failedPassiveEnvironmentBacksOffWithoutSlowingActivePeer() async throws { - let refreshSleep = ControllableAggregateRefreshSleep() + let healthyClock = ControllableAggregateRefreshSleep() + let failedClock = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( includeThirdEnvironment: true, - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? healthyClock : failedClock).sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } _ = try await fixture.client.initialSnapshot() - await fixture.transport.setShell( - multiEnvironmentShell( - projectID: "project-two", - threadID: "thread-two", - title: "Remote work", - providerID: "claudeAgent", - modelID: "claude-opus-4-1", - backgroundLiveness: .working - ), - host: "two.example" - ) + _ = await healthyClock.waitUntilRequested(count: 1) + _ = await failedClock.waitUntilRequested(count: 1) await fixture.transport.setReachable(false, host: "three.example") + await failedClock.resume() + let failureCadence = await failedClock.waitUntilRequested(count: 2) + #expect(failureCadence == .seconds(20)) + for count in 2...4 { + await healthyClock.resume() + _ = await healthyClock.waitUntilRequested(count: count) + let failedReads = await fixture.transport.shellReadCount(host: "three.example") + #expect(failedReads == 2) + } + await failedClock.resume() + _ = await failedClock.waitUntilRequested(count: 3) + let retriedReads = await fixture.transport.shellReadCount(host: "three.example") + #expect(retriedReads == 3) + await fixture.client.disconnect() + } - let firstCadence = await refreshSleep.waitUntilRequested(count: 1) - #expect(firstCadence == .seconds(5)) - await refreshSleep.resume() - let secondCadence = await refreshSleep.waitUntilRequested(count: 2) - #expect(secondCadence == .seconds(5)) - let initialFailedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(initialFailedReadCount == 2) + @Test("Healthy rows publish twice while a peer shell and optional catalogue remain held", .timeLimit(.minutes(1))) + func healthyRowsPublishTwiceWhilePeerAndCatalogueAreHeld() async throws { + let healthyClock = ControllableAggregateRefreshSleep() + let slowClock = ControllableAggregateRefreshSleep() + let connector = GatedPassiveCatalogueConnector() + let topologyGate = PassiveRequestGate() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + webSocketConnector: connector, + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? healthyClock : slowClock).sleep(for: interval) + }, + aggregateEnvironmentLoader: { runtime in + await topologyGate.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let healthyID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + let heldShell = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "three.example", gate: heldShell) + await connector.holdPassiveConnections() + await topologyGate.release() + await connector.gate.waitUntilEntered() + _ = await healthyClock.waitUntilRequested(count: 1) + await heldShell.waitUntilEntered() + for update in 1...2 { + let title = "Healthy update \(update)" + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: healthyID, title: title) + probe.start() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: title, + snapshotSequence: update + 1 + ), host: "two.example") + await healthyClock.resume() + await probe.waitUntilObserved() + #expect(probe.didObserveTitle()) + _ = await healthyClock.waitUntilRequested(count: update + 1) + #expect(await heldShell.isHeld) + #expect(await connector.gate.isHeld) + } + await fixture.client.disconnect() + await heldShell.release() + await connector.release() + } + @Test("A cancelled refresh generation cannot overwrite a restarted peer", .timeLimit(.minutes(1))) + func cancelledGenerationCannotOverwriteRestartedPeer() async throws { + let clock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + aggregatePeerRefreshSleep: { _, interval in try await clock.sleep(for: interval) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + _ = await clock.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let gate = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Stale generation", + snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: gate) + await clock.resume() + await gate.waitUntilEntered() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Current generation", + snapshotSequence: 2 + ), host: "two.example") + _ = try await fixture.client.initialSnapshot() + #expect(oldWorker.isCancelled) + await gate.release() + await oldWorker.value + // A stale 999 would survive this read of sequence 2 if it reached the cache. + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Current generation") + await fixture.client.disconnect() + } - for requestCount in 2...4 { - await refreshSleep.resume() - let cadence = await refreshSleep.waitUntilRequested(count: requestCount + 1) - #expect(cadence == .seconds(5)) - let failedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(failedReadCount == 2) + @Test("Failed environment writes preserve the passive worker", arguments: [false, true]) + func failedEnvironmentWritePreservesWorker(removal: Bool) async throws { + let clock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + aggregatePeerRefreshSleep: { _, interval in try await clock.sleep(for: interval) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + _ = await clock.waitUntilRequested(count: 1) + let worker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let catalog = fixture.directory.appendingPathComponent("environments.json") + let data = try Data(contentsOf: catalog) + try FileManager.default.removeItem(at: catalog) + try FileManager.default.createDirectory(at: catalog, withIntermediateDirectories: false) + do { + if removal { try await fixture.client.removeEnvironment(id: "two") } + else { try await fixture.client.setEnvironmentEnabled(id: "two", enabled: false) } + Issue.record("The blocked catalog write unexpectedly succeeded") + } catch { + #expect(!worker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] != nil) } + try FileManager.default.removeItem(at: catalog) + try data.write(to: catalog) + #expect(try await fixture.runtime.environments().contains { $0.id == "two" && $0.isEnabled }) + let probe = ThreadTitleEventProbe( + events: fixture.client.events(), + threadID: FeatureScopedID.thread(environmentID: "two", wireID: "thread-two"), + title: "Still updating" + ) + probe.start() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Still updating", snapshotSequence: 2 + ), host: "two.example") + await clock.resume() + await probe.waitUntilObserved() + #expect(probe.didObserveTitle()) + await fixture.client.disconnect() + } - await refreshSleep.resume() - _ = await refreshSleep.waitUntilRequested(count: 6) - let retriedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(retriedReadCount == 3) + @Test("Removal cancels a held peer and its late response never returns a row", .timeLimit(.minutes(1))) + func removedPeerCannotPublishLateResponse() async throws { + let removedClock = ControllableAggregateRefreshSleep() + let healthyClock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? removedClock : healthyClock).sleep(for: interval) + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let healthyID = FeatureScopedID.thread(environmentID: "three", wireID: "thread-three") + let probe = ThreadTitleEventProbe( + events: fixture.client.events(), threadID: healthyID, title: "Removal verified" + ) + probe.start() + _ = await removedClock.waitUntilRequested(count: 1) + _ = await healthyClock.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let gate = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Ghost removed row", + snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: gate) + await removedClock.resume() + await gate.waitUntilEntered() + try await fixture.client.removeEnvironment(id: "two") + #expect(oldWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + await gate.release() + await oldWorker.value + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-three", threadID: "thread-three", title: "Removal verified", + snapshotSequence: 2 + ), host: "three.example") + await healthyClock.resume() + await probe.waitUntilObserved() + #expect(!probe.sawThreadTitle("Ghost removed row")) + #expect(!probe.lastEnvironmentIDs.contains("two")) await fixture.client.disconnect() } + } private actor FailOnceAggregateEnvironmentLoader { @@ -1678,6 +1832,8 @@ private final class ThreadTitleEventProbe { private let threadID: String private let title: String private var observed = false + private var observedTitles = Set() + private(set) var lastEnvironmentIDs = Set() private var observedWaiters: [CheckedContinuation] = [] private var task: Task? @@ -1693,12 +1849,15 @@ private final class ThreadTitleEventProbe { for await event in events { switch event { case let .thread(thread): + observedTitles.insert(thread.title) observed = thread.id == threadID && thread.title == title case let .snapshot(snapshot): + observedTitles.formUnion(snapshot.threads.map(\.title)) + lastEnvironmentIDs = Set(snapshot.environments.map(\.id)) observed = snapshot.threads.contains { $0.id == self.threadID && $0.title == self.title } - case .connection, .threadRemoved, .detail, .detailDelta, .threadSync, .failure: + case .connection, .threadRemoved, .detail, .detailDelta, .threadSync, .threadReceipt, .failure: observed = false } if observed { @@ -1710,6 +1869,8 @@ private final class ThreadTitleEventProbe { } } + func sawThreadTitle(_ title: String) -> Bool { observedTitles.contains(title) } + func didObserveTitle() -> Bool { observed } @@ -1727,6 +1888,7 @@ private final class ThreadTitleEventProbe { } private actor ControllableAggregateRefreshSleep { + var requestCount: Int { requestedCadences.count } private var requestedCadences: [Duration] = [] private var requestWaiters: [( count: Int, @@ -1768,10 +1930,47 @@ private actor ControllableAggregateRefreshSleep { } } -private struct MultiEnvironmentFixture { +@MainActor +private final class MultiEnvironmentFixture { let directory: URL let transport: MultiEnvironmentHTTPTransport + let runtime: EnvironmentRuntime let client: NativeFeatureClient + let receipts: PassiveLiveReceipts + let requireConfiguration: Bool + private(set) var recorder: BootstrapSnapshotRecorder? + + init(directory: URL, transport: MultiEnvironmentHTTPTransport, runtime: EnvironmentRuntime, + client: NativeFeatureClient, receipts: PassiveLiveReceipts, requireConfiguration: Bool) { + self.directory = directory + self.transport = transport + self.runtime = runtime + self.client = client + self.receipts = receipts + self.requireConfiguration = requireConfiguration + } + + func hydratedSnapshot() async throws -> FeatureSnapshot { + let ids = try await runtime.environments().filter(\.isEnabled).map(\.id) + let targets = Dictionary(uniqueKeysWithValues: ids.map { ($0, receipts.httpCount($0) + 1) }) + let seed = try await client.initialSnapshot() + if let recorder { recorder.record(seed) } + else { recorder = BootstrapSnapshotRecorder(seed: seed, events: client.events()) } + for id in ids { + try await receipts.waitForHTTP(id, count: targets[id]!) + if requireConfiguration { try await receipts.waitForConfiguration(id) } + } + let needsPublishedConfiguration = requireConfiguration + return try await recorder!.wait { snapshot in + ids.allSatisfy { id in + // The configuration receipt precedes publication. Wait for + // the same event consumer to observe settings in its snapshot. + if needsPublishedConfiguration && snapshot.preferencesByEnvironment?[id] == nil { return false } + guard let state = snapshot.environments.first(where: { $0.id == id })?.connectionState else { return false } + return state == .connected || state == .disconnected + } + } + } } private actor MultiEnvironmentHTTPTransport: HTTPTransport { @@ -1780,12 +1979,15 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { private var detailData: [String: [String: Data]] = [:] private var reachableHosts: Set private var shellReadsEnabledHosts: Set + private var rejectedShellHosts: Set = [] private var shellReadCounts: [String: Int] = [:] private var dispatched: [MultiEnvironmentDispatchRecord] = [] private var hostsDroppingNextCreateReply = Set() private var diffRequests: [(host: String, input: JSONValue)] = [] func pullRequestDiffRequests() -> [(host: String, input: JSONValue)] { diffRequests } + private var nextShellGates: [String: PassiveRequestGate] = [:] + private var nextDispatchGates: [String: PassiveRequestGate] = [:] init(shells: [String: OrchestrationShellSnapshot]) { self.shells = shells @@ -1802,6 +2004,11 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { } } + /// Answers this host's shell reads with 401, as after a revoked credential. + func rejectShellReads(host: String) { + rejectedShellHosts.insert(host) + } + func setShellReadsEnabled(_ enabled: Bool, host: String) { if enabled { shellReadsEnabledHosts.insert(host) @@ -1837,11 +2044,39 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { hostsDroppingNextCreateReply.insert(host) } - func data(for request: URLRequest) throws -> (Data, HTTPURLResponse) { + func holdNextDispatch(host: String, gate: PassiveRequestGate) { nextDispatchGates[host] = gate } + + func holdNextShell(host: String, gate: PassiveRequestGate) { + nextShellGates[host] = gate + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { let host = request.url?.host ?? "" let path = request.url?.path ?? "" + if path == "/.well-known/t3/environment" { + let id = host == "one.example" ? "one" : "two" + return (try JSONEncoder.t3.encode(multiEnvironmentDescriptor( + environmentID: id, label: id == "one" ? "Left Book" : "Steam Box", pullRequestsAvailable: true + )), multiEnvironmentResponse(request)) + } + if path == "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/oauth/token" { + return (Data(#"{"access_token":"paired-token","issued_token_type":"urn:ietf:params:oauth:token-type:access_token","token_type":"Bearer","expires_in":3600,"scope":"read write"}"#.utf8), multiEnvironmentResponse(request)) + } if path == "/api/orchestration/shell" { shellReadCounts[host, default: 0] += 1 + if rejectedShellHosts.contains(host) { + return ( + Data(#"{"error":"Unauthorized"}"#.utf8), + HTTPURLResponse( + url: request.url!, statusCode: 401, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + )! + ) + } + if let gate = nextShellGates.removeValue(forKey: host), let data = shellData[host] { + await gate.enter() + return (data, multiEnvironmentResponse(request)) + } } guard reachableHosts.contains(host) else { throw URLError(.cannotConnectToHost) @@ -1876,6 +2111,7 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { dispatched.append( MultiEnvironmentDispatchRecord(host: host, command: command) ) + if let gate = nextDispatchGates.removeValue(forKey: host) { await gate.enter() } if command["type"]?.stringValue == "thread.create", hostsDroppingNextCreateReply.remove(host) != nil, let projectID = command["projectId"]?.stringValue, @@ -1935,6 +2171,7 @@ private actor MultiEnvironmentConfigurationServer { private let routingIdentityHosts: Set private let failPullRequestWrites: Bool private var prRequests: [(host: String, method: String, input: JSONValue)] = [] + private var dispatches: [(host: String, command: JSONValue)] = [] init( restartSupportHosts: Set = [], @@ -1962,6 +2199,7 @@ private actor MultiEnvironmentConfigurationServer { func settings(host: String) -> [String: JSONValue] { settingsByHost[host] ?? [:] } func fileRequests() -> [(host: String, input: JSONValue)] { directoryRequests } func pullRequestRequests() -> [(host: String, method: String, input: JSONValue)] { prRequests } + func dispatchedCommands(host: String) -> [JSONValue] { dispatches.filter { $0.host == host }.map(\.command) } func response(to request: JSONValue, host: String) throws -> JSONValue? { guard let tag = request["tag"]?.stringValue, @@ -2031,6 +2269,9 @@ private actor MultiEnvironmentConfigurationServer { } settingsByHost[host] = updated value = .object(settingsByHost[host] ?? [:]) + case RPCMethod.dispatchCommand.rawValue: + dispatches.append((host, request["payload"] ?? .object([:]))) + value = try JSONValue.encode(DispatchResult(sequence: 2)) case RPCMethod.getArchivedShellSnapshot.rawValue: value = try JSONValue.encode(OrchestrationShellSnapshot( snapshotSequence: 0, projects: [], threads: [], updatedAt: "2026-09-04T12:00:00.000Z" @@ -2333,3 +2574,1578 @@ private func multiEnvironmentResponse(_ request: URLRequest) -> HTTPURLResponse headerFields: ["Content-Type": "application/json"] )! } + +/// Deliberately ignores cancellation to model a transport completing an old read. +private actor PassiveRequestGate { + private var cancellableEntryWaiters: [UUID: CheckedContinuation] = [:] + private var entered = false + private var released = false + private var entryWaiters: [CheckedContinuation] = [] + private var releaseWaiters: [CheckedContinuation] = [] + var isHeld: Bool { entered && !released } + + func enter() async { + entered = true + let pending = cancellableEntryWaiters.values + cancellableEntryWaiters.removeAll() + pending.forEach { $0.resume() } + entryWaiters.forEach { $0.resume() } + entryWaiters.removeAll() + guard !released else { return } + await withCheckedContinuation { releaseWaiters.append($0) } + } + + func waitUntilEnteredCancellable() async throws { + try Task.checkCancellation() + guard !entered else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { cancellableEntryWaiters[id] = $0 } + } onCancel: { + Task { await self.cancelEntryWaiter(id) } + } + } + + private func cancelEntryWaiter(_ id: UUID) { + cancellableEntryWaiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } + + func waitUntilEntered() async { + guard !entered else { return } + await withCheckedContinuation { entryWaiters.append($0) } + } + + func release() { + released = true + releaseWaiters.forEach { $0.resume() } + releaseWaiters.removeAll() + } +} + +private actor GatedPassiveCatalogueConnector: WebSocketConnecting { + private var shouldHold = false + let gate = PassiveRequestGate() + + func holdPassiveConnections() { shouldHold = true } + + func connect(to url: URL) async throws -> any WebSocketConnection { + if shouldHold && url.host == "two.example" { + return GatedPassiveCatalogueConnection(gate: gate) + } + throw URLError(.cannotConnectToHost) + } + + func release() async { await gate.release() } +} + +private actor GatedPassiveCatalogueConnection: WebSocketConnection { + let gate: PassiveRequestGate + private var receiver: CheckedContinuation? + private var closed = false + + init(gate: PassiveRequestGate) { self.gate = gate } + + func send(_ data: Data) async throws { + let request = try JSONDecoder.t3.decode(JSONValue.self, from: data) + if request["tag"]?.stringValue != nil { await gate.enter() } + } + + func receive() async throws -> Data { + if closed { throw CancellationError() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + closed = true + receiver?.resume(throwing: CancellationError()) + receiver = nil + } +} + +@Suite("Native passive live shells") +@MainActor +struct NativePassiveLiveShellTests { + @Test("A live peer whose credential is rejected stops quiet reconciliation", .timeLimit(.minutes(1))) + func livePeerRejectedCredentialStopsReconciling() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clock = SilentShellClock() + let (peerSleeps, peerSleepRecorder) = AsyncStream.makeStream() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .zero, + fallbackPollingInterval: .milliseconds(1), + shellReconciliationSleep: { id, interval in + if id == "two" { try await clock.sleep(interval) } + else { try await Task.sleep(for: interval) } + }, + aggregatePeerRefreshSleep: { id, interval in + if id == "two" { peerSleepRecorder.yield(interval) } + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + try await receipts.waitForHTTP("two", count: 1) + await server.waitForSubscriptions(host: "two.example", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Live seed", snapshotSequence: 10 + ), host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + try await clock.waitForRequest() + await fixture.transport.rejectShellReads(host: "two.example") + await clock.advance() + // The socket is still open, but the credential is gone: show it and back off. + _ = try await recorder.wait { + $0.environments.first { $0.id == "two" }?.connectionState == .needsPairing + } + // The next wait is the long rejected-credential back-off, not another quiet read. + var sleeps = peerSleeps.makeAsyncIterator() + while let next = await sleeps.next(), next != .seconds(24 * 60 * 60) {} + #expect(await clock.intervals.count == 1) + // Updates still arriving on the open socket must not hide the rejection. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "After rejection", snapshotSequence: 11 + ), host: "two.example") + let afterUpdate = try await recorder.wait { $0.threads.contains { $0.title == "After rejection" } } + #expect(afterUpdate.environments.first { $0.id == "two" }?.connectionState == .needsPairing) + // A closed stream and its fresh snapshot must neither hide the rejection nor end the back-off. + let reads = await fixture.transport.shellReadCount(host: "two.example") + try await server.finish(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 2) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "After reconnect", snapshotSequence: 12 + ), host: "two.example") + let afterReconnect = try await recorder.wait { $0.threads.contains { $0.title == "After reconnect" } } + #expect(afterReconnect.environments.first { $0.id == "two" }?.connectionState == .needsPairing) + // A woken fallback read would land before this later update is published. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Still backed off", snapshotSequence: 13 + ), host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Still backed off" } } + #expect(await fixture.transport.shellReadCount(host: "two.example") == reads) + await fixture.client.disconnect() + } + + @Test( + "Stop status accepts a read superseded while the live stream is authoritative", + arguments: [("one", false), ("one", true), ("two", false)] + ) + func stopStatusDuringLiveUpdates(environmentID: String, authorityLost: Bool) async throws { + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregateRefreshInterval: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + let host = environmentID + ".example" + let projectID = "project-" + environmentID + let threadID = "thread-" + environmentID + await server.waitForSubscriptions(host: host, count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: projectID, threadID: threadID, title: "Live seed", snapshotSequence: 10 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: host, gate: held) + let status = Task { @MainActor in + try await fixture.client.stopStatus(threadID: FeatureScopedID.thread(environmentID: environmentID, wireID: threadID)) + } + try await held.waitUntilEnteredCancellable() + if authorityLost { + // The stream stops being authoritative, so the cache cannot stand in for the read. + try await server.emit([.object(["kind": .string("refresh-required")])], host: host) + _ = try await recorder.wait { $0.connection.state == .reconnecting } + } else { + // A live turn keeps advancing the shell while the Stop status read is in flight. + try await server.snapshot(multiEnvironmentShell( + projectID: projectID, threadID: threadID, title: "Live update", snapshotSequence: 11 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live update" } } + } + await held.release() + if authorityLost { + await #expect(throws: CancellationError.self) { try await status.value } + } else { + #expect(try await status.value.title == "Live update") + } + await fixture.client.disconnect() + } + + @Test("Silent connected shells reconcile through bounded HTTP", arguments: ["one", "two"], [false, true]) + func silentConnectedShellReconciles(environmentID: String, disconnect: Bool) async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clock = SilentShellClock() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .zero, + fallbackPollingInterval: .milliseconds(1), + shellReconciliationSleep: { id, interval in + if id == environmentID { try await clock.sleep(interval) } + else { try await Task.sleep(for: interval) } + }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + let host = environmentID + ".example" + let threadID = "thread-" + environmentID + try await receipts.waitForHTTP(environmentID, count: 1) + await server.waitForSubscriptions(host: host, count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "Live seed", snapshotSequence: 10 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + let reads = await fixture.transport.shellReadCount(host: host) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "HTTP repaired silent shell", snapshotSequence: 11 + ), host: host) + let held = PassiveRequestGate() + let operation = Task { @MainActor in + try await clock.waitForRequest() + #expect(await clock.intervals == [.seconds(30)]) + #expect(await fixture.transport.shellReadCount(host: host) == reads) + await clock.advance() + _ = try await recorder.wait { $0.threads.contains { $0.title == "HTTP repaired silent shell" } } + #expect(await fixture.transport.shellReadCount(host: host) == reads + 1) + try await clock.waitForRequest(2) + await fixture.transport.holdNextShell(host: host, gate: held) + await clock.advance() + try await held.waitUntilEnteredCancellable() + #expect(await clock.intervals.count == 2) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 2) + let completions = receipts.httpCount(environmentID) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "Newer live shell", snapshotSequence: 20 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Newer live shell" } } + await held.release() + try await receipts.waitForHTTP(environmentID, count: completions + 1) + try await clock.waitForRequest(3) + #expect(recorder.history.last?.threads.contains { $0.title == "Newer live shell" } == true) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 2) + // A failed quiet check neither disconnects the live stream nor + // retries until another full reconciliation interval has elapsed. + await fixture.transport.setShellReadsEnabled(false, host: host) + await clock.advance() + try await clock.waitForRequest(4) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 3) + #expect(recorder.history.last?.environments.first { $0.id == environmentID }?.connectionState == .connected) + #expect(await clock.intervals == Array(repeating: .seconds(30), count: 4)) + if disconnect { await fixture.client.disconnect() } + else { fixture.client.suspendForBackground() } + try await clock.waitForCancellation() + #expect(await fixture.transport.shellReadCount(host: host) == reads + 3) + } + // Failure watchdog only; successful verification advances an explicit clock. + let watchdog = Task { + try? await Task.sleep(for: .seconds(3)) + if !Task.isCancelled { operation.cancel() } + } + do { try await operation.value } + catch { Issue.record("Silent connected shell never scheduled or applied its HTTP repair: \(error)") } + watchdog.cancel() + await held.release() + recorder.stop() + await fixture.client.disconnect() + } + + @Test("Live peer bursts publish together without polling", .timeLimit(.minutes(1))) + func livePeerBurstsPublishTogetherWithoutPolling() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let topologyClock = ControllableAggregateRefreshSleep() + let clocks = PassiveClockBank() + let publication = ControllableAggregateRefreshSleep() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregateRefreshSleep: { try await topologyClock.sleep(for: $0) }, + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: { try await publication.sleep(for: .zero) }, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threeID = FeatureScopedID.thread(environmentID: "three", wireID: "thread-three") + let heldTwo = PassiveRequestGate() + let heldThree = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "two.example", gate: heldTwo) + await fixture.transport.holdNextShell(host: "three.example", gate: heldThree) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await server.waitForSubscriptions(host: "three.example", count: 1) + await heldTwo.waitUntilEntered() + await heldThree.waitUntilEntered() + let shellTwo = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Live base", snapshotSequence: 10) + let shellThree = multiEnvironmentShell(projectID: "project-three", threadID: "thread-three", title: "Third base", snapshotSequence: 20) + try await server.snapshot(shellTwo, host: "two.example") + try await server.snapshot(shellThree, host: "three.example") + try await receipts.waitForShells("two", count: 1) + try await receipts.waitForShells("three", count: 1) + let finalTwo = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Burst final", snapshotSequence: 12) + let finalThree = multiEnvironmentShell(projectID: "project-three", threadID: "thread-three", title: "Third final", snapshotSequence: 21) + try await server.upsert(finalTwo.threads[0], sequence: 11, host: "two.example") + try await server.upsert(finalTwo.threads[0], sequence: 12, host: "two.example") + try await server.upsert(finalThree.threads[0], sequence: 21, host: "three.example") + try await receipts.waitForShells("two", count: 3) + try await receipts.waitForShells("three", count: 2) + _ = await publication.waitUntilRequested(count: 1) + #expect(await publication.requestCount == 1) + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: threeID, title: "Third final") + probe.start() + await publication.resume() + await probe.waitUntilObserved() + #expect(probe.sawThreadTitle("Burst final")) + #expect(await heldTwo.isHeld) + #expect(await heldThree.isHeld) + await heldTwo.release() + await heldThree.release() + try await receipts.waitForHTTP("two", count: 1) + try await receipts.waitForHTTP("three", count: 1) + let reads = await fixture.transport.shellReadCount(host: "two.example") + #expect(reads == 1) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + await fixture.client.disconnect() + } + + @Test("A replacement socket accepts a low sequence and rejects old HTTP", .timeLimit(.minutes(1))) + func replacementSocketRejectsOldHTTP() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let clocks = PassiveClockBank() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let oldHTTP = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Old HTTP", snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: oldHTTP) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await oldHTTP.waitUntilEntered() + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Old socket", snapshotSequence: 100 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + await server.closeLatest(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 2) + let fresh = multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Replacement socket", snapshotSequence: 2 + ) + await fixture.transport.setShell(fresh, host: "two.example") + try await server.snapshot(fresh, host: "two.example") + try await receipts.waitForShells("two", count: 2) + await oldHTTP.release() + try await receipts.waitForHTTP("two", count: 1) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Replacement socket") + #expect(receipts.sequences["two"] == [100, 2]) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + await fixture.client.disconnect() + } + + @Test("Unknown live events trigger one immediate HTTP repair and recover", .timeLimit(.minutes(1))) + func unknownEventRepairsWithoutPollingDelay() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + await server.waitForSubscriptions(host: "two.example", count: 1) + // Drain the initial repair before making the stream authoritative. + try await receipts.waitForHTTP("two", count: 1) + let live = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Live", snapshotSequence: 10) + try await server.snapshot(live, host: "two.example") + try await receipts.waitForShells("two", count: 1) + let initialReadCount = await fixture.transport.shellReadCount(host: "two.example") + let initialReceiptCount = receipts.httpCount("two") + let held = PassiveRequestGate() + let repaired = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "HTTP repaired", snapshotSequence: 11) + await fixture.transport.setShell(repaired, host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: held) + try await server.emit([.object(["kind": .string("future-shell-event")])], host: "two.example") + await held.waitUntilEntered() + await server.waitForSubscriptions(host: "two.example", count: 2) + let readCount = await fixture.transport.shellReadCount(host: "two.example") + #expect(readCount == initialReadCount + 1) + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: threadID, title: "HTTP repaired") + probe.start() + await held.release() + try await receipts.waitForHTTP("two", count: initialReceiptCount + 1) + await probe.waitUntilObserved() + try await server.snapshot(repaired, host: "two.example") + try await receipts.waitForShells("two", count: 2) + #expect(await fixture.transport.shellReadCount(host: "two.example") == initialReadCount + 1) + await fixture.client.disconnect() + } + + @Test("Background cancels passive work and foreground replaces subscriptions", .timeLimit(.minutes(1))) + func backgroundAndRemovalCancelOwnedSubscriptions() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let publication = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: { try await publication.sleep(for: .zero) }, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Cached live peer", snapshotSequence: 3 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + _ = await publication.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + fixture.client.suspendForBackground() + await oldWorker.value + await server.waitForInterrupts(host: "two.example", count: 1) + #expect(fixture.client.aggregateRefreshWorkers.isEmpty) + #expect(oldWorker.isCancelled) + await fixture.client.resumeAfterBackground(reconnect: false) + await server.waitForSubscriptions(host: "two.example", count: 2) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + let nextWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + try await fixture.client.removeEnvironment(id: "two") + await nextWorker.value + // Successful removal closes the transport before cancelling the worker. + // There is no live socket on which to deliver another Interrupt frame. + #expect(await server.latestConnectionIsClosed(host: "two.example")) + #expect(nextWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(!snapshot.environments.contains { $0.id == "two" }) + await fixture.client.disconnect() + } + @Test("Repeated snapshot then stream end waits twenty seconds between subscriptions", .timeLimit(.minutes(1))) + func repeatedStreamEndsBackOffWhileHTTPRepairs() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, interval in try await retries.sleep(for: interval) }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + for attempt in 1...3 { + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Snapshot \(attempt)", snapshotSequence: 10 + attempt + ), host: "two.example") + try await receipts.waitForShells("two", count: attempt) + try await server.finish(host: "two.example") + let cadence = await retries.waitUntilRequested(count: attempt) + #expect(cadence == .seconds(20)) + try await receipts.waitForHTTP("two", count: attempt + 1) + #expect(await server.subscriptionCount(host: "two.example") == attempt) + if attempt < 3 { + await retries.resume() + await server.waitForSubscriptions(host: "two.example", count: attempt + 1) + } + } + await fixture.client.disconnect() + } + + @Test("Selecting a passive environment transfers shell ownership", .timeLimit(.minutes(1))) + func activeSwitchTransfersShellOwnership() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Selected peer", snapshotSequence: 10 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + // A same-socket HTTP refresh must retain the epoch association even + // when its complete snapshot is identical to the last socket snapshot. + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Selected peer", snapshotSequence: 10 + ), host: "two.example") + let readsBeforeRename = await fixture.transport.shellReadCount(host: "two.example") + try await fixture.client.renameThread( + id: FeatureScopedID.thread(environmentID: "two", wireID: "thread-two"), title: "Selected peer" + ) + #expect(await fixture.transport.shellReadCount(host: "two.example") == readsBeforeRename + 1) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Remote work", snapshotSequence: 1 + ), host: "two.example") + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let adoptedHTTPCount = receipts.httpCount("two") + 1 + _ = try await fixture.runtime.activate(id: "two") + let selected = try await fixture.client.initialSnapshot() + await oldWorker.value + await server.waitForSubscriptions(host: "two.example", count: 2) + try await receipts.waitForHTTP("two", count: adoptedHTTPCount) + #expect(oldWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + #expect(selected.environments.first { $0.isActive }?.id == "two") + #expect(selected.threads.first { $0.environmentID == "two" }?.title == "Selected peer") + await server.waitForSubscriptions(host: "one.example", count: 2) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "New passive peer", snapshotSequence: 2 + ), host: "one.example") + try await receipts.waitForShells("one", count: 2) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Selected peer") + #expect(snapshot.threads.first { $0.environmentID == "one" }?.title == "New passive peer") + + // Retaining the adopted socket's sequence must not pin a genuinely + // replacement socket to that old high watermark after a server restart. + let restarted = multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Restarted active peer", snapshotSequence: 2 + ) + await fixture.transport.setShell(restarted, host: "two.example") + await server.closeLatest(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 3) + try await server.snapshot(restarted, host: "two.example") + let recovered = try await recorder.wait { + $0.connection.state == .connected + && $0.threads.first { $0.environmentID == "two" }?.title == "Restarted active peer" + } + #expect(recovered.threads.first { $0.environmentID == "two" }?.title == "Restarted active peer") + await fixture.client.disconnect() + } + + @Test("HTTP begun before an unknown same-socket event cannot repair the gap", .timeLimit(.minutes(1))) + func sameSocketGapRejectsPreGapHTTP() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, interval in try await retries.sleep(for: interval) }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let oldRead = PassiveRequestGate() + let freshRead = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Pre-gap stale HTTP", snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: oldRead) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await oldRead.waitUntilEntered() + try await server.emit([.object(["kind": .string("unknown-before-baseline")])], host: "two.example") + _ = await retries.waitUntilRequested(count: 1) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Post-gap authoritative HTTP", snapshotSequence: 2 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: freshRead) + await oldRead.release() + try await receipts.waitForHTTP("two", count: 1) + await freshRead.waitUntilEntered() + #expect(await fixture.transport.shellReadCount(host: "two.example") == 2) + // If the old 999 was installed, this independent sequence-2 read + // cannot undo it and exposes the poisoned cache deterministically. + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Post-gap authoritative HTTP") + await freshRead.release() + try await receipts.waitForHTTP("two", count: 2) + await fixture.client.disconnect() + } + + @Test("An authoritative peer snapshot removes its selected detail subscription", .timeLimit(.minutes(1))) + func fullSnapshotRemovalStopsSelectedPeerDetail() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + // HTTP completion can mean a rejected read after the socket epoch + // changed. Establish an applied baseline before using its detail route. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Before removal", snapshotSequence: 9 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + _ = try await fixture.client.loadThread(id: threadID) + await server.waitForSubscriptions(host: "two.example#detail", count: 1) + let baseline = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Removed", snapshotSequence: 10) + try await server.snapshot(OrchestrationShellSnapshot( + snapshotSequence: 10, projects: baseline.projects, threads: [], updatedAt: baseline.updatedAt + ), host: "two.example") + try await receipts.waitForShells("two", count: 2) + await server.waitForInterrupts(host: "two.example#detail", count: 1) + #expect(await server.subscriptionCount(host: "one.example") == 1) + await fixture.client.disconnect() + } + +} + +@MainActor +private final class PassiveLiveReceipts { + private(set) var sequences: [String: [Int]] = [:] + private var counts: [String: Int] = [:] + private var waiters: [UUID: (String, Int, CheckedContinuation)] = [:] + + func record(_ receipt: NativePassiveShellReceipt) { + let key: String + switch receipt { + case let .shellApplied(id, sequence): + sequences[id, default: []].append(sequence) + key = "shell:" + id + case let .httpFinished(id): key = "http:" + id + case let .configurationApplied(id): key = "config:" + id + case let .archiveFinished(id): key = "archive:" + id + } + counts[key, default: 0] += 1 + let ready = waiters.filter { counts[$0.value.0, default: 0] >= $0.value.1 } + for (id, waiter) in ready { + waiters[id] = nil + waiter.2.resume() + } + } + + func httpCount(_ id: String) -> Int { counts["http:" + id, default: 0] } + func waitForShells(_ id: String, count: Int) async throws { try await wait("shell:" + id, count: count) } + func waitForHTTP(_ id: String, count: Int) async throws { try await wait("http:" + id, count: count) } + func waitForConfiguration(_ id: String) async throws { try await wait("config:" + id, count: 1) } + func waitForArchive(_ id: String) async throws { try await wait("archive:" + id, count: 1) } + + private func wait(_ key: String, count: Int) async throws { + try Task.checkCancellation() + guard counts[key, default: 0] < count else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (key, count, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.2.resume(throwing: CancellationError()) + } + } + } +} + +private actor PassiveClockBank { + private var clocks: [String: ControllableAggregateRefreshSleep] = [:] + + private func clock(_ id: String) -> ControllableAggregateRefreshSleep { + if let clock = clocks[id] { return clock } + let clock = ControllableAggregateRefreshSleep() + clocks[id] = clock + return clock + } + + func sleep(id: String, interval: Duration) async throws { + try await clock(id).sleep(for: interval) + } + + func resumeInitial(_ id: String) async { + let clock = clock(id) + _ = await clock.waitUntilRequested(count: 1) + await clock.resume() + } +} + +private actor PassiveLiveServer: WebSocketConnecting { + struct Subscription: Sendable { + let connection: PassiveLiveConnection + let requestID: Int + let payload: JSONValue? + } + private let configs = MultiEnvironmentConfigurationServer() + private var archiveGates: [String: PassiveRequestGate] = [:] + func holdArchive(host: String, gate: PassiveRequestGate) { archiveGates[host] = gate } + private var subscriptions: [String: [Subscription]] = [:] + private var interrupts: [String: Int] = [:] + private var waiters: [(String, Int, Bool, CheckedContinuation)] = [] + + func connect(to url: URL) -> any WebSocketConnection { + PassiveLiveConnection(host: url.host ?? "", server: self) + } + + func request(_ value: JSONValue, host: String, connection: PassiveLiveConnection) async throws -> JSONValue? { + if let tag = value["tag"]?.stringValue, + tag == RPCMethod.subscribeShell.rawValue || tag == RPCMethod.subscribeThread.rawValue, + case let .number(id)? = value["id"] { + let key = tag == RPCMethod.subscribeThread.rawValue ? host + "#detail" : host + subscriptions[key, default: []].append(Subscription(connection: connection, requestID: Int(id), payload: value["payload"])) + resumeWaiters() + return nil + } + if value["_tag"]?.stringValue == "Interrupt", case let .number(id)? = value["requestId"] { + for key in [host, host + "#detail"] where subscriptions[key, default: []].contains(where: { + $0.connection === connection && $0.requestID == Int(id) + }) { + interrupts[key, default: 0] += 1 + } + resumeWaiters() + } + if value["_tag"]?.stringValue == "Ping" { return .object(["_tag": .string("Pong")]) } + if value["tag"]?.stringValue == RPCMethod.dispatchCommand.rawValue, + ["thread.meta.update", "thread.unarchive"].contains(value["payload"]?["type"]?.stringValue ?? ""), + case let .number(id)? = value["id"] { + return .object([ + "_tag": .string("Exit"), "requestId": .number(id), + "exit": .object(["_tag": .string("Success"), "value": try JSONValue.encode(DispatchResult(sequence: 10))]), + ]) + } + if value["tag"]?.stringValue == RPCMethod.getArchivedShellSnapshot.rawValue, + let gate = archiveGates.removeValue(forKey: host) { await gate.enter() } + return try await configs.response(to: value, host: host) + } + + func lastPayload(host: String) -> JSONValue? { subscriptions[host]?.last?.payload } + + func waitForSubscriptions(host: String, count: Int) async { await wait(host: host, count: count, interrupt: false) } + func waitForInterrupts(host: String, count: Int) async { await wait(host: host, count: count, interrupt: true) } + + private func count(_ host: String, interrupt: Bool) -> Int { + interrupt ? interrupts[host, default: 0] : subscriptions[host, default: []].count + } + + private func wait(host: String, count: Int, interrupt: Bool) async { + guard self.count(host, interrupt: interrupt) < count else { return } + await withCheckedContinuation { waiters.append((host, count, interrupt, $0)) } + } + + private func resumeWaiters() { + let ready = waiters.filter { count($0.0, interrupt: $0.2) >= $0.1 } + waiters.removeAll { count($0.0, interrupt: $0.2) >= $0.1 } + ready.forEach { $0.3.resume() } + } + + func emit(_ values: [JSONValue], host: String) async throws { + guard let subscription = subscriptions[host]?.last else { throw URLError(.notConnectedToInternet) } + try await subscription.connection.enqueue(.object([ + "_tag": .string("Chunk"), "requestId": .number(Double(subscription.requestID)), "values": .array(values), + ])) + } + + func snapshot(_ shell: OrchestrationShellSnapshot, host: String) async throws { + try await emit([.object(["kind": .string("snapshot"), "snapshot": try JSONValue.encode(shell)])], host: host) + } + + func upsert(_ thread: OrchestrationThreadShell, sequence: Int, host: String) async throws { + try await emit([.object([ + "kind": .string("thread-upserted"), "sequence": .number(Double(sequence)), "thread": try JSONValue.encode(thread), + ])], host: host) + } + + func finish(host: String) async throws { + guard let subscription = subscriptions[host]?.last else { throw URLError(.notConnectedToInternet) } + try await subscription.connection.enqueue(.object([ + "_tag": .string("Exit"), "requestId": .number(Double(subscription.requestID)), + "exit": .object(["_tag": .string("Success"), "value": .null]), + ])) + } + + func subscriptionCount(host: String) -> Int { subscriptions[host, default: []].count } + + func latestConnectionIsClosed(host: String) async -> Bool { + await subscriptions[host]?.last?.connection.isClosed ?? false + } + + func closeLatest(host: String) async { await subscriptions[host]?.last?.connection.close() } +} + +private actor PassiveLiveConnection: WebSocketConnection { + var isClosed: Bool { closed } + private let host: String + private let server: PassiveLiveServer + private var responses: [Data] = [] + private var receiver: CheckedContinuation? + private var closed = false + + init(host: String, server: PassiveLiveServer) { self.host = host; self.server = server } + + func send(_ data: Data) async throws { + guard !closed else { throw URLError(.networkConnectionLost) } + let value = try JSONDecoder.t3.decode(JSONValue.self, from: data) + if let response = try await server.request(value, host: host, connection: self) { try enqueue(response) } + } + + func enqueue(_ value: JSONValue) throws { + let data = try JSONEncoder.t3.encode(value) + if let receiver { self.receiver = nil; receiver.resume(returning: data) } + else { responses.append(data) } + } + + func receive() async throws -> Data { + guard !closed else { throw URLError(.networkConnectionLost) } + if !responses.isEmpty { return responses.removeFirst() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + closed = true + receiver?.resume(throwing: URLError(.networkConnectionLost)) + receiver = nil + } +} + +@Suite("Native incremental bootstrap") +@MainActor +struct NativeIncrementalBootstrapTests { + @Test("Environment-store failures remain visible to the root model") + func environmentStoreFailureIsNotCancellation() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let catalog = directory.appendingPathComponent("environments.json") + try Data("invalid catalog".utf8).write(to: catalog) + let runtime = EnvironmentRuntime(environmentStore: EnvironmentStore(fileURL: catalog), + credentialStore: InMemoryCredentialStore()) + let client = NativeFeatureClient(runtime: runtime) + do { + _ = try await client.initialSnapshot() + Issue.record("The invalid catalog unexpectedly loaded") + } catch { + #expect(error is DecodingError) + #expect(!(error is CancellationError)) + } + let model = FeatureRootModel(client: client, + outboxStore: FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")), + draftStore: FeatureComposerDraftStore(fileURL: directory.appendingPathComponent("drafts.json"))) + await model.reload() + #expect(model.errorMessage?.isEmpty == false) + await model.disconnect() + } + + @Test("Pairing resets the previous environment's hydration and archive lifetime", .timeLimit(.minutes(1)), arguments: [false, true]) + func pairingReplacesHydrationLifetime(previousHydrated: Bool) async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + if !previousHydrated { await fixture.transport.holdNextShell(host: "one.example", gate: held) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + do { + if previousHydrated { try await receipts.waitForArchive("one") } + else { try await held.waitUntilEnteredCancellable() } + await server.waitForSubscriptions(host: "two.example", count: 1) + try await fixture.client.pair(endpoint: "https://two.example", token: "fixture-bootstrap") + await server.waitForSubscriptions(host: "two.example", count: 2) + let shell = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "New active hydration", snapshotSequence: 20) + await fixture.transport.setShell(shell, host: "two.example") + try await server.upsert(shell.threads[0], sequence: 20, host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "New active hydration" } } + try await receipts.waitForArchive("two") + if !previousHydrated { #expect(await held.isHeld) } + } catch { + await held.release() + await fixture.client.disconnect() + throw error + } + await held.release() + await fixture.client.disconnect() + } + + @Test("Foreground restores bootstrap before a client has been adopted", .timeLimit(.minutes(1))) + func foregroundRestartsUnadoptedClient() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let recorder = BootstrapSnapshotRecorder( + seed: FeatureSnapshot(connection: .init(state: .disconnected)), events: fixture.client.events() + ) + defer { recorder.stop() } + fixture.client.suspendForBackground() + await fixture.client.resumeAfterBackground(reconnect: false) + let restored = try await recorder.wait { $0.threads.contains { $0.environmentID == "one" } } + #expect(restored.environments.contains { $0.id == "one" }) + await fixture.client.disconnect() + } + + @Test("An archive refresh reads the current shell after the archive RPC settles", .timeLimit(.minutes(1))) + func archiveRefreshReadsShellAfterArchive() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + try await receipts.waitForArchive("one") + let archive = PassiveRequestGate() + await server.holdArchive(host: "one.example", gate: archive) + let refresh = Task { try await fixture.client.setThreadArchived(id: "thread-one", archived: false) } + do { + try await archive.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "Current after archive", snapshotSequence: 20 + ), host: "one.example") + await archive.release() + try await refresh.value + #expect(recorder.history.last?.threads.contains { $0.title == "Current after archive" } == true) + } catch { + await archive.release() + refresh.cancel() + _ = try? await refresh.value + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("Metadata returns while active HTTP and catalogue are held; a healthy peer publishes", .timeLimit(.minutes(1))) + func heldActiveDoesNotBlockHealthyPeer() async throws { + let connector = BootstrapHeldConnector() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, webSocketConnector: connector, + rpcConnectionWaitTimeout: .seconds(60), fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let active = PassiveRequestGate() + let passive = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: active) + await fixture.transport.holdNextShell(host: "two.example", gate: passive) + let seed = try await fixture.client.initialSnapshot() + #expect(Set(seed.environments.map(\.id)) == ["one", "two", "three"]) + #expect(seed.threads.isEmpty) + #expect(seed.environments.allSatisfy { $0.connectionState != .connected }) + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + try await active.waitUntilEnteredCancellable() + try await passive.waitUntilEnteredCancellable() + try await connector.gate.waitUntilEnteredCancellable() + let healthy = try await recorder.wait { snapshot in + snapshot.threads.contains { $0.environmentID == "three" } + && snapshot.environments.first { $0.id == "three" }?.connectionState == .connected + } + #expect(await active.isHeld) + #expect(await passive.isHeld) + #expect(await connector.gate.isHeld) + #expect(healthy.environments.first { $0.id == "one" }?.connectionState != .connected) + await active.release() + await passive.release() + await connector.gate.release() + await fixture.client.disconnect() + } + + @Test("Catalog and synchronized markers cannot authorize an empty outbox view", .timeLimit(.minutes(1))) + func authorityAndRowsPublishTogether() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: held) + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + let submission = FeatureQueuedSubmission( + environmentID: "one", identity: .init(threadID: "thread-one"), + threadID: FeatureScopedID.thread(environmentID: "one", wireID: "thread-one"), + text: "Queued before hydration", selection: nil, runtimeMode: .fullAccess, + interactionMode: .standard, attachments: [] + ) + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: seed) == .wait) + try await held.waitUntilEnteredCancellable() + await server.waitForSubscriptions(host: "one.example", count: 1) + try await server.emit([.object(["kind": .string("synchronized")])], host: "one.example") + try await receipts.waitForConfiguration("one") + let shell = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Authoritative", snapshotSequence: 10) + try await server.snapshot(shell, host: "one.example") + let ready = try await recorder.wait { $0.threads.contains { $0.title == "Authoritative" } } + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: ready) == .send) + for snapshot in recorder.history { + let connected = snapshot.environments.first { $0.id == "one" }?.connectionState == .connected + #expect(!connected || snapshot.threads.contains { $0.id == submission.threadID }) + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: snapshot) != .discard) + } + #expect(await held.isHeld) + await held.release() + await fixture.client.disconnect() + } + + @Test("Same-client reload rejects a late old HTTP response and retains cached rows", .timeLimit(.minutes(1))) + func reloadRetainsCacheAndRejectsOldRead() async throws { + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { $0.threads.count == 2 } + let old = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Stale old request", snapshotSequence: 999), host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: old) + let cached = try await fixture.client.initialSnapshot() + #expect(cached.threads.count == 2) + #expect(cached.environments.allSatisfy { $0.connectionState != .connected }) + try await old.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Current reload", snapshotSequence: 2), host: "one.example") + _ = try await fixture.client.initialSnapshot() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Current reload" } } + await old.release() + try await receipts.waitForHTTP("one", count: 3) + #expect(!recorder.history.contains { $0.threads.contains { $0.title == "Stale old request" } }) + await fixture.client.disconnect() + } + + @Test("A current HTTP command refresh keeps the live header reconnecting while its rows are ready", .timeLimit(.minutes(1))) + func currentHTTPCommandPreservesReconnectingHeader() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + do { + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "HTTP command result", snapshotSequence: 2 + ), host: "one.example") + try await fixture.client.renameThread(id: "thread-one", title: "HTTP command result") + let updated = try await recorder.wait { $0.threads.contains { $0.title == "HTTP command result" } } + #expect(updated.connection.state == .reconnecting) + #expect(updated.environments.first { $0.id == "one" }?.connectionState == .connected) + #expect(await fixture.transport.dispatchRecords().count == 1) + } catch { + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("A public rename's old optional shell cannot authorize a replacement bootstrap", .timeLimit(.minutes(1))) + func oldCommandRefreshCannotAuthorizeReload() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + let oldRead = PassiveRequestGate() + let replacementRead = PassiveRequestGate() + let stale = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Old command shell", snapshotSequence: 999) + await fixture.transport.setShell(stale, host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: oldRead) + let command = Task { try await fixture.client.renameThread(id: "thread-one", title: "Rename once") } + do { + try await oldRead.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Replacement bootstrap", snapshotSequence: 2), host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: replacementRead) + let seed = try await fixture.client.initialSnapshot() + recorder.record(seed) + try await replacementRead.waitUntilEnteredCancellable() + await oldRead.release() + try await command.value + #expect(recorder.history.last?.environments.first { $0.id == "one" }?.connectionState != .connected) + #expect(!recorder.history.contains { $0.threads.contains { $0.title == "Old command shell" } }) + await replacementRead.release() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Replacement bootstrap" } } + } catch { + await oldRead.release() + await replacementRead.release() + command.cancel() + _ = try? await command.value + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("A cancelled old archive cannot authorize a same-client reload", .timeLimit(.minutes(1))) + func oldArchiveCannotAuthorizeReload() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let archive = PassiveRequestGate() + await server.holdArchive(host: "one.example", gate: archive) + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { $0.threads.contains { $0.environmentID == "one" } } + try await archive.waitUntilEnteredCancellable() + let heldShell = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: heldShell) + let reload = try await fixture.client.initialSnapshot() + recorder.record(reload) + let startIndex = recorder.history.count - 1 + try await heldShell.waitUntilEnteredCancellable() + await archive.release() + try await receipts.waitForArchive("one") + for snapshot in recorder.history.dropFirst(startIndex) { + #expect(snapshot.environments.first { $0.id == "one" }?.connectionState != .connected) + } + #expect(await heldShell.isHeld) + await heldShell.release() + await fixture.client.disconnect() + } + + @Test("Unknown active events repair once, then a replacement stream delivers deltas", .timeLimit(.minutes(1))) + func unknownActiveStreamRepairsAndResubscribes() async throws { + let server = PassiveLiveServer() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregateStreamRetrySleep: { id, interval in + if id == "one" { try await retries.sleep(for: interval) } + }, aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + await server.waitForSubscriptions(host: "one.example", count: 1) + let baseline = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active live baseline", snapshotSequence: 10) + try await server.snapshot(baseline, host: "one.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active live baseline" } } + let held = PassiveRequestGate() + let repaired = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active HTTP repaired", snapshotSequence: 20) + await fixture.transport.setShell(repaired, host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: held) + try await server.emit([.object(["kind": .string("unknown-active-event")])], host: "one.example") + try await held.waitUntilEnteredCancellable() + #expect(await retries.waitUntilRequested(count: 1) == .seconds(20)) + // These are delivered to the discarded subscription and cannot poison repair. + for sequence in 21...23 { try await server.upsert(baseline.threads[0], sequence: sequence, host: "one.example") } + await held.release() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active HTTP repaired" } } + #expect(await server.subscriptionCount(host: "one.example") == 1) + await retries.resume() + await server.waitForSubscriptions(host: "one.example", count: 2) + try await server.snapshot(repaired, host: "one.example") + let delta = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active delta recovered", snapshotSequence: 24) + try await server.upsert(delta.threads[0], sequence: 24, host: "one.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active delta recovered" } } + await fixture.client.disconnect() + } + + @Test("A failed initial socket attempt cannot invalidate its independent HTTP read", .timeLimit(.minutes(1))) + func failedSocketDoesNotStarveHTTP() async throws { + let connector = FailedBootstrapAttemptConnector() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: connector, fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: held) + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + do { + try await held.waitUntilEnteredCancellable() + try await connector.waitForFailure() + await held.release() + let ready = try await recorder.wait { + $0.environments.first { $0.id == "one" }?.connectionState == .connected + } + #expect(ready.threads.contains { $0.environmentID == "one" }) + #expect(await fixture.transport.shellReadCount(host: "one.example") == 1) + } catch { + await held.release() + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } +} + +/// One collector per fixture; any number of predicates observe the same snapshots. +@MainActor +final class BootstrapSnapshotRecorder { + private(set) var history: [FeatureSnapshot] + private var task: Task? + private var waiters: [UUID: (@MainActor (FeatureSnapshot) -> Bool, CheckedContinuation)] = [:] + private var finished = false + + init(seed: FeatureSnapshot, events: AsyncStream) { + history = [seed] + task = Task { [weak self] in + for await event in events { + guard !Task.isCancelled else { break } + guard let self else { break } + switch event { + case let .snapshot(snapshot): self.record(snapshot) + case let .thread(thread): + guard var snapshot = self.history.last else { continue } + if let index = snapshot.threads.firstIndex(where: { $0.id == thread.id }) { + snapshot.threads[index] = thread + } else { snapshot.threads.append(thread) } + self.record(snapshot) + case let .threadRemoved(id): + guard var snapshot = self.history.last else { continue } + snapshot.threads.removeAll { $0.id == id } + self.record(snapshot) + default: break + } + } + self?.stop() + } + } + + func record(_ snapshot: FeatureSnapshot) { + history.append(snapshot) + let ready = waiters.filter { $0.value.0(snapshot) } + for (id, waiter) in ready { + waiters[id] = nil + waiter.1.resume(returning: snapshot) + } + } + + func wait(_ predicate: @escaping @MainActor (FeatureSnapshot) -> Bool) async throws -> FeatureSnapshot { + try Task.checkCancellation() + if let snapshot = history.last, predicate(snapshot) { return snapshot } + guard !finished else { throw CancellationError() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (predicate, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + } + } + + func stop() { + finished = true + task?.cancel() + task = nil + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.1.resume(throwing: CancellationError()) } + } + + deinit { task?.cancel() } +} + +private actor BootstrapHeldConnector: WebSocketConnecting { + let gate = PassiveRequestGate() + func connect(to url: URL) async throws -> any WebSocketConnection { + if url.host == "one.example" { await gate.enter() } + throw URLError(.cannotConnectToHost) + } +} + +@Suite("Native bootstrap durable outbox") +@MainActor +struct NativeBootstrapRootOutboxTests { + @Test("A persisted follow-up waits through another peer's update and sends once after its owner hydrates", .timeLimit(.minutes(1))) + func restoredFollowUpWaitsForOwningShell() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let outbox = FeatureOutboxStore(fileURL: fixture.directory.appendingPathComponent("root-outbox.json")) + let drafts = FeatureComposerDraftStore(fileURL: fixture.directory.appendingPathComponent("root-drafts.json")) + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + let identity = FeatureSubmissionIdentity(threadID: "thread-two", commandID: "bootstrap-root-command", messageID: "bootstrap-root-message") + let submission = FeatureQueuedSubmission( + environmentID: "two", identity: identity, threadID: threadID, + text: "Persisted before the peer loaded", selection: nil, + runtimeMode: .fullAccess, interactionMode: .standard, attachments: [] + ) + try await outbox.enqueue(submission) + let heldShell = PassiveRequestGate() + let heldDispatch = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "two.example", gate: heldShell) + await fixture.transport.holdNextDispatch(host: "two.example", gate: heldDispatch) + let model = FeatureRootModel(client: fixture.client, outboxStore: outbox, draftStore: drafts) + let run = Task { await model.start() } + do { + try await heldShell.waitUntilEnteredCancellable() + let healthy = RootObservationWaiter { + !model.isLoading && model.snapshot.threads.contains { $0.environmentID == "one" } + } + try await healthy.wait() + #expect(model.snapshot.environments.first { $0.id == "two" }?.connectionState != .connected) + #expect(try await outbox.submissions().map(\.id) == [submission.id]) + #expect(await fixture.transport.dispatchRecords().isEmpty) + #expect(await heldShell.isHeld) + + await heldShell.release() + try await heldDispatch.waitUntilEnteredCancellable() + // Load the real detail while acceptance is held; the root attaches its + // restored queued message, giving an observable completion after removal. + _ = await model.detail(for: threadID) + #expect(model.details[threadID]?.messages.first { $0.id == identity.messageID }?.state == .queued) + let completed = RootObservationWaiter { + model.details[threadID]?.messages.first { $0.id == identity.messageID }?.state == .complete + } + await heldDispatch.release() + try await completed.wait() + // Message acceptance may precede durable deletion. Wait for the actual + // owned outbox file change, independent of that presentation ordering. + try await OwnedOutboxRemovalWaiter(fileURL: fixture.directory.appendingPathComponent("root-outbox.json")).wait() + #expect(try await outbox.submissions().isEmpty) + let commands = await fixture.transport.dispatchRecords() + #expect(commands.count == 1) + #expect(commands.first?.host == "two.example") + #expect(commands.first?.command["commandId"]?.stringValue == identity.commandID) + #expect(commands.first?.command["threadId"]?.stringValue == identity.threadID) + #expect(commands.first?.command["message"]?["messageId"]?.stringValue == identity.messageID) + #expect(model.errorMessage == nil) + } catch { + await heldShell.release() + await heldDispatch.release() + run.cancel() + await fixture.client.disconnect() + await run.value + throw error + } + await heldShell.release() + await heldDispatch.release() + run.cancel() + await fixture.client.disconnect() + await run.value + } +} + +/// Observe the root's applied state without taking a second client event iterator. +@MainActor +private final class RootObservationWaiter { + private let predicate: @MainActor () -> Bool + private var continuation: CheckedContinuation? + + init(_ predicate: @escaping @MainActor () -> Bool) { self.predicate = predicate } + + func wait() async throws { + try Task.checkCancellation() + if predicate() { return } + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor [weak self] in + self?.finish(.failure(CancellationError())) + } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { predicate() } onChange: { [weak self] in + Task { @MainActor in self?.observe() } + } + if ready { finish(.success(())) } + } + + private func finish(_ result: Result) { + let pending = continuation + continuation = nil + pending?.resume(with: result) + } +} + + +/// The test watches only its owned directory. Atomic store writes replace the +/// file, so observing the directory avoids retaining an obsolete file inode. +@MainActor +private final class OwnedOutboxRemovalWaiter { + private let fileURL: URL + private var source: DispatchSourceFileSystemObject? + private var continuation: CheckedContinuation? + + init(fileURL: URL) { self.fileURL = fileURL } + + func wait() async throws { + try Task.checkCancellation() + let descriptor = open(fileURL.deletingLastPathComponent().path, O_EVTONLY) + guard descriptor >= 0 else { throw POSIXError(.EIO) } + let source = DispatchSource.makeFileSystemObjectSource( + fileDescriptor: descriptor, eventMask: [.write, .rename], queue: .global() + ) + self.source = source + source.setCancelHandler { close(descriptor) } + source.setEventHandler { [weak self] in + Task { @MainActor in self?.check() } + } + source.resume() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + check() + } + } onCancel: { + Task { @MainActor [weak self] in self?.finish(.failure(CancellationError())) } + } + } + + private func check() { + guard continuation != nil else { return } + do { + let document = try JSONSerialization.jsonObject(with: Data(contentsOf: fileURL)) as? [String: Any] + guard let submissions = document?["submissions"] as? [Any] else { throw URLError(.cannotParseResponse) } + if submissions.isEmpty { finish(.success(())) } + } catch { finish(.failure(error)) } + } + + private func finish(_ result: Result) { + let pending = continuation + continuation = nil + source?.cancel() + source = nil + pending?.resume(with: result) + } +} + + +/// Observe the failed connector attempt itself: the RPC subscription deliberately +/// keeps waiting for its first usable socket instead of failing with a UI header. +private actor FailedBootstrapAttemptConnector: WebSocketConnecting { + private let failures = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + + func connect(to url: URL) async throws -> any WebSocketConnection { + if url.host == "one.example" { failures.continuation.yield(()) } + throw URLError(.cannotConnectToHost) + } + + func waitForFailure() async throws { + try Task.checkCancellation() + var iterator = failures.stream.makeAsyncIterator() + guard await iterator.next() != nil else { throw CancellationError() } + try Task.checkCancellation() + } +} + +private actor SilentShellClock { + private(set) var intervals: [Duration] = [] + private let cancelled = PassiveRequestGate() + private var continuation: CheckedContinuation? + private var requestWaiters: [UUID: (Int, CheckedContinuation)] = [:] + + func sleep(_ interval: Duration) async throws { + intervals.append(interval) + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else { + self.continuation = continuation + let ready = requestWaiters.filter { intervals.count >= $0.value.0 } + for (id, waiter) in ready { + requestWaiters[id] = nil + waiter.1.resume() + } + } + } + } onCancel: { Task { await self.cancel() } } + } + func waitForRequest(_ count: Int = 1) async throws { + try Task.checkCancellation() + guard intervals.count < count else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { requestWaiters[id] = (count, $0) } + } onCancel: { Task { await self.cancelWaiter(id) } } + } + private func cancelWaiter(_ id: UUID) { + requestWaiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + func advance() { continuation?.resume(); continuation = nil } + func waitForCancellation() async throws { try await cancelled.waitUntilEnteredCancellable() } + func cancel() async { + continuation?.resume(throwing: CancellationError()) + continuation = nil + await cancelled.release() + await cancelled.enter() + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index 0ce3e6a1a4f7..1046e7b45430 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -1,4 +1,5 @@ import Foundation +import Observation import XCTest @testable import T3Code @@ -71,6 +72,459 @@ final class NativeRetryIdentityTests: XCTestCase { } } + func testOlderToolPageNeverInheritsCurrentSessionLiveness() async throws { + var capture = try JSONSerialization.jsonObject(with: JSONEncoder.t3.encode( + NativeStopProjectionFixtures.snapshot("approval"))) as! [String: Any] + var before = capture["before"] as! [String: Any] + var detail = before["detail"] as! [String: Any] + let sequence = detail["snapshotSequence"] as! Int + var thread = detail["thread"] as! [String: Any] + func tool(_ id: String, turn: String, at time: String) -> [String: Any] { + ["id": id, "tone": "info", "kind": "tool.started", "summary": id, + "payload": ["toolCallId": id, "title": id], "turnId": turn, "createdAt": time] + } + thread["messages"] = [["id": "new-text", "role": "assistant", "text": "Current turn", + "turnId": "outcome-proof-A", "streaming": false, + "createdAt": "2026-01-01T00:00:03Z", "updatedAt": "2026-01-01T00:00:03Z"]] + thread["activities"] = [tool("new-tool", turn: "outcome-proof-A", at: "2026-01-01T00:00:04Z")] + detail["thread"] = thread + detail["page"] = ["beforeCursor": "older", "hasMore": true, "snapshotSequence": sequence, "threadSequence": sequence] + before["detail"] = detail + capture["before"] = before + var older = before + thread["messages"] = [] + thread["activities"] = [tool("old-tool", turn: "old-turn", at: "2026-01-01T00:00:01Z")] + detail["thread"] = thread + detail["page"] = ["hasMore": false, "snapshotSequence": sequence, "threadSequence": sequence] + older["detail"] = detail + capture["older"] = older + let fixture = try await AcceptedSendFixture.make(captured: + JSONDecoder.t3.decode(JSONValue.self, from: JSONSerialization.data(withJSONObject: capture)), supportsPagination: true) + addTeardownBlock { await fixture.cleanUp() } + XCTAssertEqual(fixture.model.details[fixture.threadID]?.page?.hasMore, true) + await fixture.transport.useCapturedPhase("older") + let result = try await fixture.client.loadEarlierThreadTurns(id: fixture.threadID) + let rows = try XCTUnwrap(result?.messages) + XCTAssertEqual(rows.map(\.id), ["work-log-old-tool", "new-text", "work-log-new-tool"]) + XCTAssertNil(rows.first?.activeWorkLabel) + XCTAssertEqual(rows.last?.activeWorkLabel, "new-tool") + } + + func testToolUpdatesContinueAfterInterveningText() async throws { + let base = try NativeStopProjectionFixtures.snapshot("approval") + var capture = try JSONSerialization.jsonObject(with: JSONEncoder.t3.encode(base)) as! [String: Any] + let times = ["2026-01-01T00:00:01.000Z", "2026-01-01T00:00:02.000Z", "2026-01-01T00:00:03.000Z"] + let activities: [[String: Any]] = [0, 2].map { index in + ["id": "call-event-\(index)", "tone": "info", "kind": "tool.updated", + "summary": "Read \(index)", "payload": ["toolCallId": "call-a", "title": "Read \(index)"], + "turnId": "outcome-proof-A", "sequence": index + 10, "createdAt": times[index]] + } + let message: [String: Any] = ["id": "between", "role": "assistant", "text": "Checking the result", + "turnId": "outcome-proof-A", "streaming": false, "createdAt": times[1], "updatedAt": times[1]] + var phase = capture["before"] as! [String: Any] + var detail = phase["detail"] as! [String: Any] + var thread = detail["thread"] as! [String: Any] + thread["messages"] = [message] + thread["activities"] = activities + detail["thread"] = thread + phase["detail"] = detail + capture["before"] = phase + let fixture = try await AcceptedSendFixture.make(captured: + JSONDecoder.t3.decode(JSONValue.self, from: JSONSerialization.data(withJSONObject: capture))) + addTeardownBlock { await fixture.cleanUp() } + let rows = try XCTUnwrap(fixture.model.details[fixture.threadID]?.messages) + XCTAssertEqual(rows.map(\.role), [.tool, .assistant, .tool]) + XCTAssertEqual(rows.filter { $0.role == .tool }.map(\.createdAt), + [NativeTimestampParser.parse(times[0])!, NativeTimestampParser.parse(times[2])!]) + XCTAssertEqual(rows.last?.activeWorkLabel, "Read 2") + XCTAssertNil(rows.first?.activeWorkLabel) + } + + func testCapturedStopProjectionsRetainFeedbackUntilInactiveSession() async throws { + for scenario in ["approval", "approval-starting", "input", "background-ready-null", "active-running-null"] { + let fixture = try await AcceptedSendFixture.make( + captured: NativeStopProjectionFixtures.snapshot(scenario) + ) + addTeardownBlock { await fixture.cleanUp() } + let expectedPhase: FeatureStopPhase = scenario == "background-ready-null" + ? .unconfirmed : .awaitingOutcome + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + await fixture.model.cancelTurn(threadID: fixture.threadID) + XCTAssertEqual(fixture.model.stopPhase(threadID: fixture.threadID), expectedPhase, scenario) + let next = await reads.next() + let held = try XCTUnwrap(next) + XCTAssertEqual(held.path, "/api/orchestration/shell") + await fixture.transport.useCapturedPhase("requestHeld") + await fixture.transport.release(held.id) + await CapturedStopProjectionReceipt(model: fixture.model, threadID: fixture.threadID).wait() + XCTAssertEqual(fixture.model.stopPhase(threadID: fixture.threadID), expectedPhase, scenario) + await fixture.transport.allowFollowups() + await fixture.transport.useCapturedPhase("afterAcknowledged") + _ = await fixture.model.detail(for: fixture.threadID, force: true, fresh: true) + XCTAssertEqual(fixture.model.stopPhase(threadID: fixture.threadID), expectedPhase, scenario) + await fixture.transport.useCapturedPhase("afterTerminal") + _ = await fixture.model.detail(for: fixture.threadID, force: true, fresh: true) + XCTAssertEqual(fixture.model.stopPhase(threadID: fixture.threadID), + scenario == "background-ready-null" ? .unconfirmed : nil, scenario) + } + } + + func testCapturedInputStopFailureMapsToTranscript() async throws { + let fixture = try await AcceptedSendFixture.make( + captured: NativeStopProjectionFixtures.snapshot("input-failure") + ) + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + await fixture.model.cancelTurn(threadID: fixture.threadID) + let next = await reads.next() + let held = try XCTUnwrap(next) + await fixture.transport.useCapturedPhase("requestHeld") + await fixture.transport.release(held.id) + await CapturedStopProjectionReceipt(model: fixture.model, threadID: fixture.threadID).wait() + XCTAssertEqual(fixture.model.stopPhase(threadID: fixture.threadID), .awaitingOutcome) + await fixture.transport.allowFollowups() + await fixture.transport.useCapturedPhase("afterFailure") + _ = await fixture.model.detail(for: fixture.threadID, force: true, fresh: true) + XCTAssertTrue(fixture.model.details[fixture.threadID]?.messages.contains { + $0.toolName == "provider.turn.interrupt.failed" && $0.role == .system + } == true) + XCTAssertNil(fixture.model.errorMessage) + } + + + func testExpectedStopTurnMismatchDoesNotDispatch() async throws { + let fixture = try await AcceptedSendFixture.make(turnID: "current-turn") + addTeardownBlock { await fixture.cleanUp() } + do { + try await fixture.client.cancelTurn(threadID: fixture.threadID, expectedTurnID: "older-turn") + XCTFail("Mismatched turn must be refused before dispatch") + } catch is FeatureStopTurnChangedError {} + let commands = await fixture.socket.commands + XCTAssertTrue(commands.isEmpty) + XCTAssertEqual(fixture.model.snapshot.threads.first?.latestTurnID, "current-turn") + XCTAssertEqual(fixture.model.details[fixture.threadID]?.thread.latestTurnID, "current-turn") + XCTAssertEqual(fixture.model.details[fixture.threadID]?.thread.latestTurnState, "running") + } + + func testExpectedStopTurnPayloadAndAckDoNotWaitForRefresh() async throws { + let fixture = try await AcceptedSendFixture.make(turnID: "current-turn") + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID, expectedTurnID: "current-turn") + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["turnId"]?.stringValue, "current-turn") + let read = await reads.next() + XCTAssertEqual(read?.path, "/api/orchestration/shell") + await fixture.transport.failFollowups() + } + + func testStopStatusWaitsForHTTPWithoutDispatching() async throws { + let fixture = try await AcceptedSendFixture.make(turnID: "current-turn") + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let status = Task { try await fixture.client.stopStatus(threadID: fixture.threadID) } + let nextRead = await reads.next() + let read = try XCTUnwrap(nextRead) + XCTAssertEqual(read.path, "/api/orchestration/shell") + let commands = await fixture.socket.commands + XCTAssertTrue(commands.isEmpty) + await fixture.transport.release(read.id) + let thread = try await status.value + XCTAssertEqual(thread.latestTurnID, "current-turn") + XCTAssertEqual(thread.latestTurnState, "running") + await fixture.transport.failFollowups() + } + + func testLateStopAcknowledgementCannotCancelReplacementGenerationRefresh() async throws { + let fixture = try await AcceptedSendFixture.make(includePeer: true) + addTeardownBlock { await fixture.cleanUp() } + var acknowledgements = fixture.socket.heldInterrupts.makeAsyncIterator() + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.socket.holdInterruptAcknowledgements() + let stopping = Task { try await fixture.client.cancelTurn(threadID: fixture.threadID) } + addTeardownBlock { + await fixture.socket.releaseInterruptAcknowledgements() + _ = await stopping.result + } + guard await acknowledgements.next() != nil else { throw CancellationError() } + + _ = try await fixture.runtime.activate(id: "accepted-peer") + _ = try await fixture.client.initialSnapshot() + await fixture.transport.holdFollowups(includeShell: false) + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Replacement generation", selection: nil) + let replacementRead = await reads.next() + let replacement = try XCTUnwrap(replacementRead) + XCTAssertTrue(replacement.path.hasPrefix("/api/orchestration/threads/")) + + await fixture.socket.releaseInterruptAcknowledgements() + try await stopping.value + let marker = "Replacement generation published its uniquely held response" + XCTAssertNotEqual(fixture.model.details[fixture.threadID]?.messages.last?.text, marker) + await fixture.transport.release(replacement.id, detailMessage: marker) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: marker + ).wait() + XCTAssertEqual(fixture.model.details[fixture.threadID]?.messages.last?.text, marker) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.map { $0["type"]?.stringValue }, ["thread.turn.interrupt", "thread.turn.start"]) + } + + func testRejectedStopPropagatesRemoteError() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + await fixture.transport.holdFollowups() + let before = await fixture.transport.snapshotReadCount + await fixture.socket.rejectInterruptAcknowledgements() + do { + try await fixture.client.cancelTurn(threadID: fixture.threadID) + XCTFail("Rejected interrupt must remain a command error") + } catch let error as RPCError { + guard case .remote("Stop rejected by fixture") = error else { throw error } + } + // These counters cover the command-return boundary only; they are + // not a proof that no mistakenly queued future task could read later. + let after = await fixture.transport.snapshotReadCount + let pending = await fixture.transport.heldCount + XCTAssertEqual(after, before) + XCTAssertEqual(pending, 0) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["type"]?.stringValue, "thread.turn.interrupt") + } + + func testStopDuringHeldSendDetailPreservesDetailAndShellRecovery() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Send before Stop", selection: nil) + let firstRead = await reads.next() + let detail = try XCTUnwrap(firstRead) + XCTAssertTrue(detail.path.hasPrefix("/api/orchestration/threads/")) + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let count = await fixture.transport.heldCount + XCTAssertEqual(count, 1) + let detailStillPending = await fixture.transport.isPending(detail.id) + XCTAssertTrue(detailStillPending) + await fixture.transport.release(detail.id) + let nextRead = await reads.next() + let shell = try XCTUnwrap(nextRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + await fixture.transport.release(shell.id) + let trailingRead = await reads.next() + let trailing = try XCTUnwrap(trailingRead) + XCTAssertTrue( + trailing.path.hasPrefix("/api/orchestration/threads/"), + "Stop superseded the in-flight send detail, so the send's repair reads detail again" + ) + let marker = "Send detail recovered after Stop" + await fixture.transport.release(trailing.id, detailMessage: marker) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: marker + ).wait() + let finalRead = await reads.next() + let final = try XCTUnwrap(finalRead) + XCTAssertEqual(final.path, "/api/orchestration/shell") + await fixture.client.disconnect() + let cancelled = await cancellations.next() + XCTAssertEqual(cancelled, final.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2) + } + + func testAcceptedStopCompletesBeforeOptionalShellReadFinishes() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let completed = expectation(description: "Accepted Stop completes without snapshot reads") + let stopping = Task { + await fixture.model.cancelTurn(threadID: fixture.threadID) + completed.fulfill() + } + let read = await reads.next() + XCTAssertEqual(read?.path, "/api/orchestration/shell") + await fulfillment(of: [completed], timeout: 2) + XCTAssertFalse(fixture.model.isPerformingAction) + + await fixture.transport.failFollowups() + await stopping.value + XCTAssertNil(fixture.model.errorMessage) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["type"]?.stringValue, "thread.turn.interrupt") + } + + func testSendDuringStopRefreshRetainsItsDetailRefresh() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let shellRead = await reads.next() + let shell = try XCTUnwrap(shellRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "After Stop", selection: nil) + let heldCount = await fixture.transport.heldCount + XCTAssertEqual(heldCount, 1) + await fixture.transport.release(shell.id) + let detailRead = await reads.next() + let detail = try XCTUnwrap(detailRead) + XCTAssertTrue(detail.path.hasPrefix("/api/orchestration/threads/")) + + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, detail.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2) + } + + func testDisconnectCancelsStopRefreshWithoutAnotherInterrupt() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let read = await reads.next() + let held = try XCTUnwrap(read) + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + let pending = await fixture.transport.heldCount + XCTAssertEqual(pending, 0) + } + + func testAcceptedSendCompletesOutboxBeforeOptionalReadsFinish() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let completed = expectation(description: "Accepted send completes without snapshot reads") + let sending = Task { + let sent = await fixture.model.sendMessage(threadID: fixture.threadID, text: "Accepted now", selection: nil) + completed.fulfill() + return sent + } + _ = await reads.next() + await fulfillment(of: [completed], timeout: 2) + let queued = try await fixture.outbox.submissions() + XCTAssertTrue(queued.isEmpty, "Acknowledgement must retire the outbox while optional reads are held.") + XCTAssertFalse(fixture.model.isPerformingAction) + + await fixture.transport.failFollowups() + let sent = await sending.value + XCTAssertTrue(sent, "Optional read failures cannot turn acceptance into a failed send.") + XCTAssertNil(fixture.model.errorMessage) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(fixture.model.details[fixture.threadID]?.messages.last?.state, .complete) + await fixture.client.disconnect() + } + + func testAcceptedSendRefreshesCoalesceAndCancelWhenThreadCloses() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "First", selection: nil) + let firstRead = await reads.next() + let first = try XCTUnwrap(firstRead) + XCTAssertTrue(first.path.hasPrefix("/api/orchestration/threads/")) + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Second", selection: nil) + let heldCount = await fixture.transport.heldCount + XCTAssertEqual(heldCount, 1, "A second acceptance must coalesce behind the current read.") + + await fixture.transport.release(first.id) + let shellRead = await reads.next() + let shell = try XCTUnwrap(shellRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + await fixture.transport.release(shell.id) + let trailingRead = await reads.next() + let trailing = try XCTUnwrap(trailingRead) + XCTAssertTrue(trailing.path.hasPrefix("/api/orchestration/threads/")) + fixture.client.releaseThread(id: fixture.threadID) + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, trailing.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2, "Read reconciliation must never redispatch a command.") + await fixture.transport.failFollowups() + await fixture.client.disconnect() + } + + func testSupersededAcceptedSendRefreshDoesNotPublishItsOlderDetail() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups(includeShell: false) + let history = AcceptedSendDetailHistory(model: fixture.model, threadID: fixture.threadID) + history.start() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "First", selection: nil) + let firstRead = await reads.next() + let first = try XCTUnwrap(firstRead) + XCTAssertTrue(first.path.hasPrefix("/api/orchestration/threads/")) + // The second acceptance lands while the first read is still in flight. + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Second", selection: nil) + let stale = "Read started before the second send was accepted" + await fixture.transport.release(first.id, detailMessage: stale) + let followUpRead = await reads.next() + let followUp = try XCTUnwrap(followUpRead) + XCTAssertTrue(followUp.path.hasPrefix("/api/orchestration/threads/")) + let fresh = "Read after the second send was accepted" + await fixture.transport.release(followUp.id, detailMessage: fresh) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: fresh + ).wait() + XCTAssertFalse(history.seen.contains(stale), "A superseded read must not replace newer accepted state.") + await fixture.client.disconnect() + } + + func testDeletingThreadCancelsItsAcceptedSendRefresh() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Accepted before delete", selection: nil) + let heldRead = await reads.next() + let held = try XCTUnwrap(heldRead) + let deletion = Task { try await fixture.client.deleteThread(id: fixture.threadID) } + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id, "A deleted thread's pending refresh must not publish it again.") + await fixture.transport.failFollowups() + _ = try? await deletion.value + await fixture.client.disconnect() + } + + func testDisconnectCancelsAcceptedSendRefreshWithoutAnotherCommand() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Accepted before disconnect", selection: nil) + let heldRead = await reads.next() + let held = try XCTUnwrap(heldRead) + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + let pending = await fixture.transport.heldCount + XCTAssertEqual(pending, 0) + } + func testSavedSettingsSurviveAConnectionRepublish() async throws { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("t3-native-settings-republish-\(UUID().uuidString)") @@ -100,32 +554,21 @@ final class NativeRetryIdentityTests: XCTestCase { let settingsStore = UserDefaults(suiteName: settingsSuite)! defer { settingsStore.removePersistentDomain(forName: settingsSuite) } let client = NativeFeatureClient(runtime: runtime, settingsStore: settingsStore) - let initial = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + let initial = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() var updated = initial.settings updated.textSize = FeatureTextSizeAdjustment(steps: 2) updated.codeSize = FeatureTextSizeAdjustment(steps: -1) try await client.saveSettings(updated) - var events = client.events().makeAsyncIterator() - await connection.failReceive() - - var receivedReconnect = false - while let event = await events.next() { - guard case let .connection(state, _) = event, - state.state == .reconnecting else { - continue - } - receivedReconnect = true - break - } - XCTAssertTrue(receivedReconnect) - // A reconnect patches the connection instead of republishing the - // snapshot. The next snapshot the client builds must still carry the - // saved sizes. - let republished = try await client.backgroundSnapshot() - XCTAssertEqual(republished.settings.textSize.steps, 2) - XCTAssertEqual(republished.settings.codeSize.steps, -1) + let snapshot = try await recorder.wait { + $0.connection.state == .reconnecting && $0.settings.textSize.steps == 2 + } + XCTAssertEqual(snapshot.settings.textSize.steps, 2) + XCTAssertEqual(snapshot.settings.codeSize.steps, -1) await client.disconnect() } @@ -160,7 +603,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-concurrent-retry-\(UUID().uuidString)" )! ) - _ = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() await transport.rejectShellReads() @@ -226,7 +672,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-retry-\(UUID().uuidString)" )! let client = NativeFeatureClient(runtime: runtime, settingsStore: settings) - let initial = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + let initial = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } XCTAssertEqual(initial.threads.first?.runtimeMode, .approvalRequired) XCTAssertEqual(initial.threads.first?.interactionMode, .standard) await connection.waitUntilConnected() @@ -341,7 +790,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-partial-\(UUID().uuidString)" )! let client = NativeFeatureClient(runtime: runtime, settingsStore: settings) - _ = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() let identity = FeatureSubmissionIdentity( @@ -848,9 +1300,9 @@ private actor PartialBootstrapWebSocketConnection: WebSocketConnection { } } -private func retryConfigResponse(for request: JSONValue) throws -> Data? { +private func retryConfigResponse(for request: JSONValue, supportsPagination: Bool = false) throws -> Data? { guard case let .number(requestID)? = request["id"] else { return nil } - let config = JSONValue.object(["providers": .array([])]) + let config = JSONValue.object(["providers": .array([]), "threadSnapshotPagination": .bool(supportsPagination)]) if request["tag"]?.stringValue == RPCMethod.subscribeServerConfig.rawValue { return try JSONEncoder.t3.encode( JSONValue.object([ @@ -923,3 +1375,535 @@ private func retryDispatchCommand(from request: URLRequest) throws -> JSONValue } return try JSONDecoder.t3.decode(JSONValue.self, from: body) } + +@MainActor +private struct AcceptedSendFixture { + let modelTask: Task + let directory: URL + let threadID: String + let settingsName: String + let client: NativeFeatureClient + let model: FeatureRootModel + let outbox: FeatureOutboxStore + let transport: AcceptedSendHTTPTransport + let socket: AcceptedSendSocket + let runtime: EnvironmentRuntime + + static func make( + includePeer: Bool = false, turnID: String? = nil, captured: JSONValue? = nil, supportsPagination: Bool = false + ) async throws -> Self { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("t3-accepted-send-\(UUID().uuidString)") + let environment = Environment( + id: "accepted-send", label: "Accepted send", + httpBaseURL: URL(string: "https://accepted-send.example")!, + webSocketBaseURL: URL(string: "wss://accepted-send.example")! + ) + let store = EnvironmentStore(fileURL: directory.appendingPathComponent("environments.json")) + let peer = Environment( + id: "accepted-peer", label: "Accepted peer", + httpBaseURL: URL(string: "https://accepted-peer.example")!, + webSocketBaseURL: URL(string: "wss://accepted-peer.example")! + ) + try await store.save(includePeer ? [environment, peer] : [environment]) + try await store.setActiveEnvironment(id: environment.id) + let transport = AcceptedSendHTTPTransport(turnID: turnID, captured: captured) + let socket = AcceptedSendSocket(supportsPagination: supportsPagination) + let runtime = EnvironmentRuntime( + environmentStore: store, + credentialStore: InMemoryCredentialStore(credentials: [ + environment.id: EnvironmentCredential(accessToken: "fixture-token"), + peer.id: EnvironmentCredential(accessToken: "fixture-peer-token"), + ]), + httpTransport: transport, + webSocketConnector: AcceptedSendConnector(socket: socket, peer: AcceptedSendSocket()) + ) + let settingsName = "t3-accepted-send-\(UUID().uuidString)" + let configurationReady = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + defer { configurationReady.continuation.finish() } + let client = NativeFeatureClient( + runtime: runtime, settingsStore: UserDefaults(suiteName: settingsName)!, + fallbackPollingInitialDelay: .seconds(3600), aggregateRefreshInterval: .seconds(3600), + aggregateIdleRefreshInterval: .seconds(3600), + aggregateRefreshReceipt: { receipt in + if case .configurationApplied(environmentID: environment.id) = receipt { + configurationReady.continuation.yield(()) + configurationReady.continuation.finish() + } + } + ) + let outbox = FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")) + let model = FeatureRootModel(client: client, outboxStore: outbox) + let modelTask = Task { await model.start() } + do { + let wireID = captured?["threadId"]?.stringValue ?? "thread-existing" + try await AcceptedSendRootReadiness(model: model, wireID: wireID).wait() + if supportsPagination { + var readiness = configurationReady.stream.makeAsyncIterator() + guard let _ = await readiness.next() else { throw CancellationError() } + } + let threadID = FeatureScopedID.thread(environmentID: environment.id, wireID: wireID) + _ = await model.detail(for: threadID) + return Self(modelTask: modelTask, directory: directory, threadID: threadID, + settingsName: settingsName, client: client, model: model, + outbox: outbox, transport: transport, socket: socket, runtime: runtime) + } catch { + modelTask.cancel() + await client.disconnect() + await modelTask.value + UserDefaults.standard.removePersistentDomain(forName: settingsName) + try? FileManager.default.removeItem(at: directory) + throw error + } + } + + func cleanUp() async { + modelTask.cancel() + await client.disconnect() + await modelTask.value + UserDefaults.standard.removePersistentDomain(forName: settingsName) + try? FileManager.default.removeItem(at: directory) + } +} + +@MainActor +private final class AcceptedSendRootReadiness { + private let model: FeatureRootModel + private let wireID: String + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel, wireID: String = "thread-existing") { + self.model = model + self.wireID = wireID + } + + func wait() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor in self.finish(CancellationError()) } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + !model.isLoading + && model.snapshot.projects.contains { $0.id == FeatureScopedID.project(environmentID: "accepted-send", wireID: "project-1") } + && model.snapshot.threads.contains { $0.id == FeatureScopedID.thread(environmentID: "accepted-send", wireID: wireID) } + && model.snapshot.environments.first(where: { $0.id == "accepted-send" })?.connectionState == .connected + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { finish(nil) } + } + + private func finish(_ error: Error?) { + guard let pending = continuation else { return } + continuation = nil + if let error { pending.resume(throwing: error) } else { pending.resume() } + } +} + +/// Records every last-message text the model shows for a thread. +@MainActor +private final class AcceptedSendDetailHistory { + private let model: FeatureRootModel + private let threadID: String + private(set) var seen: [String] = [] + + init(model: FeatureRootModel, threadID: String) { + self.model = model + self.threadID = threadID + } + + func start() { + let text = withObservationTracking { + model.details[threadID]?.messages.last?.text + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.start() } + } + if let text, seen.last != text { seen.append(text) } + } +} + +@MainActor +private final class AcceptedSendDetailReceipt { + private let model: FeatureRootModel + private let threadID: String + private let expectedMessage: String + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel, threadID: String, expectedMessage: String) { + self.model = model + self.threadID = threadID + self.expectedMessage = expectedMessage + } + + func wait() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor in self.finish(CancellationError()) } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + model.details[threadID]?.messages.last?.text == expectedMessage + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { finish(nil) } + } + + private func finish(_ error: Error?) { + guard let pending = continuation else { return } + continuation = nil + if let error { pending.resume(throwing: error) } else { pending.resume() } + } +} + +private actor AcceptedSendHTTPTransport: HTTPTransport { + struct HeldRead: Sendable { + let id: UUID + let path: String + } + nonisolated let heldReads: AsyncStream + nonisolated let cancellations: AsyncStream + private let reads: AsyncStream.Continuation + private let cancelled: AsyncStream.Continuation + private(set) var snapshotReadCount = 0 + private var holdsShell = true + private var holds = false + private var fails = false + private var pending: [UUID: CheckedContinuation] = [:] + private var detailMessages: [UUID: String] = [:] + + private let captured: JSONValue? + private var capturedPhase = "before" + + func useCapturedPhase(_ phase: String) { capturedPhase = phase } + func allowFollowups() { holds = false } + + private let turnID: String? + + init(turnID: String? = nil, captured: JSONValue? = nil) { + self.captured = captured + self.turnID = turnID + (heldReads, reads) = AsyncStream.makeStream() + (cancellations, cancelled) = AsyncStream.makeStream() + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let path = request.url?.path ?? "" + if path == "/api/auth/websocket-ticket" { + return (Data(#"{"ticket":"fixture-ticket","expiresAt":"2026-09-07T12:05:00Z"}"#.utf8), retryHTTPResponse(request)) + } + guard path == "/api/orchestration/shell" || path.hasPrefix("/api/orchestration/threads/") else { + throw URLError(.unsupportedURL) + } + snapshotReadCount += 1 + if fails { throw URLError(.networkConnectionLost) } + var heldID: UUID? + if holds && (holdsShell || path != "/api/orchestration/shell") { + let id = UUID() + heldID = id + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + pending[id] = continuation + reads.yield(HeldRead(id: id, path: path)) + } + } onCancel: { + Task { await self.cancel(id) } + } + } + if let captured { + let key = path == "/api/orchestration/shell" ? "shell" : "detail" + guard let snapshot = captured[capturedPhase]?[key] else { throw HTTPError.invalidResponse } + return (try JSONEncoder.t3.encode(snapshot), retryHTTPResponse(request)) + } + let data: Data + if path == "/api/orchestration/shell" { + data = try JSONEncoder.t3.encode(retryShellSnapshot()) + } else { + let snapshot = retryEmptyThreadDetail(id: request.url!.lastPathComponent) + if let heldID, let text = detailMessages.removeValue(forKey: heldID) { + var thread = try JSONValue.encode(snapshot.thread).decode([String: JSONValue].self) + thread["messages"] = .array([.object([ + "id": .string("replacement-generation-message"), "role": .string("assistant"), + "text": .string(text), "turnId": .null, "streaming": .bool(false), + "attachments": .array([]), "createdAt": .string("2026-07-30T12:00:00.000Z"), + "updatedAt": .string("2026-07-30T12:00:00.000Z"), + ])]) + data = try JSONEncoder.t3.encode(OrchestrationThreadDetailSnapshot( + snapshotSequence: 99, + thread: try JSONValue.object(thread).decode(OrchestrationThread.self) + )) + } else { data = try JSONEncoder.t3.encode(snapshot) } + } + if let turnID { + var root = try JSONSerialization.jsonObject(with: data) as! [String: Any] + let latest: [String: Any] = ["turnId": turnID, "state": "running", "requestedAt": "2026-07-30T12:00:00.000Z"] + if var threads = root["threads"] as? [[String: Any]] { + for index in threads.indices { threads[index]["latestTurn"] = latest } + root["threads"] = threads + } + if var thread = root["thread"] as? [String: Any] { + thread["latestTurn"] = latest + root["thread"] = thread + } + return (try JSONSerialization.data(withJSONObject: root), retryHTTPResponse(request)) + } + return (data, retryHTTPResponse(request)) + } + + var heldCount: Int { pending.count } + + func isPending(_ id: UUID) -> Bool { pending[id] != nil } + + func holdFollowups(includeShell: Bool = true) { holds = true; holdsShell = includeShell } + + func release(_ id: UUID, detailMessage: String? = nil) { + guard let continuation = pending.removeValue(forKey: id) else { return } + if let detailMessage { detailMessages[id] = detailMessage } + continuation.resume() + } + + func failFollowups() { + fails = true + holds = false + let waiting = pending.values + pending.removeAll() + waiting.forEach { $0.resume(throwing: URLError(.networkConnectionLost)) } + } + + private func cancel(_ id: UUID) { + guard let continuation = pending.removeValue(forKey: id) else { return } + continuation.resume(throwing: CancellationError()) + cancelled.yield(id) + } +} + +private struct AcceptedSendConnector: WebSocketConnecting { + let socket: AcceptedSendSocket + let peer: AcceptedSendSocket + func connect(to url: URL) -> any WebSocketConnection { + url.host == "accepted-peer.example" ? peer : socket + } +} + +private actor AcceptedSendSocket: WebSocketConnection { + nonisolated let heldInterrupts: AsyncStream + private let interruptReceipts: AsyncStream.Continuation + private var holdsInterrupts = false + private var rejectsInterrupts = false + private var pendingInterruptResponses: [Data] = [] + + private let supportsPagination: Bool + init(supportsPagination: Bool = false) { + self.supportsPagination = supportsPagination + (heldInterrupts, interruptReceipts) = AsyncStream.makeStream() + } + func holdInterruptAcknowledgements() { holdsInterrupts = true } + func rejectInterruptAcknowledgements() { rejectsInterrupts = true } + func releaseInterruptAcknowledgements() { + holdsInterrupts = false + let responses = pendingInterruptResponses + pendingInterruptResponses.removeAll() + responses.forEach { enqueue($0) } + } + private(set) var commands: [JSONValue] = [] + private var queued: [Data] = [] + private var receiver: CheckedContinuation? + private var receiverID: UUID? + + func send(_ data: Data) throws { + let request = try JSONDecoder.t3.decode(JSONValue.self, from: data) + let tag = request["tag"]?.stringValue + if tag == RPCMethod.serverGetConfig.rawValue || tag == RPCMethod.subscribeServerConfig.rawValue, + let response = try retryConfigResponse(for: request, supportsPagination: supportsPagination) { + enqueue(response) + } else if tag == RPCMethod.dispatchCommand.rawValue, let payload = request["payload"] { + commands.append(payload) + let isInterrupt = payload["type"]?.stringValue == "thread.turn.interrupt" + let exit: JSONValue = isInterrupt && rejectsInterrupts + ? .object(["_tag": .string("Failure"), "cause": .array([ + .object(["_tag": .string("Fail"), "error": .object(["message": .string("Stop rejected by fixture")])]), + ])]) + : .object(["_tag": .string("Success"), "value": .object(["sequence": .number(2)])]) + let response = try JSONEncoder.t3.encode(JSONValue.object([ + "_tag": .string("Exit"), "requestId": request["id"]!, "exit": exit, + ])) + if isInterrupt && holdsInterrupts { + pendingInterruptResponses.append(response) + interruptReceipts.yield(()) + } else { enqueue(response) } + } + } + + func receive() async throws -> Data { + if !queued.isEmpty { return queued.removeFirst() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + guard !Task.isCancelled else { + continuation.resume(throwing: CancellationError()) + return + } + // This fixture reuses its socket across connector generations. + receiver?.resume(throwing: CancellationError()) + receiver = continuation + receiverID = id + } + } onCancel: { + Task { await self.cancelReceive(id) } + } + } + + private func cancelReceive(_ id: UUID) { + guard receiverID == id else { return } + receiver?.resume(throwing: CancellationError()) + receiver = nil + receiverID = nil + } + + func close() { + receiver?.resume(throwing: CancellationError()) + receiver = nil + receiverID = nil + } + + private func enqueue(_ data: Data) { + if let receiver { self.receiver = nil; receiverID = nil; receiver.resume(returning: data) } + else { queued.append(data) } + } +} + +@MainActor +private final class AcceptedCommandSnapshotRecorder { + private(set) var history: [FeatureSnapshot] + private var task: Task? + private var waiters: [UUID: (@MainActor (FeatureSnapshot) -> Bool, CheckedContinuation)] = [:] + private var finished = false + + init(seed: FeatureSnapshot, events: AsyncStream) { + history = [seed] + task = Task { [weak self] in + for await event in events { + guard !Task.isCancelled else { break } + guard let self else { break } + switch event { + case let .snapshot(snapshot): self.record(snapshot) + case let .thread(thread): + guard var snapshot = self.history.last else { continue } + if let index = snapshot.threads.firstIndex(where: { $0.id == thread.id }) { + snapshot.threads[index] = thread + } else { snapshot.threads.append(thread) } + self.record(snapshot) + case let .threadRemoved(id): + guard var snapshot = self.history.last else { continue } + snapshot.threads.removeAll { $0.id == id } + self.record(snapshot) + case let .connection(connection, environmentID): + guard var snapshot = self.history.last else { continue } + snapshot.connection = connection + if let index = snapshot.environments.firstIndex(where: { $0.id == environmentID }) { + snapshot.environments[index].connectionState = connection.state + } + self.record(snapshot) + default: break + } + } + self?.stop() + } + } + + func record(_ snapshot: FeatureSnapshot) { + history.append(snapshot) + let ready = waiters.filter { $0.value.0(snapshot) } + for (id, waiter) in ready { + waiters[id] = nil + waiter.1.resume(returning: snapshot) + } + } + + func wait(_ predicate: @escaping @MainActor (FeatureSnapshot) -> Bool) async throws -> FeatureSnapshot { + try Task.checkCancellation() + if let snapshot = history.last, predicate(snapshot) { return snapshot } + guard !finished else { throw CancellationError() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (predicate, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + } + } + + func stop() { + finished = true + task?.cancel() + task = nil + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.1.resume(throwing: CancellationError()) } + } + + deinit { task?.cancel() } +} + +@MainActor +private final class CapturedStopProjectionReceipt { + let model: FeatureRootModel + let threadID: String + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel, threadID: String) { + self.model = model + self.threadID = threadID + } + + func wait() async { + await withCheckedContinuation { continuation = $0; observe() } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + model.snapshot.threads.first { $0.id == threadID }?.latestTurnState == "interrupted" + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { continuation?.resume(); continuation = nil } + } +} + +private extension NativeFeatureClient { + /// Sends plain text through the full submission API with fixture defaults. + func sendMessage(threadID: String, text: String, selection: FeatureSelection?) async throws { + try await sendMessage( + threadID: threadID, text: text, selection: selection, runtimeMode: .fullAccess, + attachments: [], identity: FeatureSubmissionIdentity(threadID: threadID) + ) + } +} + +private extension FeatureRootModel { + func sendMessage(threadID: String, text: String, selection: FeatureSelection?) async -> Bool { + await sendMessage(FeatureMessageSubmission(threadID: threadID, text: text, selection: selection)) + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeStopProjectionFixtures.swift b/apps/swift-ios/Tests/FeatureTests/NativeStopProjectionFixtures.swift new file mode 100644 index 000000000000..c1924e05d4b1 --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/NativeStopProjectionFixtures.swift @@ -0,0 +1,3227 @@ +import Foundation +@testable import T3Code + +// Captured from the b592 engine/projector with a held in-memory provider. +// Terminal phases are explicit resolution + ready-session commands, not live provider completion. +enum NativeStopProjectionFixtures { + static func snapshot(_ scenario: String) throws -> JSONValue { + try JSONDecoder.t3.decode(JSONValue.self, from: Data(captures[scenario]!.utf8)) + } + + private static let captures: [String: String] = [ + // Source capture SHA256: 409c2824312c4d9bb71400fdcdc3be67b3259169bf25a545a330a2e6047d6a8c + "approval": #""" + { + "threadId": "thread-1", + "before": { + "shell": { + "snapshotSequence": 4, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": true, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "detail": { + "snapshotSequence": 4, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "requestHeld": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": true, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "afterAcknowledged": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": true, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "afterTerminal": { + "shell": { + "snapshotSequence": 7, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "detail": { + "snapshotSequence": 7, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + }, + { + "id": "outcome-proof-resolved-activity", + "tone": "info", + "kind": "approval.resolved", + "summary": "Pending request resolved (fixture terminal command)", + "payload": { + "requestId": "outcome-proof-approval-1", + "decision": "cancel" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:03.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + } + } + } + } + } + """#, + // Source capture SHA256: 6633d0a12343a5ac72daa0f6caa6fbc872a3208449329f80b7ca535d3f232cab + "approval-starting": #""" + { + "threadId": "thread-1", + "before": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "starting", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": true, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "starting", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "requestHeld": { + "shell": { + "snapshotSequence": 6, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "starting", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": true, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 6, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "starting", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "afterAcknowledged": { + "shell": { + "snapshotSequence": 6, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "starting", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": true, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 6, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "starting", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "afterTerminal": { + "shell": { + "snapshotSequence": 8, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "detail": { + "snapshotSequence": 8, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-approval-activity", + "tone": "approval", + "kind": "approval.requested", + "summary": "Command approval requested", + "payload": { + "requestId": "outcome-proof-approval-1", + "requestKind": "command" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + }, + { + "id": "outcome-proof-resolved-activity", + "tone": "info", + "kind": "approval.resolved", + "summary": "Pending request resolved (fixture terminal command)", + "payload": { + "requestId": "outcome-proof-approval-1", + "decision": "cancel" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:03.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + } + } + } + } + } + """#, + // Source capture SHA256: 627f5b3e790c48b03a849b152d658430ee77f79049c2b3a02a4e8f67d0095098 + "input": #""" + { + "threadId": "thread-1", + "before": { + "shell": { + "snapshotSequence": 4, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": true, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "detail": { + "snapshotSequence": 4, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "requestHeld": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": true, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "afterAcknowledged": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": true, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "afterTerminal": { + "shell": { + "snapshotSequence": 7, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "detail": { + "snapshotSequence": 7, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + }, + { + "id": "outcome-proof-resolved-activity", + "tone": "info", + "kind": "user-input.resolved", + "summary": "Pending request resolved (fixture terminal command)", + "payload": { + "requestId": "outcome-proof-input-1", + "decision": "cancel" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:03.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + } + } + } + } + } + """#, + // Source capture SHA256: 699096e61aa6bfc392a77ea33d1915f2c64f3bb1041197d0e5c85ab909ea63da + "background-ready-null": #""" + { + "threadId": "thread-1", + "before": { + "shell": { + "snapshotSequence": 4, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "completed", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "detail": { + "snapshotSequence": 4, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "completed", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "requestHeld": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "afterAcknowledged": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "afterTerminal": { + "shell": { + "snapshotSequence": 6, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "detail": { + "snapshotSequence": 6, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:01.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + } + } + } + } + } + """#, + // Source capture SHA256: 2068406d2ec5e566c4bb64000e394c9d27e1f8c9533e35a992ae63ff18a4d559 + "active-running-null": #""" + { + "threadId": "thread-1", + "before": { + "shell": { + "snapshotSequence": 4, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "detail": { + "snapshotSequence": 4, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "requestHeld": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "afterAcknowledged": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:01.000Z" + } + } + } + }, + "afterTerminal": { + "shell": { + "snapshotSequence": 6, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": false, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:03.000Z" + }, + "detail": { + "snapshotSequence": 6, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:03.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "ready", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": null, + "updatedAt": "2026-01-01T00:00:03.000Z" + } + } + } + } + } + """#, + // Source capture SHA256: ed3d66ad519f7b40052e722c8a00d5a9b5815c78c99714abe28ffc9e192d0183 + "input-failure": #""" + { + "threadId": "thread-1", + "before": { + "shell": { + "snapshotSequence": 4, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": true, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:01.000Z" + }, + "detail": { + "snapshotSequence": 4, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "running", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": null, + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "requestHeld": { + "shell": { + "snapshotSequence": 5, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": true, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 5, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:01.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "running", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": "outcome-proof-A", + "lastError": null, + "updatedAt": "2026-01-01T00:00:00.000Z" + } + } + } + }, + "afterFailure": { + "shell": { + "snapshotSequence": 7, + "projects": [ + { + "id": "project-1", + "title": "Provider Project", + "workspaceRoot": "/tmp/provider-project", + "repositoryIdentity": null, + "defaultModelSelection": null, + "defaultThreadEnvMode": null, + "autoPull": false, + "faviconPath": null, + "projectIcon": null, + "scripts": [], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z" + } + ], + "threads": [ + { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:02.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "session": { + "threadId": "thread-1", + "status": "stopped", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": "Held fixture provider interrupt failed (stub)", + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "latestUserMessageAt": null, + "hasPendingApprovals": false, + "hasPendingUserInput": true, + "hasActionableProposedPlan": false, + "backgroundLiveness": null, + "planProgress": null + } + ], + "updatedAt": "2026-01-01T00:00:02.000Z" + }, + "detail": { + "snapshotSequence": 7, + "thread": { + "id": "thread-1", + "projectId": "project-1", + "title": "Thread", + "modelSelection": { + "instanceId": "codex", + "model": "gpt-5-codex" + }, + "runtimeMode": "approval-required", + "interactionMode": "default", + "branch": null, + "worktreePath": null, + "branchPullRequest": null, + "latestTurn": { + "turnId": "outcome-proof-A", + "state": "interrupted", + "requestedAt": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-01T00:00:00.000Z", + "completedAt": "2026-01-01T00:00:02.000Z", + "assistantMessageId": null + }, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:02.000Z", + "archivedAt": null, + "settledOverride": null, + "settledAt": null, + "unsettledAt": null, + "snoozedUntil": null, + "snoozedAt": null, + "pinnedAt": null, + "pinOrderKey": null, + "activeOrderKey": null, + "titleRegeneration": null, + "deletedAt": null, + "messages": [], + "proposedPlans": [], + "activities": [ + { + "id": "outcome-proof-input-activity", + "tone": "info", + "kind": "user-input.requested", + "summary": "User input requested", + "payload": { + "requestId": "outcome-proof-input-1", + "questions": [ + { + "id": "sandbox_mode", + "header": "Sandbox", + "question": "Which mode should be used?", + "options": [ + { + "label": "workspace-write", + "description": "Allow workspace writes only" + } + ] + } + ] + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:01.000Z" + }, + { + "id": "8853c32a-affd-4ce5-97ea-39b0b71e024d", + "tone": "error", + "kind": "provider.turn.interrupt.failed", + "summary": "Provider turn interrupt failed", + "payload": { + "detail": "Held fixture provider interrupt failed (stub)" + }, + "turnId": "outcome-proof-A", + "createdAt": "2026-01-01T00:00:02.000Z" + } + ], + "checkpoints": [], + "session": { + "threadId": "thread-1", + "status": "stopped", + "providerName": "codex", + "runtimeMode": "approval-required", + "activeTurnId": null, + "lastError": "Held fixture provider interrupt failed (stub)", + "updatedAt": "2026-01-01T00:00:02.000Z" + } + } + } + } + } + """#, + ] +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index e97c5d3f5718..678d7a5c6761 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -5,6 +5,960 @@ import XCTest @MainActor @available(iOS 18.0, *) final class NativeThreadCatchUpTests: XCTestCase { + func testFaultInjectedOmittedMessageRepairsAtEqualWatermark() async throws { + // Deliberate application-delivery omission, not simulated TCP packet loss. + // HTTP is authoritative for both messages; only event 4 reaches the client. + for paginated in [false, true] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, + reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + // The missing event would have sequence 3; later event 4 is valid. + try await detail.sendMessage(text: "Later delivered message", sequence: 4) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let before = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(before.messages.map(\.id), ["message-4"]) + await fixture.http.setFaultInjectionMessages(paginated: paginated) + for attempt in 1...2 { + let pendingTick = await ticks.next() + let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)) + tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + let raw = try rawThreadForHistoryTest(fixture.client) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, attempt + 1) + print("FAULT_INJECTED_EQUAL_WATERMARK paginated=\(paginated) attempt=\(attempt) reads=\(reads) messageIDs=\(raw.messages.map(\.id))") + XCTAssertEqual(Set(raw.messages.map(\.id)), Set(["message-3", "message-4"]), + "An authoritative equal-watermark read must repair omitted message 3; attempt \(attempt).") + } + await fixture.client.disconnect() + } + } + + + func testSelectedReconciliationPreservesExplicitOlderRawFanoutBelowTenUsers() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300) + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.messages.count, 150) + XCTAssertEqual(initial.messages.filter { $0.role == .user }.count, 1) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let loaded = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(loaded?.messages.count, 300) + XCTAssertEqual(loaded?.messages.filter { $0.role == .user }.count, 2) + XCTAssertEqual(loaded?.messages.first?.text, "Raw 0") + XCTAssertEqual(loaded?.page?.hasMore, false) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.last?.text, "Changed raw tail") + XCTAssertEqual(value?.messages.count, 300, "Quiet repair must preserve explicitly loaded raw-turn history even below ten user rows.") + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertNotEqual(value?.page?.hasMore, true) + print("RAW_CAP_PROOF initial=\(initial.messages.count) loaded=\(loaded?.messages.count ?? -1) loadedUsers=\(loaded?.messages.filter { $0.role == .user }.count ?? -1) repaired=\(value?.messages.count ?? -1)") + print("RAW_CAP_REQUESTS \(await fixture.http.threadRequests.map { $0.url!.absoluteString })") + } + + func testSelectedReconciliationPreservesExplicitOlderRawFanoutWithZeroUsers() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.messages.count, 150) + XCTAssertEqual(initial.messages.filter { $0.role == .user }.count, 0) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let loaded = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(loaded?.messages.count, 300) + XCTAssertEqual(loaded?.messages.filter { $0.role == .user }.count, 0) + XCTAssertEqual(loaded?.messages.first?.text, "Raw 0") + XCTAssertEqual(loaded?.page?.hasMore, false) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.last?.text, "Changed raw tail") + XCTAssertEqual(value?.messages.count, 300, "Quiet repair must preserve explicitly loaded raw-turn history even below ten user rows.") + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertNotEqual(value?.page?.hasMore, true) + print("RAW_CAP_PROOF initial=\(initial.messages.count) loaded=\(loaded?.messages.count ?? -1) loadedUsers=\(loaded?.messages.filter { $0.role == .user }.count ?? -1) repaired=\(value?.messages.count ?? -1)") + print("RAW_CAP_REQUESTS \(await fixture.http.threadRequests.map { $0.url!.absoluteString })") + } + + func testSelectedReconciliationRetainsRawCollectionsAndAcceptsAuthoritativeDeletion() async throws { + for collection in ["activities", "checkpoints"] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: [], collection: collection) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + var raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(raw.messages.count, 0) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 300) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release(); _ = await completed.next() + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 300) + var urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, collection == "activities" ? 4 : 3) + XCTAssertEqual(urls.last!.contains("turnLimit="), collection == "checkpoints", + "Only windowed activities need full fallback for an ordinary changed tail.") + await fixture.http.removePaginatedUser(id: "raw-0", sequence: 302) + let pendingDelete = await ticks.next(); let deleteTick = try XCTUnwrap(pendingDelete) + clock.advance(by: .seconds(30)); deleteTick.release(); _ = await completed.next() + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) + XCTAssertFalse(raw.activities.contains { $0.id == "activity-raw-0" }) + XCTAssertFalse(raw.checkpoints.contains { $0.turnId == "turn-raw-0" }) + urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, collection == "activities" ? 6 : 5) + XCTAssertFalse(urls.last!.contains("turnLimit=")) + let pendingUnchanged = await ticks.next(); let unchangedTick = try XCTUnwrap(pendingUnchanged) + clock.advance(by: .seconds(30)); unchangedTick.release(); _ = await completed.next() + let unchangedURLs = await fixture.http.threadRequests + XCTAssertEqual(unchangedURLs.count, collection == "activities" ? 8 : 6, "A matching cursor cannot certify omitted rows outside a capped recent window.") + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) + } + } + + private func rawThreadForHistoryTest(_ client: NativeFeatureClient) throws -> OrchestrationThread { + // Read-only test inspection: checkpoints are retained raw state and have + // no standalone published collection in FeatureThreadDetail. + try XCTUnwrap(Mirror(reflecting: client).children.first { $0.label == "activeRawThread" }?.value as? OrchestrationThread) + } + + func testSelectedReconciliationRepairsSilentDetailWithoutSyncFlicker() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Quiet HTTP repair", sequence: 20) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + var messages: [String] = [] + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): messages = value.messages.map(\.text) + case .threadSync(fixture.firstID, .live): break + case .threadSync(fixture.firstID, .catchingUp): XCTFail("Quiet repair must not flash catch-up.") + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(messages, ["Quiet HTTP repair"]) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } + + func testSelectedReconciliationUnchangedSnapshotPublishesNothing() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + var publications = 0 + while let event = await events.next(isolation: #isolation) { + if case .detail = event { publications += 1 } + if case .detailDelta = event { publications += 1 } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(publications, 0) + await fixture.client.disconnect() + } + + func testSelectedReconciliationActualProgressPostponesButMarkerDoesNot() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pendingFirst = await ticks.next(); let first = try XCTUnwrap(pendingFirst) + clock.advance(by: .seconds(20)) + try await detail.sendMessage(text: "Live progress", sequence: 3) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + clock.advance(by: .seconds(10)); first.release() + let deferred = await completed.next() + XCTAssertEqual(deferred, .deferred(threadID: fixture.firstID)) + let pendingNext = await ticks.next(); let next = try XCTUnwrap(pendingNext) + XCTAssertEqual(next.duration, .seconds(20)) + clock.advance(by: .seconds(10)) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + clock.advance(by: .seconds(10)); next.release() + let finished = await completed.next() + XCTAssertEqual(finished, .finished(threadID: fixture.firstID)) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } + + func testSelectedReconciliationRejectsOlderHeldHTTPAfterNewerDetail() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Old HTTP", sequence: 3) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + try await detail.sendMessage(text: "Newer live response", sequence: 4) + try await detail.synchronize() + let live = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(live, ["Newer live response"]) + held.succeed() + _ = await completed.next() + try await detail.synchronize() + let replaced = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertTrue(replaced.isEmpty, "Older HTTP must not republish over newer detail.") + await fixture.client.disconnect() + } + + func testSelectedReconciliationBackgroundCancelsItsHeldRead() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + fixture.client.suspendForBackground() + held.succeed() + let cancelled = await held.finished.first { _ in true } + XCTAssertEqual(cancelled, true) + await fixture.client.disconnect() + } + + func testSelectedReconciliationRequiredReadKeepsItsFailureAfterQuietRead() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + try await detail.invalidate(sequence: 3) + await nextCatchUp(&events, threadID: fixture.firstID) + quiet.succeed() + let required = try await nextHeldRead(&reads) + required.fail() + var failure: String? + while let event = await events.next(isolation: #isolation) { + if case let .threadSync(id, .failed(message)) = event, id == fixture.firstID { + failure = message; break + } + } + XCTAssertEqual(failure, URLError(.notConnectedToInternet).localizedDescription) + await fixture.client.disconnect() + } + + private func retainForSelectedReconciliationTest(_ fixture: CatchUpFixture, clock: SelectedReconciliationClock? = nil) { + addTeardownBlock { + await MainActor.run { clock?.cancelAll() } + await fixture.http.cancelHeldReads() + await fixture.client.disconnect() + await MainActor.run { fixture.cleanUp() } + } + } + + func testSelectedReconciliationPreservesLoadedHistoryWhenUnchanged() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let older = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(older?.messages.count, 30) + let retainedPage = older?.page + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let unchanged = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertTrue(unchanged.isEmpty, "An unchanged enlarged window must not publish extra history.") + let earlier = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(retainedPage?.beforeCursor, "user-20") + XCTAssertEqual(earlier?.messages.count, 50) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("beforeCursor=user-20")) + } + + func testSelectedReconciliationExpandedGapAndDeletedBoundaryUseAuthoritativeFullRead() async throws { + for deletedBoundary in [false, true] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let lower = deletedBoundary ? 21 : 0 + await fixture.http.setPaginatedUserMessages(lower..<110, sequence: 200) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages.first, "User \(lower)") + XCTAssertEqual(messages.last, "User 109") + XCTAssertEqual(messages.count, 110 - lower) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertFalse(urls[3].contains("turnLimit=")) + } + } + + func testSelectedReconciliationUnchangedReadDoesNotInvalidatePendingOlderPage() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed() + _ = await completed.next() + olderRead.succeed() + let older = try await olderTask.value + XCTAssertEqual(older?.messages.count, 30) + XCTAssertEqual(older?.page?.beforeCursor, "user-20") + XCTAssertEqual(older?.page?.isLoading, false) + } + + func testSelectedReconciliationNavigationRejectsHeldResponse() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Must not arrive after navigation", sequence: 20) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + fixture.client.releaseThread(id: fixture.firstID) + await fixture.http.holdThreadReads(false) + _ = try await fixture.client.loadThread(id: fixture.secondID) + _ = try await nextThreadRequest(&requests) + held.succeed() + let cancelled = await held.finished.first { _ in true } + XCTAssertEqual(cancelled, true) + fixture.client.releaseThread(id: fixture.secondID) + let restored = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertFalse(restored.messages.contains { $0.text == "Must not arrive after navigation" }) + } + + func testSelectedReconciliationUsesSelectedPassivePeerRoute() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + includePeer: true, reconciliationClock: clock, + reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + let peerID = FeatureScopedID.thread(environmentID: "two", wireID: "first") + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: peerID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: peerID) + await fixture.http.setResponse(text: "Peer HTTP repair", sequence: 20) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: peerID)) + try await detail.synchronize() + let messages = await messagesBeforeLive(&events, threadID: peerID) + XCTAssertEqual(messages, ["Peer HTTP repair"]) + let reads = await fixture.http.threadRequests + XCTAssertEqual(reads.count, 2) + XCTAssertTrue(reads.allSatisfy { $0.url?.host == "two.example" }) + } + + func testSelectedReconciliationDefaultDeadlineRepairsMessageAndWorkingState() async throws { + let fixture = try await CatchUpFixture.make() + retainForSelectedReconciliationTest(fixture) + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.thread.state, .idle) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setRunningResponse(text: "Default deadline repair", sequence: 20) + let start = ContinuousClock.now + var repaired: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + if value.thread.id == fixture.firstID, + value.messages.contains(where: { $0.text == "Default deadline repair" }) { + repaired = value + } + case .threadSync(fixture.firstID, .catchingUp): XCTFail("Quiet recovery must not flash catch-up.") + default: break + } + if repaired != nil { break } + } + let elapsed = start.duration(to: .now) + XCTAssertEqual(repaired?.thread.state, .working) + XCTAssertGreaterThanOrEqual(elapsed, .seconds(29)) + XCTAssertLessThan(elapsed, .seconds(40)) + let reads = await fixture.http.threadRequests + XCTAssertEqual(reads.count, 2) + print("DEFAULT_RECONCILIATION elapsed=\(elapsed) state=\(String(describing: repaired?.thread.state)) reads=\(reads.count) bytes=\(await fixture.http.threadResponseBytes)") + } + + func testSelectedReconciliationRepeatedTailEditsKeepExplicitHistoryExtent() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + for edit in 1...3 { + await fixture.http.editLatestUserText("Tail edit \(edit)", sequence: 100 + edit) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 70") + XCTAssertEqual(value?.messages.last?.text, "Tail edit \(edit)") + XCTAssertEqual(value?.page?.beforeCursor, "user-70") + } + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 5) + XCTAssertTrue(urls.dropFirst(2).allSatisfy { $0.contains("turnLimit=30") }) + } + + func testSelectedReconciliationNewUserAddsOnlyRequiredWindowExtent() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 31) + XCTAssertEqual(value?.messages.first?.text, "User 70") + XCTAssertEqual(value?.messages.last?.text, "User 100") + XCTAssertEqual(value?.page?.beforeCursor, "user-70") + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + if urls.count > 3 { XCTAssertTrue(urls[3].contains("turnLimit=31")) } + } + + func testSelectedReconciliationUserlessPendingAheadPageRecoversWithoutStreamProgress() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("New head", sequence: 301) + let pendingPage = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(pendingPage?.page?.isLoading, true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 300) + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertEqual(value?.messages.last?.text, "New head") + XCTAssertNotEqual(value?.page?.isLoading, true) + } + + func testSelectedReconciliationPendingAheadPageRecoversWithoutStreamProgress() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("New head", sequence: 51) + let pendingPage = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(pendingPage?.page?.isLoading, true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 20") + XCTAssertEqual(value?.messages.last?.text, "New head") + XCTAssertEqual(value?.page?.isLoading, false) + } + + func testSelectedReconciliationUserlessChangedQuietReadYieldsToOlderPageRequest() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed head", sequence: 301) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed(); _ = await completed.next() + olderRead.succeed() + let waitingOlder = try await olderTask.value + XCTAssertEqual(waitingOlder?.page?.isLoading, true, "Optional quiet read must not invalidate the user's older-page epoch.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let nextTick = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); nextTick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 300) + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertEqual(value?.messages.last?.text, "Changed head") + XCTAssertNotEqual(value?.page?.isLoading, true) + } + + func testSelectedReconciliationChangedQuietReadYieldsToOlderPageRequest() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed head", sequence: 51) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed(); _ = await completed.next() + olderRead.succeed() + let waitingOlder = try await olderTask.value + XCTAssertEqual(waitingOlder?.page?.isLoading, true, "Optional quiet read must not invalidate the user's older-page epoch.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let nextTick = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); nextTick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 20") + XCTAssertEqual(value?.messages.last?.text, "Changed head") + XCTAssertEqual(value?.page?.isLoading, false) + } + + private func selectedDetailBeforeLive(_ events: inout AsyncStream.Iterator, threadID: String) async -> FeatureThreadDetail? { + var value: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(detail), let .detailDelta(detail, _): + if detail.thread.id == threadID { value = detail } + case .threadSync(threadID, .live): return value + default: break + } + } + return value + } + + func testSelectedReconciliationMovingAdaptiveHeadDefersWithoutFullRead() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let first = try await nextHeldRead(&reads) + await fixture.http.setPaginatedUserMessages(0..<102, sequence: 102) + first.succeed() + let adaptive = try await nextHeldRead(&reads) + adaptive.succeed(); _ = await completed.next() + var urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("turnLimit=31")) + try await detail.synchronize() + let unchanged = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertNil(unchanged, "A moving second read must not widen or replace the retained range.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let next = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); next.release(); _ = await completed.next() + try await detail.synchronize() + let repaired = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired?.messages.count, 32) + XCTAssertEqual(repaired?.messages.first?.text, "User 70") + XCTAssertEqual(repaired?.messages.last?.text, "User 101") + urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 6) + XCTAssertTrue(urls[5].contains("turnLimit=32")) + } + + func testSelectedReconciliationDeletedAnchorAfterAdaptiveReadUsesOneFullFallback() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + await fixture.http.removePaginatedUser(id: "user-70", sequence: 102) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release(); _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 100) + XCTAssertFalse(value?.messages.contains(where: { $0.text == "User 70" }) ?? true) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 5) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("turnLimit=31")) + XCTAssertFalse(urls[4].contains("turnLimit=")) + } + + func testStaleDetailReplaySkipsReductionOnlyAfterEnvelopeValidation() throws { + let thread = multiEnvironmentDetail( + projectID: "project", threadID: "first", snapshotSequence: 2, messages: [] + ).thread + let event = replayMessage(sequence: 2, text: "Duplicate") + let ordinary = NativeThreadDetailReducer.apply(event, to: thread) + guard case .updated = ordinary.result else { + return XCTFail("The control must exercise a real message reduction.") + } + let skipped = NativeThreadDetailReducer.apply(event, to: thread, afterSequence: 2) + XCTAssertEqual(skipped.sequence, 2) + guard case .unchanged = skipped.result, case .none = skipped.renderMutation else { + return XCTFail("A validated stale event must bypass message reduction.") + } + let newer = NativeThreadDetailReducer.apply(event, to: thread, afterSequence: 1) + guard case .updated = newer.result else { return XCTFail("New events must still reduce.") } + } + + func testStaleDetailReplayPreservesNewerMessageAndExplicitMarker() async throws { + let fixture = try await CatchUpFixture.make(completionMarker: false) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + var values = (0..<100).map { index in + JSONValue.object(["kind": .string("event"), "event": replayMessage( + sequence: Double(index % 3), text: "Must not appear" + )]) + } + // These valid envelopes were ignored by the existing post-reduction cursor guard. + for type in ["future.event", "thread.message-sent"] { + values.append(.object(["kind": .string("event"), "event": .object([ + "type": .string(type), "sequence": .number(2), + "occurredAt": .string("2026-09-02T12:00:00Z"), + "payload": .object(["threadId": .string("first")]), + ])])) + } + values.append(.object(["kind": .string("event"), "event": replayMessage( + sequence: 3, text: "New content" + )])) + values.append(.object(["kind": .string("synchronized")])) + try await stream.socket.chunk(id: stream.id, values: values) + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages, ["New content"]) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 1) + await fixture.client.disconnect() + } catch { + await fixture.client.disconnect() + throw error + } + } + + func testMalformedStaleLookingDetailEnvelopesStillRepair() async throws { + var malformed: [JSONValue] = [.null] + for field in ["type", "occurredAt"] { + var object = try replayMessage(sequence: 2, text: "Invalid").decode([String: JSONValue].self) + object.removeValue(forKey: field) + malformed.append(.object(object)) + } + var wrongThread = try replayMessage(sequence: 2, text: "Invalid").decode([String: JSONValue].self) + wrongThread["payload"] = .object(["threadId": .string("another-thread")]) + malformed.append(.object(wrongThread)) + malformed.append(replayMessage(sequence: 1.5, text: "Invalid")) + malformed.append(replayMessage(sequence: -1, text: "Invalid")) + var items = malformed.map { JSONValue.object(["kind": .string("event"), "event": $0]) } + items.append(.object(["kind": .string("unknown")])) + for (index, item) in items.enumerated() { + let fixture = try await CatchUpFixture.make(completionMarker: false) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Recovered \(index)", sequence: 10) + try await stream.socket.chunk(id: stream.id, values: [ + item, .object(["kind": .string("synchronized")]), + ]) + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages, ["Recovered \(index)"], "Malformed case \(index) must repair.") + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } catch { + await fixture.client.disconnect() + throw error + } + } + } + + private func replayMessage(sequence: Double, text: String) -> JSONValue { + .object([ + "type": .string("thread.message-sent"), "sequence": .number(sequence), + "occurredAt": .string("2026-09-02T12:00:00Z"), "payload": .object([ + "threadId": .string("first"), "messageId": .string("replay-message"), + "role": .string("assistant"), "text": .string(text), "streaming": .bool(false), + "createdAt": .string("2026-09-02T12:00:00Z"), + "updatedAt": .string("2026-09-02T12:00:00Z"), + ]), + ]) + } + func testLegacyReplayPublishesOncePerReceivedBatchAndResumesAfterAppliedEvents() async throws { for batchSize in [1, 16, 500] { let fixture = try await CatchUpFixture.make(completionMarker: false) @@ -187,6 +1141,286 @@ final class NativeThreadCatchUpTests: XCTestCase { await fixture.client.disconnect() } + func testLegacyBufferedFinalPrecedesShellDone() async throws { + for shellSequence in [3, 10] { + let clock = CatchUpPublicationClock() + let fixture = try await CatchUpFixture.make( + completionMarker: nil, detailPublicationSleep: { try await clock.wait() } + ) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + var entries = clock.entries.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.sendBurst(count: 1, includeMarker: false) + guard await entries.next(isolation: #isolation) != nil else { throw CancellationError() } + try await shell.completeShell(sequence: shellSequence, assistantMessageID: "burst-message") + var reachedDone = false + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + guard value.thread.id == fixture.firstID else { continue } + if value.thread.state == .completed { + XCTAssertEqual(value.messages.last?.text, "x") + reachedDone = true + } + default: continue + } + if reachedDone { break } + } + try Task.checkCancellation() + XCTAssertTrue(reachedDone) + } catch { + await clock.release() + await fixture.client.disconnect() + throw error + } + await clock.release() + await fixture.client.disconnect() + } + } + + func testDelayedCompletedShellPreservesNewerRunningDetailAndBackgroundLiveness() async throws { + for background: String? in [nil, "working"] { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + try await shell.completeShell(sequence: 10, assistantMessageID: nil, title: "Shell 10") + while let event = await events.next(isolation: #isolation) { + if case let .snapshot(snapshot) = event, + snapshot.threads.contains(where: { $0.id == fixture.firstID && $0.title == "Shell 10" }) { break } + } + try Task.checkCancellation() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.runningTurnSnapshot(sequence: 12) + var latest: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): latest = value + case .threadSync(fixture.firstID, .live): break + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(latest?.thread.state, .working) + let startedAt = try XCTUnwrap(latest?.thread.workingStartedAt) + try await shell.completeShell( + sequence: 11, assistantMessageID: nil, backgroundLiveness: background, title: "Shell 11" + ) + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + latest = value + XCTAssertEqual(value.thread.state, .working) + XCTAssertEqual(value.thread.workingStartedAt, startedAt) + default: break + } + if case let .snapshot(snapshot) = event, + snapshot.threads.contains(where: { $0.id == fixture.firstID && $0.title == "Shell 11" }) { break } + } + try Task.checkCancellation() + // The published shell title proves that the delayed shell was applied + // before this detail marker. + try await detail.synchronize() + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + latest = value + XCTAssertEqual(value.thread.state, .working) + XCTAssertEqual(value.thread.workingStartedAt, startedAt) + case .threadSync(fixture.firstID, .live): break + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + try Task.checkCancellation() + XCTAssertEqual(latest?.thread.state, .working) + XCTAssertEqual(latest?.backgroundWorkIsActive, background == "working") + } catch { + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + } + + func testShellCompletionRepairsMissingOrStreamingFinalWithoutClosingDetailStream() async throws { + for mode in ["missing", "streaming", "no-message-id", "no-message-id-streaming", "no-message-id-completed"] { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + if mode == "streaming" || mode == "no-message-id-streaming" { + await fixture.http.setCompletionResponse(text: "Partial", sequence: 2, streaming: true) + } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + if mode == "no-message-id-streaming" || mode == "no-message-id-completed" { + try await detail.completeTurnWithoutMessageID(sequence: 3) + } + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + + await fixture.http.setCompletionResponse(text: "Final assistant response", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Completion starts a required snapshot: \(mode)") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + try await shell.completeShell(sequence: 10, assistantMessageID: mode.hasPrefix("no-message-id") ? nil : "answer-0") + // Failure watchdog only: successful progress is signaled by the held HTTP read. + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let read = await waiting.value else { + await fixture.client.disconnect() + continue + } + read.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final assistant response"], mode) + + // The same subscription remains usable after HTTP repair. + try await detail.sendMessage(text: "Next response", sequence: 11) + try await detail.synchronize() + let next = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(next, ["Final assistant response", "Next response"], mode) + try await shell.completeShell( + sequence: 30, assistantMessageID: mode.hasPrefix("no-message-id") ? nil : "answer-0", + activeOrderKey: "already-complete" + ) + while let event = await events.next(isolation: #isolation) { + if case .threadSync(fixture.firstID, .catchingUp) = event { + XCTFail("Already-complete detail must not start another repair") + } + if case let .detail(value) = event, value.thread.activeOrderKey == "already-complete" { break } + if case let .detailDelta(value, _) = event, value.thread.activeOrderKey == "already-complete" { break } + } + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 2, mode) + await fixture.client.disconnect() + } + } + + func testShellCompletionCoalescesWhileRequiredSnapshotTracksNewerDetailEvents() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setCompletionResponse(text: "Final", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Completion starts required read") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + try await shell.completeShell(sequence: 10, assistantMessageID: "answer-0") + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let first = await waiting.value else { + await fixture.client.disconnect() + return + } + await nextCatchUp(&events, threadID: fixture.firstID) + + // An unrelated shell cursor advance must not invalidate the same completion twice. + try await shell.completeShell(sequence: 100, assistantMessageID: "answer-0", activeOrderKey: "completion-seen-again") + while let event = await events.next(isolation: #isolation) { + if case let .detail(value) = event, value.thread.activeOrderKey == "completion-seen-again" { break } + if case let .detailDelta(value, _) = event, value.thread.activeOrderKey == "completion-seen-again" { break } + } + await fixture.http.setCompletionResponse(text: "Final with newer detail", sequence: 20) + try await detail.sendMessage(text: "Final with newer detail", sequence: 20) + await nextCatchUp(&events, threadID: fixture.firstID) + var reads = fixture.http.heldRequests.makeAsyncIterator() + first.succeed() + let replacement = try await nextHeldRead(&reads) + replacement.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final with newer detail"]) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 3, "Initial read plus one stale repair and one cursor replacement") + await fixture.client.disconnect() + } + + func testWarmCachedOpenRepairsAlreadyCompletedShellWithoutAnotherShellEvent() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let first = try await nextThreadRequest(&requests) + try await first.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + fixture.client.releaseThread(id: fixture.firstID) + try await shell.completeShell(sequence: 10, assistantMessageID: "answer-0", activeOrderKey: "cached-completion") + while let event = await events.next(isolation: #isolation) { + if case let .thread(value) = event, value.activeOrderKey == "cached-completion" { break } + if case let .snapshot(value) = event, + value.threads.contains(where: { $0.activeOrderKey == "cached-completion" }) { break } + } + await fixture.http.setCompletionResponse(text: "Final while closed", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Cached completion starts required read on reopen") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + _ = try await fixture.client.loadThread(id: fixture.firstID) + let resumed = try await nextThreadRequest(&requests) + try await resumed.synchronize() + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let read = await waiting.value else { + await fixture.client.disconnect() + return + } + // Ignore the warm-cache immediate live receipt preceding the repair. + while let event = await events.next(isolation: #isolation) { + if case .threadSync(fixture.firstID, .catchingUp) = event { break } + } + read.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final while closed"]) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 2) + await fixture.client.disconnect() + } + + private func nextShellRequest( + _ iterator: inout AsyncStream.Iterator + ) async throws -> CatchUpRequest { + while let request = await iterator.next(isolation: #isolation) { + if request.tag == RPCMethod.subscribeShell.rawValue { return request } + } + throw CancellationError() + } + func testRequestSnapshotsKeepTerminalRequestsClosedAndOtherFailuresRetryable() async throws { var activities: [OrchestrationActivity] = [] for kind in ["approval", "user-input"] { @@ -444,6 +1678,34 @@ final class NativeThreadCatchUpTests: XCTestCase { await fixture.client.disconnect() } + func testWarmReopenClearsAnInterruptedOlderPageLoadingState() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + await fixture.http.setPage(.init(beforeCursor: "older", hasMore: true, snapshotSequence: 2)) + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + var reads = fixture.http.heldRequests.makeAsyncIterator() + let older = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let held = try await nextHeldRead(&reads) + fixture.client.releaseThread(id: fixture.firstID) + let restored = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(restored.page?.isLoading, false) + XCTAssertEqual(restored.page?.hasMore, true) + held.succeed() + _ = try await older.value + await fixture.http.holdThreadReads(false) + await fixture.http.setPage(.init(beforeCursor: nil, hasMore: false, snapshotSequence: 2)) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 3, "Reopened history must allow another older-page request") + await fixture.client.disconnect() + } + func testWarmNavigationResumesAfterAppliedMessagesWithoutAnotherHTTPRead() async throws { let fixture = try await CatchUpFixture.make() defer { fixture.cleanUp() } @@ -1116,8 +2378,15 @@ private struct CatchUpFixture { var secondID: String { FeatureScopedID.thread(environmentID: "one", wireID: "second") } static func make( - completionMarker: Bool = true, + completionMarker: Bool? = true, + includePeer: Bool = false, + reconciliationClock: SelectedReconciliationClock? = nil, + reconciliationReceipt: @escaping @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void = { _ in }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, activities: [OrchestrationActivity] = [], + detailPublicationSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(80)) + }, threadRetryDelay: @escaping @Sendable (Int) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(250)) } @@ -1127,7 +2396,10 @@ private struct CatchUpFixture { try await store.save([Environment( id: "one", label: "Computer", httpBaseURL: URL(string: "https://one.example")!, webSocketBaseURL: URL(string: "wss://one.example/ws")! - )]) + )] + (includePeer ? [Environment( + id: "two", label: "Peer", httpBaseURL: URL(string: "https://two.example")!, + webSocketBaseURL: URL(string: "wss://two.example/ws")! + )] : [])) try await store.setActiveEnvironment(id: "one") let http = CatchUpHTTPTransport() await http.setActivities(activities) @@ -1135,20 +2407,40 @@ private struct CatchUpFixture { let delay = CatchUpDelay() let runtime = EnvironmentRuntime( environmentStore: store, - credentialStore: InMemoryCredentialStore(credentials: ["one": .init(accessToken: "test")]), + credentialStore: InMemoryCredentialStore(credentials: ["one": .init(accessToken: "test"), "two": .init(accessToken: "test")]), httpTransport: http, webSocketConnector: CatchUpConnector( requests: requests.continuation, completionMarker: completionMarker ) ) + let readiness = CatchUpBootstrapReadiness() + let peerReadiness = CatchUpBootstrapReadiness() let client = NativeFeatureClient( runtime: runtime, settingsStore: UserDefaults(suiteName: UUID().uuidString)!, fallbackPollingInitialDelay: .seconds(3_600), aggregateRefreshInterval: .seconds(3_600), + aggregateRefreshReceipt: { + readiness.record($0) + switch $0 { + case let .shellApplied("two", sequence): peerReadiness.record(.shellApplied(environmentID: "one", sequence: sequence)) + case .configurationApplied("two"): peerReadiness.record(.configurationApplied(environmentID: "one")) + default: break + } + aggregateRefreshReceipt($0) + }, + detailPublicationSleep: detailPublicationSleep, catchUpDelay: { try await delay.wait() }, + selectedThreadReconciliationSleep: { duration in + if let reconciliationClock { try await reconciliationClock.sleep(for: duration) } + else { try await Task.sleep(for: duration) } + }, + selectedThreadReconciliationNow: { reconciliationClock?.now ?? .now }, + selectedThreadReconciliationReceipt: reconciliationReceipt, threadRetryDelay: threadRetryDelay ) _ = try await client.initialSnapshot() + try await readiness.wait() + if includePeer { try await peerReadiness.wait() } return Self(client: client, http: http, requests: requests.stream, delay: delay, directory: directory) } @@ -1156,10 +2448,30 @@ private struct CatchUpFixture { } private actor CatchUpHTTPTransport: HTTPTransport { + func setFaultInjectionMessages(paginated: Bool) { + let authoritative = [3, 4].map { index in + OrchestrationMessage(id: "message-\(index)", role: "assistant", + text: index == 3 ? "Omitted message" : "Later delivered message", + attachments: [], turnId: nil, streaming: false, + createdAt: "2026-09-02T12:00:00Z", updatedAt: "2026-09-02T12:00:00Z") + } + messages = authoritative + paginatedMessages = paginated ? authoritative : nil + sequence = 4 + } + + private(set) var threadRequests: [URLRequest] = [] + private(set) var threadResponseBytes: [Int] = [] private var messages: [OrchestrationMessage] = [] + private var paginatedMessages: [OrchestrationMessage]? + private var rawTurnCap: Int? + private var rawCollection: String? + private var pendingHeldReads: [CatchUpHTTPRead] = [] private var activities: [OrchestrationActivity] = [] - private var page: OrchestrationThreadDetailPage? + private var completionResponse = false + private var responsePage: OrchestrationThreadDetailPage? + private var runningResponse = false private var sequence = 2 private var holdsThreadReads = false private let heldReadContinuation: AsyncStream.Continuation @@ -1171,6 +2483,47 @@ private actor CatchUpHTTPTransport: HTTPTransport { heldReadContinuation = reads.continuation } + func setRawFanoutMessages(sequence: Int, userSlots: Set = [0, 200], collection: String? = nil) { + rawTurnCap = 150 + rawCollection = collection + paginatedMessages = (0..<300).map { index in + let stamp = String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60) + return OrchestrationMessage(id: "raw-\(index)", role: userSlots.contains(index) ? "user" : "assistant", + text: "Raw \(index)", attachments: [], turnId: userSlots.contains(index) ? nil : "turn-\(index)", + streaming: false, createdAt: stamp, updatedAt: stamp) + } + self.sequence = sequence + } + + func setPaginatedUserMessages(_ range: Range, sequence: Int) { + paginatedMessages = range.map { index in + OrchestrationMessage( + id: "user-\(index)", role: "user", text: "User \(index)", attachments: [], + turnId: nil, streaming: false, + createdAt: String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60), + updatedAt: String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60) + ) + } + self.sequence = sequence + } + func removePaginatedUser(id: String, sequence: Int) { + paginatedMessages?.removeAll { $0.id == id } + self.sequence = sequence + } + func editLatestUserText(_ text: String, sequence: Int) { + guard let latest = paginatedMessages?.popLast() else { return } + paginatedMessages?.append(OrchestrationMessage( + id: latest.id, role: latest.role, text: text, attachments: latest.attachments, + turnId: latest.turnId, streaming: latest.streaming, createdAt: latest.createdAt, + updatedAt: latest.updatedAt + )) + self.sequence = sequence + } + func cancelHeldReads() { + pendingHeldReads.forEach { $0.fail() } + pendingHeldReads.removeAll() + } + func setResponse(text: String, sequence: Int, attachment: ChatAttachment? = nil) { messages = [catchUpMessage(text, index: 0, attachment: attachment)] self.sequence = sequence @@ -1183,11 +2536,26 @@ private actor CatchUpHTTPTransport: HTTPTransport { self.sequence = sequence } + func setRunningResponse(text: String, sequence: Int) { + setResponse(text: text, sequence: sequence) + runningResponse = true + } + + func setCompletionResponse(text: String, sequence: Int, streaming: Bool = false) { + messages = [OrchestrationMessage( + id: "answer-0", role: "assistant", text: text, attachments: [], turnId: "turn-1", + streaming: streaming, createdAt: "2026-09-02T12:00:00Z", updatedAt: "2026-09-02T12:00:00Z" + )] + self.sequence = sequence + completionResponse = !streaming + } + + func setPage(_ page: OrchestrationThreadDetailPage) { responsePage = page } + func holdThreadReads(_ hold: Bool) { holdsThreadReads = hold } func setActivities(_ activities: [OrchestrationActivity]) { self.activities = activities } - func setPage(_ page: OrchestrationThreadDetailPage) { self.page = page } func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { let value: JSONValue @@ -1206,19 +2574,77 @@ private actor CatchUpHTTPTransport: HTTPTransport { throw URLError(.unsupportedURL) } threadRequests.append(request) + var responseMessages = messages + var page: OrchestrationThreadDetailPage? + if let paginatedMessages { + let query = URLComponents(url: request.url!, resolvingAgainstBaseURL: false)?.queryItems ?? [] + let limit = query.first { $0.name == "turnLimit" }?.value.flatMap(Int.init) + let cursor = query.first { $0.name == "beforeCursor" }?.value + let end = cursor.flatMap { value in paginatedMessages.firstIndex { $0.id == value } } ?? paginatedMessages.count + var start = max(0, end - (limit ?? end)) + if let cap = rawTurnCap, let limit { + // One fixture message per raw turn. Match server candidates LIMIT150, + // then walk descending until the user-anchored turn limit is reached. + start = max(0, end - cap) + var usersSeen = 0 + for index in stride(from: end - 1, through: start, by: -1) { + if paginatedMessages[index].role == "user" { usersSeen += 1 } + if usersSeen == limit { start = index; break } + } + } + responseMessages = Array(paginatedMessages[start.. 0, + snapshotSequence: sequence, threadSequence: sequence + ) + } + } let snapshot = multiEnvironmentDetail( projectID: "project", threadID: request.url!.lastPathComponent, - snapshotSequence: sequence, messages: messages + snapshotSequence: sequence, messages: responseMessages ) var thread = snapshot.thread thread.activities = activities + if let rawCollection { + thread.messages = [] + if rawCollection == "activities" { + thread.activities = responseMessages.map { row in + OrchestrationActivity(id: "activity-\(row.id)", tone: "info", kind: "fixture.raw-turn", + summary: row.text, payload: .null, turnId: "turn-\(row.id)", sequence: nil, createdAt: row.createdAt) + } + } else { + // Server checkpoints are unwindowed even when messages/activities + // hit the raw-turn cap. Keep this fixture faithful to that contract. + thread.checkpoints = (paginatedMessages ?? responseMessages).enumerated().map { offset, row in + CheckpointSummary(turnId: "turn-\(row.id)", checkpointTurnCount: offset, + checkpointRef: row.text, status: "completed", files: [], assistantMessageId: nil, completedAt: row.createdAt) + } + } + } + if runningResponse { + var object = try JSONValue.encode(thread).decode([String: JSONValue].self) + object["latestTurn"] = .object([ + "turnId": .string("synthetic-turn"), "state": .string("running"), + "requestedAt": .string("2026-09-02T12:00:00Z"), "startedAt": .string("2026-09-02T12:00:00Z"), + "completedAt": .null, "assistantMessageId": .null + ]) + object["session"] = catchUpRunningSession(threadID: thread.id, turnID: "synthetic-turn") + thread = try JSONValue.object(object).decode(OrchestrationThread.self) + } + if completionResponse { + var object = try JSONValue.encode(thread).decode([String: JSONValue].self) + object["latestTurn"] = catchUpCompletedTurn(assistantMessageID: "answer-0") + thread = try JSONValue.object(object).decode(OrchestrationThread.self) + } value = try .encode(OrchestrationThreadDetailSnapshot( - snapshotSequence: snapshot.snapshotSequence, thread: thread, page: page ?? snapshot.page + snapshotSequence: snapshot.snapshotSequence, thread: thread, page: responsePage ?? page ?? snapshot.page )) } let response = (try JSONEncoder.t3.encode(value), HTTPURLResponse( url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: nil )!) + if request.url!.path.hasPrefix("/api/orchestration/threads/") { threadResponseBytes.append(response.0.count) } if holdsThreadReads, request.url!.path.hasPrefix("/api/orchestration/threads/") { let finished = AsyncStream.makeStream() defer { @@ -1226,21 +2652,30 @@ private actor CatchUpHTTPTransport: HTTPTransport { finished.continuation.finish() } return try await withCheckedThrowingContinuation { continuation in - heldReadContinuation.yield(CatchUpHTTPRead( - response: response, continuation: continuation, finished: finished.stream - )) + let read = CatchUpHTTPRead(response: response, continuation: continuation, finished: finished.stream) + pendingHeldReads.append(read) + heldReadContinuation.yield(read) } } return response } } -private struct CatchUpHTTPRead: Sendable { +private final class CatchUpHTTPRead: @unchecked Sendable { let response: (Data, HTTPURLResponse) - let continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error> + private var continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error>? + private let lock = NSLock() let finished: AsyncStream - func succeed() { continuation.resume(returning: response) } - func fail() { continuation.resume(throwing: URLError(.notConnectedToInternet)) } + init(response: (Data, HTTPURLResponse), continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error>, finished: AsyncStream) { + self.response = response; self.continuation = continuation; self.finished = finished + } + private func take() -> CheckedContinuation<(Data, HTTPURLResponse), any Error>? { + lock.lock(); defer { lock.unlock() } + defer { continuation = nil } + return continuation + } + func succeed() { take()?.resume(returning: response) } + func fail() { take()?.resume(throwing: URLError(.notConnectedToInternet)) } } private func catchUpMessage( @@ -1258,9 +2693,9 @@ private func catchUpMessage( private struct CatchUpConnector: WebSocketConnecting { let requests: AsyncStream.Continuation - let completionMarker: Bool + let completionMarker: Bool? func connect(to url: URL) async throws -> any WebSocketConnection { - CatchUpSocket(requests: requests, completionMarker: completionMarker) + CatchUpSocket(requests: requests, completionMarker: completionMarker, publishInitialShell: url.host == "two.example") } } @@ -1288,6 +2723,35 @@ private struct CatchUpRequest: Sendable { ]) } + func completeShell(sequence: Int, assistantMessageID: String?, activeOrderKey: String? = nil, backgroundLiveness: String? = nil, title: String = "First") async throws { + let shell = multiEnvironmentShell(projectID: "project", threadID: "first", title: title) + var thread = try JSONValue.encode(shell.threads[0]).decode([String: JSONValue].self) + thread["latestTurn"] = catchUpCompletedTurn(assistantMessageID: assistantMessageID) + thread["activeOrderKey"] = activeOrderKey.map(JSONValue.string) + thread["backgroundLiveness"] = backgroundLiveness.map(JSONValue.string) + let snapshot = OrchestrationShellSnapshot( + snapshotSequence: sequence, projects: shell.projects, + threads: [try JSONValue.object(thread).decode(OrchestrationThreadShell.self)], updatedAt: shell.updatedAt + ) + try await socket.chunk(id: id, values: [.object([ + "kind": .string("snapshot"), "snapshot": try .encode(snapshot), + ])]) + } + + func completeTurnWithoutMessageID(sequence: Int) async throws { + try await socket.chunk(id: id, values: [.object([ + "kind": .string("event"), "event": .object([ + "type": .string("thread.turn-diff-completed"), "sequence": .number(Double(sequence)), + "occurredAt": .string("2026-09-02T12:01:00Z"), "payload": .object([ + "threadId": payload["threadId"]!, "turnId": .string("turn-1"), + "checkpointTurnCount": .number(1), "checkpointRef": .string("refs/t3/checkpoints/turn-1"), + "status": .string("ready"), "files": .array([]), + "completedAt": .string("2026-09-02T12:01:00Z"), "assistantMessageId": .null, + ]), + ]), + ])]) + } + func terminate(_ failure: CatchUpStreamFailure) async throws { switch failure { case .malformed: @@ -1342,6 +2806,45 @@ private struct CatchUpRequest: Sendable { ])]) } + func runningTurnSnapshot(sequence: Int) async throws { + let snapshot = multiEnvironmentDetail( + projectID: "project", threadID: payload["threadId"]!.stringValue!, snapshotSequence: sequence + ) + var thread = try JSONValue.encode(snapshot.thread).decode([String: JSONValue].self) + thread["latestTurn"] = .object([ + "turnId": .string("turn-2"), "state": .string("running"), + "requestedAt": .string("2026-09-02T12:02:00Z"), + "startedAt": .string("2026-09-02T12:02:00Z"), + "completedAt": .null, "assistantMessageId": .null, + ]) + thread["session"] = catchUpRunningSession(threadID: payload["threadId"]!.stringValue!, turnID: "turn-2") + let updated = OrchestrationThreadDetailSnapshot( + snapshotSequence: sequence, thread: try JSONValue.object(thread).decode(OrchestrationThread.self) + ) + try await socket.chunk(id: id, values: [ + .object(["kind": .string("snapshot"), "snapshot": try .encode(updated)]), + ]) + } + + func sendBurst(count: Int, includeMarker: Bool) async throws { + var values: [JSONValue] = (1...count).map { index in + .object([ + "kind": .string("event"), "event": .object([ + "type": .string("thread.message-sent"), "sequence": .number(Double(index + 2)), + "occurredAt": .string("2026-09-02T12:00:00Z"), "payload": .object([ + "threadId": payload["threadId"]!, "messageId": .string("burst-message"), + "role": .string("assistant"), "text": .string(index < count ? "x" : String(repeating: "x", count: count)), + "streaming": .bool(index < count), + "createdAt": .string("2026-09-02T12:00:00Z"), + "updatedAt": .string("2026-09-02T12:00:00Z"), + ]), + ]), + ]) + } + if includeMarker { values.append(.object(["kind": .string("synchronized")])) } + try await socket.chunk(id: id, values: values) + } + func sendMessage(text: String, sequence: Int) async throws { try await socket.chunk(id: id, values: [.object([ "kind": .string("event"), "event": .object([ @@ -1359,15 +2862,18 @@ private struct CatchUpRequest: Sendable { private actor CatchUpSocket: WebSocketConnection { let requests: AsyncStream.Continuation - let completionMarker: Bool + let completionMarker: Bool? private(set) var assetRequestCount = 0 private var pending: [Data] = [] private var receiver: CheckedContinuation? private var closed = false - init(requests: AsyncStream.Continuation, completionMarker: Bool) { + private let publishInitialShell: Bool + + init(requests: AsyncStream.Continuation, completionMarker: Bool?, publishInitialShell: Bool = false) { self.requests = requests self.completionMarker = completionMarker + self.publishInitialShell = publishInitialShell } func send(_ data: Data) throws { @@ -1377,13 +2883,20 @@ private actor CatchUpSocket: WebSocketConnection { try enqueue(.object(["_tag": .string("Pong")])) } guard let tag = request["tag"]?.stringValue, case let .number(id) = request["id"] else { return } + if publishInitialShell, tag == RPCMethod.subscribeShell.rawValue { + let snapshot = multiEnvironmentShell(projectID: "project", threadID: "first", title: "Peer") + try chunk(id: Int(id), values: [.object([ + "kind": .string("snapshot"), "snapshot": try .encode(snapshot) + ])]) + } if tag == RPCMethod.assetsCreateURL.rawValue { assetRequestCount += 1 } if tag == RPCMethod.subscribeServerConfig.rawValue { + var config: [String: JSONValue] = [ + "providers": .array([]), "threadSnapshotPagination": .bool(true), + ] + if let completionMarker { config["threadResumeCompletionMarker"] = .bool(completionMarker) } try chunk(id: Int(id), values: [.object([ - "type": .string("snapshot"), "config": .object([ - "providers": .array([]), "threadSnapshotPagination": .bool(true), - "threadResumeCompletionMarker": .bool(completionMarker), - ]), + "type": .string("snapshot"), "config": .object(config), ])]) } requests.yield(.init(tag: tag, id: Int(id), payload: request["payload"]!, socket: self)) @@ -1506,3 +3019,109 @@ private actor CatchUpDelay { waiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) } } + +/// The session, not `latestTurn`, marks a thread as working. +private func catchUpRunningSession(threadID: String, turnID: String) -> JSONValue { + .object([ + "threadId": .string(threadID), "status": .string("running"), + "providerName": .string("codex"), "providerInstanceId": .null, + "runtimeMode": .string("full-access"), "activeTurnId": .string(turnID), + "lastError": .null, "updatedAt": .string("2026-09-02T12:02:00Z"), + ]) +} + +private func catchUpCompletedTurn(assistantMessageID: String?) -> JSONValue { + .object([ + "turnId": .string("turn-1"), "state": .string("completed"), + "requestedAt": .string("2026-09-02T12:00:00Z"), + "startedAt": .string("2026-09-02T12:00:00Z"), + "completedAt": .string("2026-09-02T12:01:00Z"), + "assistantMessageId": assistantMessageID.map(JSONValue.string) ?? .null, + ]) +} + +private final class CatchUpPublicationClock: Sendable { + private let gate = CatchUpDelay() + private let receipt: AsyncStream.Continuation + let entries: AsyncStream + + init() { + let pair = AsyncStream.makeStream() + entries = pair.stream + receipt = pair.continuation + } + + func wait() async throws { + receipt.yield(()) + try await gate.wait() + } + + func release() async { await gate.release() } +} + +@MainActor +private final class CatchUpBootstrapReadiness { + private var hasShell = false + private var hasConfig = false + private var waiters: [UUID: CheckedContinuation] = [:] + + func record(_ receipt: NativePassiveShellReceipt) { + switch receipt { + case .shellApplied(environmentID: "one", sequence: _): hasShell = true + case .configurationApplied(environmentID: "one"): hasConfig = true + default: break + } + if hasShell && hasConfig { + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.resume() } + } + } + + func wait() async throws { + try Task.checkCancellation() + guard !hasShell || !hasConfig else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { waiters[id] = $0 } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } + } + } +} +@MainActor +private final class SelectedReconciliationClock { + var now = ContinuousClock.now + let requests = AsyncStream.makeStream() + private var pending: [UUID: CheckedContinuation] = [:] + struct Wait: Sendable { + let duration: Duration + let release: @MainActor @Sendable () -> Void + } + func advance(by duration: Duration) { now = now.advanced(by: duration) } + func cancelAll() { + let waits = pending.values + pending.removeAll() + waits.forEach { $0.resume(throwing: CancellationError()) } + requests.continuation.finish() + } + func sleep(for duration: Duration) async throws { + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else { + pending[id] = continuation + requests.continuation.yield(Wait(duration: duration, release: { [weak self] in + self?.pending.removeValue(forKey: id)?.resume() + })) + } + } + pending.removeValue(forKey: id) + } onCancel: { Task { @MainActor [weak self] in + self?.pending.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } } + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeTranscriptTimelineTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeTranscriptTimelineTests.swift new file mode 100644 index 000000000000..6eb81175d286 --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/NativeTranscriptTimelineTests.swift @@ -0,0 +1,151 @@ +import Foundation +import Testing +@testable import T3Code + +@MainActor +@Suite("Chronological tool activity") +struct NativeTranscriptTimelineTests { + private func text(_ id: String, at time: Double) -> FeatureMessage { + FeatureMessage(id: id, role: .assistant, text: id, createdAt: Date(timeIntervalSince1970: time)) + } + + private func activity( + _ id: String, call: String? = "a", at time: Double, + kind: String = "tool.updated", turn: String = "turn", sequence: Int? = nil + ) -> OrchestrationActivity { + var payload: [String: JSONValue] = ["title": .string(id)] + if let call { payload["toolCallId"] = .string(call) } + return OrchestrationActivity( + id: id, tone: "info", kind: kind, summary: id, payload: .object(payload), + turnId: turn, sequence: sequence, + createdAt: Date(timeIntervalSince1970: time).ISO8601Format() + ) + } + + @Test func questionAnswersSurviveSnapshotAndLiveTimelineMapping() { + let answer = OrchestrationActivity( + id: "answer", tone: "info", kind: "user-input.answer-submitted", + summary: "Answer submitted", payload: .object([ + "answers": .object(["q": .string("Blue")]), + "questionTextById": .object(["q": .string("Choose a color")]), + "attachmentsByQuestionId": .object([:]), + ]), turnId: "turn", sequence: 2, + createdAt: Date(timeIntervalSince1970: 2).ISO8601Format() + ) + let snapshot = NativeTranscriptTimeline(messages: [], activities: [answer], sessionIsLive: false) + var live = NativeTranscriptTimeline() + live.append(answer) + #expect(snapshot.messages.map(\.id) == ["question-answer:answer:q"]) + #expect(live.messages.map(\.id) == snapshot.messages.map(\.id)) + #expect(live.messages.first?.text == "Choose a color\n\nBlue") + #expect(live.changedIDs.contains("question-answer:answer:q")) + } + + @Test func adjacentSameCallCoalescesAtActualLatestTime() { + var timeline = NativeTranscriptTimeline() + timeline.append(activity("start", at: 1, kind: "tool.started")) + timeline.append(activity("progress", at: 2)) + #expect(timeline.messages.count == 1) + #expect(timeline.messages[0].id == "work-log-start") + #expect(timeline.messages[0].createdAt == Date(timeIntervalSince1970: 2)) + #expect(timeline.messages[0].activeWorkLabel == "progress") + timeline.append(activity("end", at: 3, kind: "tool.completed")) + #expect(timeline.messages.count == 1) + #expect(timeline.messages[0].toolName == "end") + #expect(timeline.messages[0].activeWorkLabel == nil) + } + + @Test func textAndOtherCallsCloseRowsAndLateCompletionAppends() { + var timeline = NativeTranscriptTimeline() + timeline.append(text("text1", at: 1)) + timeline.append(activity("startA", at: 2)) + timeline.append(text("text2", at: 3)) + let prefix = timeline.messages + timeline.append(activity("updateA", at: 4)) + timeline.append(activity("startB", call: "b", at: 5)) + let earlier = timeline.messages + timeline.append(activity("lateA", at: 6, kind: "tool.completed")) + #expect(Array(timeline.messages.prefix(3)) == prefix) + #expect(Array(timeline.messages.prefix(4)) == Array(earlier.prefix(4))) + #expect(timeline.messages.map(\.id) == [ + "text1", "work-log-startA", "text2", "work-log-updateA", "work-log-startB", "work-log-lateA", + ]) + #expect(timeline.messages.allSatisfy { $0.activeWorkLabel == nil }) + #expect(timeline.messages.last?.createdAt == Date(timeIntervalSince1970: 6)) + } + + @Test func fullSnapshotMatchesIncrementalChronology() { + let first = text("text1", at: 1) + let second = text("text2", at: 3) + let activities = [activity("start", at: 2), activity("update", at: 4), + activity("more", at: 5), activity("other", call: "b", at: 6)] + let snapshot = NativeTranscriptTimeline(messages: [first, second], activities: activities, sessionIsLive: true) + var live = NativeTranscriptTimeline() + live.append(first) + live.append(activities[0]) + live.append(second) + for item in activities.dropFirst() { live.append(item) } + #expect(snapshot.messages == live.messages) + live.append(activities[0]) + #expect(snapshot.messages == live.messages) + } + + @Test func missingCallIdentityAndDifferentTurnsNeverCoalesce() { + var timeline = NativeTranscriptTimeline() + timeline.append(activity("no-id-1", call: nil, at: 1)) + timeline.append(activity("no-id-2", call: nil, at: 2)) + timeline.append(activity("a1", at: 3)) + timeline.append(activity("a2", at: 4, turn: "other")) + #expect(timeline.messages.count == 4) + } + + @Test func noticesSeparateCallsAndInactiveSnapshotHasNoLiveLabels() { + let events = [activity("start", at: 1), activity("warning", at: 2, kind: "runtime.warning"), + activity("update", at: 3)] + let timeline = NativeTranscriptTimeline(messages: [], activities: events, sessionIsLive: false) + #expect(timeline.messages.map(\.role) == [.tool, .system, .tool]) + #expect(timeline.messages.allSatisfy { $0.activeWorkLabel == nil }) + #expect(timeline.messages.map(\.createdAt) == [1.0, 2.0, 3.0].map(Date.init(timeIntervalSince1970:))) + } + + @Test func snapshotTiesAreExplicitlyDeterministicAndLiveTiesKeepArrivalOrder() { + let message = text("text", at: 1) + let first = activity("first", at: 1, sequence: 10) + let second = activity("second", call: "b", at: 1, sequence: 11) + let snapshot = NativeTranscriptTimeline(messages: [message], activities: [second, first], sessionIsLive: true) + #expect(snapshot.messages.map(\.id) == ["text", "work-log-first", "work-log-second"]) + var live = NativeTranscriptTimeline() + live.append(first) + live.append(message) + live.append(second) + #expect(live.messages.map(\.id) == ["work-log-first", "text", "work-log-second"]) + } + + @Test func mixedLiveBatchRetainsBoundariesBetweenRepeatedMessageUpdates() { + let message = OrchestrationMessage(id: "text", role: "assistant", text: "hello", attachments: nil, + turnId: "turn", streaming: true, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z") + let tool = activity("tool", at: 1) + var batch = NativeDetailRenderMutations() + batch.formUnion(.message(message)) + batch.formUnion(.message(message)) + batch.formUnion(.activity(tool)) + batch.formUnion(.message(message)) + #expect(batch.ordered == [.message(message), .activity(tool), .message(message)]) + } + + @Test func receiptGateScopesSelectionAndLimitsOnlySameSecondPublication() { + var gate = NativeThreadReceiptGate() + #expect(gate.receive(threadID: "other", selectedThreadID: "selected", generation: 1, + now: Date(timeIntervalSince1970: 1), source: .detailEvent) == nil) + #expect(gate.receive(threadID: "selected", selectedThreadID: "selected", generation: 1, + now: Date(timeIntervalSince1970: 1), source: .detailEvent) != nil) + #expect(gate.receive(threadID: "selected", selectedThreadID: "selected", generation: 1, + now: Date(timeIntervalSince1970: 1.9), source: .detailEvent) == nil) + #expect(gate.receive(threadID: "selected", selectedThreadID: "selected", generation: 2, + now: Date(timeIntervalSince1970: 1.9), source: .detailSnapshot) != nil) + #expect(gate.receive(threadID: "selected", selectedThreadID: "selected", generation: 2, + now: Date(timeIntervalSince1970: 1.9), source: .detailEvent) != nil) + #expect(gate.receive(threadID: "selected", selectedThreadID: "selected", generation: 2, + now: Date(timeIntervalSince1970: 2), source: .shellThreadUpdate) != nil) + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeWorkLogAccumulatorTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeWorkLogAccumulatorTests.swift index ed45301d8840..3ffcf9ef19b3 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeWorkLogAccumulatorTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeWorkLogAccumulatorTests.swift @@ -47,7 +47,7 @@ struct NativeWorkLogAccumulatorTests { ) message = accumulator.message(groupID: "turn-1") #expect(message.activeWorkLabel == nil) - #expect(message.toolName == "Work log · 1") + #expect(message.toolName == "Run focused tests completed") #expect(message.text == "• 2 tests passed") } diff --git a/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift b/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift index 259fdca781fa..71782c698f3c 100644 --- a/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift @@ -39,7 +39,9 @@ final class T3ConnectNativeCapabilityTests: XCTestCase { guard case let .snapshot(snapshot)? = event else { return XCTFail("Managed connect did not publish its initial Home snapshot") } - XCTAssertEqual(snapshot.connection.state, .connected) + // HTTP hydration publishes content before the deliberately blocked live stream. + XCTAssertEqual(snapshot.connection.state, .reconnecting) + XCTAssertEqual(snapshot.environments.first(where: { $0.id == "managed-1" })?.connectionState, .connected) XCTAssertEqual(snapshot.connection.environmentName, "Managed Studio") let saved = try await store.load() diff --git a/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift b/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift new file mode 100644 index 000000000000..3ae6755bf29b --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift @@ -0,0 +1,33 @@ +import Foundation +import Testing +@testable import T3Code + +struct WorkspaceThreadPresentationTests { + @Test(arguments: [false, true], [false, true]) + func presentationTracksCompactColumnWithoutHidingRegularDetail(isCompact: Bool, showsDetail: Bool) { + var selection = WorkspaceThreadSelection() + selection.open("a") + let isHidden = isCompact && !showsDetail + #expect(selection.presentedID(isCompact: isCompact, showsDetailColumn: showsDetail) == (isHidden ? nil : "a")) + selection.reconcilePresentation(isCompact: isCompact, showsDetailColumn: showsDetail) + #expect(selection.selectedID == (isHidden ? nil : "a")) + #expect(selection.highlightedID == "a") + } + + @Test + func compactBackAndRegularToCompactHideCannotResurrectClosedThread() { + var selection = WorkspaceThreadSelection() + selection.open("a") + selection.reconcilePresentation(isCompact: false, showsDetailColumn: false) + #expect(selection.selectedID == "a") + selection.reconcilePresentation(isCompact: true, showsDetailColumn: false) + #expect(selection.selectedID == nil) + selection.reconcilePresentation(isCompact: false, showsDetailColumn: false) + #expect(selection.presentedID(isCompact: false, showsDetailColumn: false) == nil) + #expect(selection.highlightedID == "a") + selection.open("a") + #expect(selection.presentedID(isCompact: true, showsDetailColumn: true) == "a") + selection.close() + #expect(selection.presentedID(isCompact: true, showsDetailColumn: true) == nil) + } +} diff --git a/docs/user/swiftui-mobile.md b/docs/user/swiftui-mobile.md index e0fa8fde9fb9..61f085187572 100644 --- a/docs/user/swiftui-mobile.md +++ b/docs/user/swiftui-mobile.md @@ -81,3 +81,22 @@ or is canceled. Codex file citations open the cited file when available. Artifact templates include a **Use** action. **Use** inserts an editable prompt into the composer. Review or change it before you send it. + +## Stopping a turn + +Tap **Stop** to request cancellation. **Stopping…** remains visible while the app waits for the +thread to confirm its outcome, including after closing and reopening the thread. Repeated taps +will not send duplicate requests. Acceptance of the request does not mean the agent has finished. + +If the connection fails or the app cannot confirm the outcome, it shows **Stop unconfirmed**. +**Retry stop** checks the computer’s current thread status before sending another request. Some +background work does not report a turn outcome, so its stop may remain unconfirmed. + +While the agent works, its activity row shows how long ago the last message was sent, such as +**Last message · 5m ago**. The age advances while the thread is open. + +## Tool activity + +Tool activity stays in transcript order. Consecutive updates to the same tool call share a row; +an update after text or another tool call starts a new row, preserving the earlier activity. +Each tool and activity row shows its latest update’s local date and time on the right.