diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index a011101359bc..62ea4e7c2af7 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -21,6 +21,11 @@ private struct T3ConnectManagedCleanupError: LocalizedError { } } +enum NativeSelectedThreadReconciliationReceipt: Equatable { + case finished(threadID: String) + case deferred(threadID: String) +} + /// Composes the transport-focused Core layer with the UI-focused Features layer. @MainActor final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, @@ -49,13 +54,23 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private let projectFaviconStore: FeatureProjectFaviconStore private let fallbackPollingInitialDelay: Duration private let fallbackPollingInterval: Duration + private static let shellReconciliationInterval: Duration = .seconds(30) + private let shellReconciliationSleep: @Sendable (String, Duration) async throws -> Void private let aggregateRefreshInterval: Duration private let aggregateIdleRefreshInterval: Duration private let aggregateFailureRefreshInterval: Duration private let aggregateRefreshSleep: @Sendable (Duration) async throws -> Void + private let aggregateRefreshReceipt: @MainActor @Sendable (NativePassiveShellReceipt) -> Void + private let aggregateStreamRetrySleep: @Sendable (String, Duration) async throws -> Void + private let aggregatePublishSleep: @Sendable () async throws -> Void + private let aggregatePeerRefreshSleep: @Sendable (String, Duration) async throws -> Void private let environmentShellTimeoutInterval: TimeInterval private let threadSnapshotTimeoutInterval: TimeInterval + private let detailPublicationSleep: @Sendable () async throws -> Void private let catchUpDelay: @Sendable () async throws -> Void + private let selectedThreadReconciliationSleep: @Sendable (Duration) async throws -> Void + private let selectedThreadReconciliationNow: @MainActor @Sendable () -> ContinuousClock.Instant + private let selectedThreadReconciliationReceipt: @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void private let threadRetryDelay: @Sendable (Int) async throws -> Void private let aggregateEnvironmentLoader: @Sendable (EnvironmentRuntime) async throws -> [Environment] private let stream: AsyncStream @@ -104,6 +119,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var pendingTurnSubmissions: [String: PendingTurnSubmission] = [:] private var projectSettingsWriteTask: Task? private var projectSettingsWriteGeneration: UInt64 = 0 + private struct AcceptedCommandRefresh: Sendable { + let id: UUID + let client: T3Client + let generation: Int + let task: Task + var pending = false + var needsDetail: Bool + } + private var acceptedCommandRefreshes: [String: AcceptedCommandRefresh] = [:] private var approvalRoutes: [String: PendingRequestRoute] = [:] private var inputRoutes: [String: PendingRequestRoute] = [:] private var relayDeviceSessionIDs: Set = [] @@ -115,13 +139,32 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var terminalSnapshots: [TerminalKey: FeatureTerminalSnapshot] = [:] // Keep versions unique when a terminal cache is evicted or an environment reconnects. private var terminalLifecycleVersion = 0 + private var foregroundBootstrapID = UUID() + private var activeHydrationTask: Task? + private var activeHydrationID: UUID? + private var activeHydrationPending = false + private var activeHTTPAuthorityRevision = 0 + private var shellConnectionIDsByEnvironmentID: [String: UUID] = [:] + private var activeShellConnectionID: UUID? + private var activeShellEpochHasSnapshot = false + private var activeHasHydrated = false + private var activeStreamIsAuthoritative = false private var pollingTask: Task? private var fallbackPollingTask: Task? + private var shellReconciliationTask: Task? private var configurationTask: Task? private var aggregateRefreshTask: Task? private var aggregateRefreshID: UUID? + private var aggregatePublishTask: Task? + private var isForeground = true + private(set) var aggregateRefreshWorkers: [ + String: (environment: Environment, task: Task) + ] = [:] private var shellPublishTask: Task? private var archivedRefreshTask: Task? + private var selectedThreadReconciliationTask: Task? + private var selectedThreadReconciliationRefreshGeneration: Int? + private var selectedThreadLastProgressAt: ContinuousClock.Instant? private var detailRefreshTask: Task? private var detailStreamTask: Task? private var detailCatchUpTask: Task? @@ -135,7 +178,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private var detailStreamGeneration = 0 private var pendingDetailRenderMutations = NativeDetailRenderMutations() private var environmentGeneration = 0 - private var lastShellEventAt: Date? private var activeRawThread: OrchestrationThread? private var activeThreadSequence: Int? private var activeThreadPage: FeatureThreadPage? @@ -155,17 +197,38 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, projectFaviconStore: FeatureProjectFaviconStore = FeatureProjectFaviconStore(), fallbackPollingInitialDelay: Duration = .seconds(3), fallbackPollingInterval: Duration = .seconds(2), + shellReconciliationSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, aggregateRefreshInterval: Duration = NativeFeatureClient.defaultAggregateRefreshInterval, aggregateIdleRefreshInterval: Duration = NativeFeatureClient.defaultAggregateIdleRefreshInterval, aggregateFailureRefreshInterval: Duration = NativeFeatureClient.defaultAggregateFailureRefreshInterval, aggregateRefreshSleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }, + aggregatePeerRefreshSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(250)) + }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, environmentShellTimeoutInterval: TimeInterval = 6, threadSnapshotTimeoutInterval: TimeInterval = 8, + detailPublicationSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(80)) + }, catchUpDelay: @escaping @Sendable () async throws -> Void = { try await Task.sleep(for: .seconds(2)) }, + selectedThreadReconciliationSleep: @escaping @Sendable (Duration) async throws -> Void = { + try await Task.sleep(for: $0) + }, + selectedThreadReconciliationNow: @escaping @MainActor @Sendable () -> ContinuousClock.Instant = { .now }, + selectedThreadReconciliationReceipt: @escaping @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void = { _ in }, threadRetryDelay: @escaping @Sendable (Int) async throws -> Void = { attempt in try await Task.sleep(for: .seconds(min(5, 0.25 * pow(2, Double(min(5, attempt - 1)))))) }, @@ -197,13 +260,22 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, self.projectFaviconStore = projectFaviconStore self.fallbackPollingInitialDelay = fallbackPollingInitialDelay self.fallbackPollingInterval = fallbackPollingInterval + self.shellReconciliationSleep = shellReconciliationSleep self.aggregateRefreshInterval = aggregateRefreshInterval self.aggregateIdleRefreshInterval = aggregateIdleRefreshInterval self.aggregateFailureRefreshInterval = aggregateFailureRefreshInterval + self.aggregateRefreshReceipt = aggregateRefreshReceipt + self.aggregateStreamRetrySleep = aggregateStreamRetrySleep + self.aggregatePublishSleep = aggregatePublishSleep + self.aggregatePeerRefreshSleep = aggregatePeerRefreshSleep self.aggregateRefreshSleep = aggregateRefreshSleep self.environmentShellTimeoutInterval = environmentShellTimeoutInterval self.threadSnapshotTimeoutInterval = threadSnapshotTimeoutInterval + self.detailPublicationSleep = detailPublicationSleep self.catchUpDelay = catchUpDelay + self.selectedThreadReconciliationSleep = selectedThreadReconciliationSleep + self.selectedThreadReconciliationNow = selectedThreadReconciliationNow + self.selectedThreadReconciliationReceipt = selectedThreadReconciliationReceipt self.threadRetryDelay = threadRetryDelay self.aggregateEnvironmentLoader = aggregateEnvironmentLoader let pair = AsyncStream.makeStream() @@ -212,10 +284,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } deinit { + selectedThreadReconciliationTask?.cancel() + activeHydrationTask?.cancel() pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() aggregateRefreshTask?.cancel() + aggregatePublishTask?.cancel() + aggregateRefreshWorkers.values.forEach { $0.task.cancel() } shellPublishTask?.cancel() archivedRefreshTask?.cancel() detailRefreshTask?.cancel() @@ -223,50 +300,179 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detailCatchUpTask?.cancel() detailPublishTask?.cancel() projectFaviconRefreshTasks.values.forEach { $0.cancel() } + acceptedCommandRefreshes.values.forEach { $0.task.cancel() } continuation.finish() } func initialSnapshot() async throws -> FeatureSnapshot { + beginForegroundBootstrap() + let bootstrapID = foregroundBootstrapID let environments = try await runtime.environments() + guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } guard let activeClient = try await runtime.activeClient() else { - await clearActiveEnvironment() + await clearActiveEnvironment(disconnectClient: false) let snapshot = disconnectedSnapshot(environments: environments) latestSnapshot = snapshot return snapshot } - // The runtime actor can change its active selection at any suspension - // point. Derive both values from one client so the snapshot cannot pair - // one environment with another environment's connection. + guard bootstrapID == foregroundBootstrapID else { throw CancellationError() } let environment = activeClient.environment - - await adoptEnvironment(environment, client: activeClient) - let generation = environmentGeneration - let loads = await loadEnvironmentShells(environments.filter(\.isEnabled)) - guard isCurrentSession(client: activeClient, generation: generation) else { + guard try await adoptEnvironment(environment, client: activeClient) else { throw CancellationError() } + guard bootstrapID == foregroundBootstrapID, + isCurrentSession(client: activeClient, generation: environmentGeneration) else { throw CancellationError() } - reconcileEnvironmentLoads(loads, savedEnvironments: environments) - latestShell = shellsByEnvironmentID[environment.id] - startPolling(activeClient) - let activeLoad = loads.first { $0.environment.id == environment.id } - let activeIsReachable = activeLoad?.shell != nil - if activeIsReachable { - scheduleArchivedRefresh(client: activeClient, environment: environment) - } - if activeLoad?.credentialRejected == true { - markActiveEnvironmentNeedsPairing(detail: activeLoad?.failureDetail) + reconcileEnvironmentLoads([], savedEnvironments: environments) + for saved in environments where saved.isEnabled { + environmentConnectionStates[saved.id] = .reconnecting + environmentConnectionDetails[saved.id] = nil } + latestShell = shellsByEnvironmentID[environment.id] let snapshot = makeSnapshot( - environments: environments, - activeEnvironment: environment, - connectionState: environmentConnectionStates[environment.id] ?? .disconnected, - connectionDetail: environmentConnectionDetails[environment.id] + environments: environments, activeEnvironment: environment, + connectionState: .reconnecting ) latestSnapshot = snapshot + // No suspension after assigning the seed: the root installs it before + // it consumes buffered hydration events through its single iterator. + if isForeground { + startPolling(activeClient) + requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + startAggregateRefresh(activeClient, refreshImmediately: true) + } return snapshot } + private func beginForegroundBootstrap() { + foregroundBootstrapID = UUID() + activeHydrationTask?.cancel() + activeHydrationTask = nil + activeHydrationID = nil + activeHydrationPending = false + activeHTTPAuthorityRevision &+= 1 + activeHasHydrated = false + activeStreamIsAuthoritative = false + pollingTask?.cancel() + fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() + configurationTask?.cancel() + archivedRefreshTask?.cancel() + archivedRefreshTask = nil + shellPublishTask?.cancel() + shellPublishTask = nil + cancelAggregateRefresh() + } + + private func isCurrentForeground(_ client: T3Client, generation: Int, bootstrapID: UUID) -> Bool { + !Task.isCancelled && isForeground && foregroundBootstrapID == bootstrapID + && isCurrentSession(client: client, generation: generation) + } + + /// Bootstrap, fallback polling, and unknown stream events share one HTTP + /// request owner. An invalidated request drains before its pending repair. + private func requestActiveShellHydration(_ activeClient: T3Client, bootstrapID: UUID, force: Bool = false) { + let generation = environmentGeneration + guard isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } + guard activeHydrationTask == nil else { + if force { activeHydrationPending = true } + return + } + let requestID = UUID() + let revision = activeHTTPAuthorityRevision + let runtime = runtime + let timeout = environmentShellTimeoutInterval + activeHydrationID = requestID + activeHydrationPending = false + activeHydrationTask = Task { [weak self] in + let shell: OrchestrationShellSnapshot? + var hydrationError: (any Error)? + do { shell = try await activeClient.shellSnapshot(timeoutInterval: timeout) } + catch { shell = nil; hydrationError = error } + let environments = try? await runtime.environments() + guard let self else { return } + defer { self.aggregateRefreshReceipt(.httpFinished(environmentID: activeClient.environment.id)) } + guard self.activeHydrationID == requestID, + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) + else { return } + self.activeHydrationTask = nil + self.activeHydrationID = nil + guard let environments, environments.contains(activeClient.environment), + activeClient.environment.isEnabled else { return } + if revision == self.activeHTTPAuthorityRevision { + if let error = hydrationError, error.isRejectedAuthorization { + self.markActiveEnvironmentNeedsPairing(detail: error.localizedDescription) + return + } + if let shell { + self.activeHydrationPending = false + self.acceptActiveShell(shell, client: activeClient, environments: environments, + socketAuthoritative: false, connectHeader: false) + } else if !self.activeStreamIsAuthoritative { + self.emitConnection( + self.activeHasHydrated ? .reconnecting : .disconnected, + detail: "Server unreachable. Retrying automatically." + ) + } + } + if self.activeHydrationPending && !self.activeStreamIsAuthoritative { + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + } + } + } + + private func acceptActiveShell( + _ candidate: OrchestrationShellSnapshot, client: T3Client, + environments: [Environment], socketAuthoritative: Bool, connectHeader: Bool + ) { + let shell: OrchestrationShellSnapshot + if activeShellEpochHasSnapshot, let cached = latestShell, + cached.snapshotSequence > candidate.snapshotSequence { shell = cached } + else { shell = candidate } + let firstHydration = !activeHasHydrated + activeShellEpochHasSnapshot = true + activeHasHydrated = true + if socketAuthoritative { + shellConnectionIDsByEnvironmentID[client.environment.id] = activeShellConnectionID + activeStreamIsAuthoritative = true + activeHTTPAuthorityRevision &+= 1 + activeHydrationPending = false + } + let removed = Set(shellsByEnvironmentID[client.environment.id]?.threads.map(\.id) ?? []) + .subtracting(shell.threads.map(\.id)) + for id in removed { + clearRemovedThreadDetail(FeatureScopedID.thread(environmentID: client.environment.id, wireID: id)) + } + latestShell = shell + shellsByEnvironmentID[client.environment.id] = shell + environmentConnectionStates[client.environment.id] = .connected + environmentConnectionDetails[client.environment.id] = nil + rebuildEntityIndexes(environments) + synchronizeActiveDetail(with: shell, environment: client.environment) + guard let activeEnvironment else { return } + publish(makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: connectHeader ? .connected : (latestSnapshot?.connection.state ?? .reconnecting), + connectionDetail: connectHeader ? nil : latestSnapshot?.connection.detail + )) + aggregateRefreshReceipt(.shellApplied(environmentID: client.environment.id, sequence: shell.snapshotSequence)) + if firstHydration { scheduleArchivedRefresh(client: client, environment: client.environment) } + } + + func suspendForBackground() { + stopSelectedThreadReconciliation() + isForeground = false + beginForegroundBootstrap() + } + func resumeAfterBackground(reconnect: Bool) async { + isForeground = true + if client == nil { + if let snapshot = try? await initialSnapshot(), isForeground { + continuation.yield(.snapshot(snapshot)) + } + return + } + if let client { startAggregateRefresh(client) } let sessionGeneration = environmentGeneration let selectedRoute = activeThreadID.flatMap { try? threadRoute(for: $0) } detailWasSynchronized = false @@ -291,7 +497,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } guard sessionGeneration == environmentGeneration else { return } - if let client { startPolling(client) } + if let client { + startPolling(client) + requestActiveShellHydration(client, bootstrapID: foregroundBootstrapID) + } if let selectedRoute, activeThreadID == selectedRoute.uiID, isKnownClient(selectedRoute.client, environmentID: selectedRoute.environmentID, generation: sessionGeneration) { resetDetailRefresh() @@ -340,7 +549,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } else { pairedClient = try await runtime.pair(url: endpoint, clientLabel: "T3 Code Swift") } - await adoptEnvironment(pairedClient.environment, client: pairedClient) + guard try await adoptEnvironment(pairedClient.environment, client: pairedClient) else { throw CancellationError() } + startAggregateRefresh(pairedClient) startPolling(pairedClient) } @@ -404,7 +614,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environment, credential: savedCredential ) - await adoptEnvironment(environment, client: managedClient) + guard try await adoptEnvironment(environment, client: managedClient) else { throw CancellationError() } do { try await refresh(client: managedClient) } catch { @@ -417,6 +627,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) publish(snapshot) } + startAggregateRefresh(managedClient) startPolling(managedClient) } @@ -470,6 +681,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func setEnvironmentEnabled(id: String, enabled: Bool) async throws { try await runtime.setEnabled(id: id, enabled: enabled) if !enabled { + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() + cancelAcceptedCommandRefreshes(environmentID: id) environmentConnectionStates[id] = .disconnected environmentConnectionDetails[id] = nil environmentClients[id] = nil @@ -490,6 +704,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } try await runtime.remove(id: id) saveGitHubRoutingGrants(gitHubRoutingGrants.filter { $0.environmentID != id }) + shellConnectionIDsByEnvironmentID[id] = nil + aggregateRefreshWorkers.removeValue(forKey: id)?.task.cancel() + cancelAcceptedCommandRefreshes(environmentID: id) if removesActiveEnvironment { await clearActiveEnvironment(disconnectClient: false) } @@ -970,46 +1187,59 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func adoptEnvironment( _ environment: Environment, client newClient: T3Client - ) async { + ) async throws -> Bool { + let bootstrapID = foregroundBootstrapID + let generation = environmentGeneration + let adoptedConnectionID = await newClient.currentConnectionID() + let selectedClient = try await runtime.activeClient() + guard bootstrapID == foregroundBootstrapID, generation == environmentGeneration, + selectedClient === newClient, selectedClient?.environment == environment else { return false } + cancelAggregateRefresh() if activeEnvironment?.id == environment.id, client === newClient { activeEnvironment = environment environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - startAggregateRefresh(newClient) - return + return true } - let previousClient = client pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() - aggregateRefreshTask?.cancel() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil + shellReconciliationTask = nil configurationTask = nil aggregateRefreshTask = nil aggregateRefreshID = nil archivedRefreshTask = nil + // A passive snapshot is authoritative for adoption only while its + // exact socket is still current. A replacement socket starts a new epoch. + activeShellConnectionID = adoptedConnectionID + activeShellEpochHasSnapshot = adoptedConnectionID != nil + && shellConnectionIDsByEnvironmentID[environment.id] == adoptedConnectionID clearEnvironmentState(preserveEnvironmentSnapshots: true) activeEnvironment = environment client = newClient environmentClients[environment.id] = newClient latestShell = shellsByEnvironmentID[environment.id] - if let previousClient, previousClient !== newClient { - await previousClient.disconnect() - } - startAggregateRefresh(newClient) + // Runtime owns shared transports. The former inbox client may now + // serve a passive shell or a selected detail, so adoption does not close it. + return true } private func clearActiveEnvironment(disconnectClient: Bool = true) async { + beginForegroundBootstrap() let previousClient = client pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() - aggregateRefreshTask?.cancel() + cancelAggregateRefresh() archivedRefreshTask?.cancel() pollingTask = nil fallbackPollingTask = nil + shellReconciliationTask = nil configurationTask = nil aggregateRefreshTask = nil aggregateRefreshID = nil @@ -1023,7 +1253,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func clearEnvironmentState(preserveEnvironmentSnapshots: Bool = false) { + activeHydrationTask?.cancel() + activeHydrationTask = nil + activeHydrationID = nil + activeHydrationPending = false + activeHTTPAuthorityRevision &+= 1 + activeHasHydrated = false + activeStreamIsAuthoritative = false environmentGeneration &+= 1 + cancelAcceptedCommandRefreshes() resetDetailRefresh() resetDetailStream() archivedRefreshTask?.cancel() @@ -1031,10 +1269,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, shellPublishTask?.cancel() shellPublishTask = nil latestShell = nil - lastShellEventAt = nil latestServerConfig = nil if !preserveEnvironmentSnapshots { environmentClients.removeAll() + shellConnectionIDsByEnvironmentID.removeAll() shellsByEnvironmentID.removeAll() shellProjectionCache.removeAll() indexedShellMembership = nil @@ -1984,6 +2222,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func deleteThread(id: String) async throws { let route = try threadRoute(for: id) _ = try await route.client.delete(threadID: route.wireID) + // A pending accepted-command refresh would publish the deleted thread again. + cancelAcceptedCommandRefreshes(threadID: route.uiID) archivedThreadsByEnvironmentID[route.environmentID]?.removeAll { $0.id == route.uiID } @@ -2012,6 +2252,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func loadThread(id: String, fresh: Bool) async throws -> FeatureThreadDetail { let route = try threadRoute(for: id) + if let previous = activeThreadID, previous != route.uiID { + cancelAcceptedCommandRefreshes(threadID: previous) + } let client = route.client let environment = client.environment let generation = environmentGeneration @@ -2051,9 +2294,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, activeThreadSequence = cached.sequence activeThreadPage = cached.page detail.page = cached.page + if latestDetails[route.uiID]?.page != detail.page { + publish(detail, threadID: route.uiID, renderCacheIsSource: true) + } markThreadCacheRecentlyUsed(route.uiID) startDetailStream(route, warmConnectionID: warmConnectionID) - return detail + if let shell = shellsByEnvironmentID[environment.id] { + synchronizeActiveDetail(with: shell, environment: environment) + } + return latestDetails[route.uiID] ?? detail } continuation.yield(.threadSync(id: route.uiID, state: .catchingUp)) let snapshot: OrchestrationThreadDetailSnapshot @@ -2154,6 +2403,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } func releaseThread(id: String) { + cancelAcceptedCommandRefreshes(threadID: id) guard activeThreadID == id else { return } retainActiveThread() resetDetailRefresh() @@ -2273,11 +2523,67 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, if pendingTurnSubmissions[route.uiID]?.identity == pending.identity { pendingTurnSubmissions[route.uiID] = nil } - // Live sync reconciles these snapshots. Refreshes are opportunistic - // after the accepted command so transient reads cannot invite a - // duplicate user turn. - try? await refreshThread(id: route.uiID, client: client) - try? await refresh(client: client) + scheduleAcceptedCommandRefresh(threadID: route.uiID, client: client, generation: generation) + } + + /// Complete accepted commands without waiting for optional reads. A send + /// needs detail and shell recovery; Stop retains its shell-only refresh. + private func scheduleAcceptedCommandRefresh( + threadID: String, client: T3Client, generation: Int, refreshDetail: Bool = true + ) { + guard isKnownClient(client, environmentID: client.environment.id, generation: generation) else { + return + } + if let current = acceptedCommandRefreshes[threadID], + current.client === client, current.generation == generation { + acceptedCommandRefreshes[threadID]?.pending = true + acceptedCommandRefreshes[threadID]?.needsDetail = current.needsDetail || refreshDetail + return + } + cancelAcceptedCommandRefreshes(threadID: threadID) + let id = UUID() + let task = Task { [weak self] in + while self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true { + let needsDetail = self?.acceptedCommandRefreshes[threadID]?.needsDetail == true + self?.acceptedCommandRefreshes[threadID]?.needsDetail = false + self?.acceptedCommandRefreshes[threadID]?.pending = false + if needsDetail { + try? await self?.refreshThread(id: threadID, client: client) { [weak self] in + guard self?.acceptedCommandRefreshes[threadID]?.pending == true else { return false } + // The discarded detail must be read again, even for a shell-only Stop. + self?.acceptedCommandRefreshes[threadID]?.needsDetail = true + return true + } + } + guard self?.isCurrentAcceptedCommandRefresh(threadID: threadID, id: id) == true else { break } + try? await self?.refresh(client: client) + guard self?.acceptedCommandRefreshes[threadID]?.pending == true else { break } + } + if self?.acceptedCommandRefreshes[threadID]?.id == id { + self?.acceptedCommandRefreshes[threadID] = nil + } + } + acceptedCommandRefreshes[threadID] = AcceptedCommandRefresh( + id: id, client: client, generation: generation, task: task, needsDetail: refreshDetail + ) + } + + private func isCurrentAcceptedCommandRefresh(threadID: String, id: UUID) -> Bool { + guard !Task.isCancelled, let refresh = acceptedCommandRefreshes[threadID], refresh.id == id else { + return false + } + return isKnownClient( + refresh.client, environmentID: refresh.client.environment.id, generation: refresh.generation + ) + } + + private func cancelAcceptedCommandRefreshes(threadID: String? = nil, environmentID: String? = nil) { + for (key, refresh) in acceptedCommandRefreshes { + guard threadID.map({ $0 == key }) ?? true, + environmentID.map({ $0 == refresh.client.environment.id }) ?? true else { continue } + refresh.task.cancel() + acceptedCommandRefreshes[key] = nil + } } private func messageWasCommitted( @@ -2293,12 +2599,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, func cancelTurn(threadID: String) async throws { let route = try threadRoute(for: threadID) + let generation = environmentGeneration let turnID = shellsByEnvironmentID[route.environmentID]?.threads .first(where: { $0.id == route.wireID })? .latestTurn? .turnId _ = try await route.client.interrupt(threadID: route.wireID, turnID: turnID) - try? await refresh(client: route.client) + scheduleAcceptedCommandRefresh( + threadID: route.uiID, client: route.client, generation: generation, refreshDetail: false + ) } func canRewindConversation(threadID: String, messageID: String) -> Bool { @@ -3757,66 +4066,70 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func startPolling(_ activeClient: T3Client) { pollingTask?.cancel() fallbackPollingTask?.cancel() + shellReconciliationTask?.cancel() configurationTask?.cancel() let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + let streamRetrySleep = aggregateStreamRetrySleep pollingTask = Task { [weak self] in while !Task.isCancelled, - self?.isCurrentSession(client: activeClient, generation: generation) == true + self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true { do { await activeClient.connect() guard - self?.isCurrentSession( - client: activeClient, - generation: generation - ) == true + self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true else { return } - let sequence = self?.latestShell?.snapshotSequence - let events = await activeClient.shellEventBatches(after: sequence, reconnect: false) + let subscription = try await activeClient.shellEventBatchesOnCurrentConnection() + guard self?.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) == true + else { return } + if let previous = self?.activeShellConnectionID, previous != subscription.connectionID { + self?.activeShellEpochHasSnapshot = false + self?.activeHTTPAuthorityRevision &+= 1 + self?.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + } + self?.activeShellConnectionID = subscription.connectionID + let events = subscription.events // Re-bind self per event instead of holding it strongly across // the indefinite stream, so the client can deinit mid-stream. - for try await batch in events { + shellEvents: for try await batch in events { guard !Task.isCancelled, let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { break } - self.lastShellEventAt = .now - self.emitConnection(.connected) + let environments = try await self.runtime.environments() + let connectionID = await activeClient.currentConnectionID() + guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), + connectionID == subscription.connectionID, + environments.contains(activeClient.environment) else { break } var deltas: [ShellStreamItem] = [] for item in batch { - switch item { - case let .snapshot(shell): - await self.consume(deltas: deltas, client: activeClient, generation: generation) - deltas.removeAll(keepingCapacity: true) - await self.consume( - shell: shell, - client: activeClient, - generation: generation, - refreshActiveThread: true - ) - case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: - deltas.append(item) - case .refreshRequired: - await self.consume(deltas: deltas, client: activeClient, generation: generation) - deltas.removeAll(keepingCapacity: true) - if let shell = try? await activeClient.shellSnapshot() { - await self.consume( - shell: shell, - client: activeClient, - generation: generation, - refreshActiveThread: true - ) - } - case .synchronized: - break + switch item { + case let .snapshot(shell): + await self.consume(deltas: deltas, client: activeClient, generation: generation) + deltas.removeAll(keepingCapacity: true) + self.acceptActiveShell(shell, client: activeClient, environments: environments, + socketAuthoritative: true, connectHeader: true) + case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: + guard self.activeStreamIsAuthoritative else { + self.activeHTTPAuthorityRevision &+= 1 + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + break shellEvents } + deltas.append(item) + case .refreshRequired: + self.activeStreamIsAuthoritative = false + self.activeHTTPAuthorityRevision &+= 1 + self.emitConnection(.reconnecting, detail: "Refreshing environment data.") + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) + break shellEvents + case .synchronized: + break + } } await self.consume(deltas: deltas, client: activeClient, generation: generation) } @@ -3835,16 +4148,21 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard !Task.isCancelled, let self, - self.isCurrentSession(client: activeClient, generation: generation) + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } - self.lastShellEventAt = nil + self.shellConnectionIDsByEnvironmentID[activeClient.environment.id] = nil + if self.activeStreamIsAuthoritative { + self.activeHTTPAuthorityRevision &+= 1 + } + self.activeStreamIsAuthoritative = false + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID, force: true) self.emitConnection( .reconnecting, detail: "Live updates paused. Refreshing over HTTP." ) - do { try await Task.sleep(for: .milliseconds(250)) } catch { return } + do { try await streamRetrySleep(activeClient.environment.id, .seconds(20)) } catch { return } } } let fallbackPollingInitialDelay = fallbackPollingInitialDelay @@ -3857,59 +4175,31 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } while !Task.isCancelled { guard let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { return } - let socketIsSynchronized = - await activeClient.liveConnectionActive() - && self.lastShellEventAt != nil - if !socketIsSynchronized { - self.emitConnection( - .reconnecting, - detail: "Live updates reconnecting. Refreshing over HTTP." - ) - do { - let shell = try await activeClient.shellSnapshot() - guard !Task.isCancelled, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { - return - } - await self.consumeFallbackShell( - shell: shell, - client: activeClient, - generation: generation - ) - } catch is CancellationError { - return - } catch { - guard !Task.isCancelled, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { - return - } - if error.isRejectedAuthorization { - self.markActiveEnvironmentNeedsPairing(detail: error.localizedDescription) - return - } - self.emitConnection( - .reconnecting, - detail: "Server unreachable. Retrying automatically." - ) - } - } - do { - try await Task.sleep(for: fallbackPollingInterval) - } catch { - return + if !self.activeStreamIsAuthoritative { + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) } + do { try await Task.sleep(for: fallbackPollingInterval) } + catch { return } + } + } + let reconciliationSleep = shellReconciliationSleep + shellReconciliationTask = Task { [weak self] in + while !Task.isCancelled { + // Ping health cannot prove shell completeness. Keep the faster + // outage fallback independent of this quiet reconciliation. + do { try await reconciliationSleep(activeClient.environment.id, Self.shellReconciliationInterval) } + catch { return } + guard let self, + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) + else { return } + guard self.activeStreamIsAuthoritative else { continue } + self.requestActiveShellHydration(activeClient, bootstrapID: bootstrapID) + // One owner coalesces all HTTP requests. The next interval starts + // after this read drains, including failures and supersession. + await self.activeHydrationTask?.value } } configurationTask = Task { [weak self] in @@ -3917,10 +4207,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, for try await event in await activeClient.serverConfigEvents() { guard !Task.isCancelled, let self, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID) else { break } switch event { @@ -3982,11 +4269,16 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, case .unrelated: continue } - if let shell = self.latestShell { - await self.emitSnapshot( - shell, client: activeClient, expectedGeneration: generation - ) - } + // A failed catalog read skips this publish; later events still apply. + guard let environments = try? await self.runtime.environments() else { continue } + guard self.isCurrentForeground(activeClient, generation: generation, bootstrapID: bootstrapID), + environments.contains(activeClient.environment), let activeEnvironment = self.activeEnvironment + else { return } + self.publish(self.makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: self.latestSnapshot?.connection.state ?? .reconnecting, + connectionDetail: self.latestSnapshot?.connection.detail + )) } } catch is CancellationError { return @@ -4000,124 +4292,490 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, /// Non-active environments do not hold WebSocket subscriptions. A quiet /// HTTP refresh keeps their home rows and reachability useful without /// multiplying live streams or creating a high-frequency battery cost. - private func startAggregateRefresh(_ activeClient: T3Client) { + private func cancelAggregateRefresh() { aggregateRefreshTask?.cancel() - let generation = environmentGeneration - let refreshID = UUID() - let fastInterval = aggregateRefreshInterval - let idleInterval = aggregateIdleRefreshInterval - let failureInterval = aggregateFailureRefreshInterval - let sleep = aggregateRefreshSleep - let loadEnvironments = aggregateEnvironmentLoader - aggregateRefreshID = refreshID - aggregateRefreshTask = Task { [weak self] in - var nextInterval = fastInterval - var failureBackoffs: [String: Duration] = [:] - while !Task.isCancelled { - let elapsedInterval = nextInterval + aggregatePublishTask?.cancel() + aggregateRefreshWorkers.values.forEach { $0.task.cancel() } + aggregateRefreshWorkers.removeAll() + aggregatePublishTask = nil + aggregateRefreshID = nil + } + + private func startAggregateRefresh(_ activeClient: T3Client, refreshImmediately: Bool = false) { + cancelAggregateRefresh() + guard isForeground else { return } + let context = AggregateRefreshContext(owner: self, activeClient: activeClient, refreshImmediately: refreshImmediately) + aggregateRefreshID = context.refreshID + aggregateRefreshTask = Task { + defer { context.cancelWorkers() } + while context.isCurrent { + var interval = context.fastInterval do { - try await sleep(nextInterval) - } catch { + let environments = try await context.loadEnvironments(context.runtime) + guard context.isCurrent else { return } + interval = context.reconcileWorkers(environments) + } catch is CancellationError where Task.isCancelled { return + } catch { + interval = context.failureInterval } - guard let self, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ), - let activeEnvironment = self.activeEnvironment else { - return + do { try await context.sleep(interval) } catch { return } + } + } + } + + /// A worker retains runtime inputs, never its UI owner across a suspension. + /// This lets deinit cancel the exact outstanding shell/catalogue tasks. + @MainActor + private final class AggregateRefreshContext { + weak var owner: NativeFeatureClient? + let runtime: EnvironmentRuntime + let activeClient: T3Client + let generation: Int + let refreshID = UUID() + let refreshImmediately: Bool + let fastInterval: Duration + let idleInterval: Duration + let failureInterval: Duration + let shellTimeout: TimeInterval + let sleep: @Sendable (Duration) async throws -> Void + let peerSleep: @Sendable (String, Duration) async throws -> Void + let reconciliationSleep: @Sendable (String, Duration) async throws -> Void + let loadEnvironments: @Sendable (EnvironmentRuntime) async throws -> [Environment] + let streamRetrySleep: @Sendable (String, Duration) async throws -> Void + let publishSleep: @Sendable () async throws -> Void + let refreshReceipt: @MainActor @Sendable (NativePassiveShellReceipt) -> Void + var savedEnvironments: [Environment] = [] + var dirtyEnvironmentIDs = Set() + + init(owner: NativeFeatureClient, activeClient: T3Client, refreshImmediately: Bool) { + self.owner = owner + self.refreshImmediately = refreshImmediately + runtime = owner.runtime + self.activeClient = activeClient + generation = owner.environmentGeneration + fastInterval = owner.aggregateRefreshInterval + idleInterval = owner.aggregateIdleRefreshInterval + failureInterval = owner.aggregateFailureRefreshInterval + shellTimeout = owner.environmentShellTimeoutInterval + sleep = owner.aggregateRefreshSleep + peerSleep = owner.aggregatePeerRefreshSleep + reconciliationSleep = owner.shellReconciliationSleep + loadEnvironments = owner.aggregateEnvironmentLoader + streamRetrySleep = owner.aggregateStreamRetrySleep + publishSleep = owner.aggregatePublishSleep + refreshReceipt = owner.aggregateRefreshReceipt + } + + var isCurrent: Bool { + !Task.isCancelled && owner?.isForeground == true && owner?.aggregateRefreshID == refreshID + && owner?.isCurrentSession(client: activeClient, generation: generation) == true + } + + func cancelWorkers() { + guard let owner, owner.aggregateRefreshID == refreshID else { return } + owner.aggregateRefreshWorkers.values.forEach { $0.task.cancel() } + owner.aggregateRefreshWorkers.removeAll() + owner.aggregatePublishTask?.cancel() + owner.aggregatePublishTask = nil + } + + func owns(_ environment: Environment) -> Bool { + isCurrent && owner?.aggregateRefreshWorkers[environment.id]?.environment == environment + && owner?.activeEnvironment?.id != environment.id + } + + func reconcileWorkers(_ environments: [Environment]) -> Duration { + guard let owner, isCurrent else { return idleInterval } + savedEnvironments = environments + let passive = environments.filter { $0.isEnabled && $0.id != owner.activeEnvironment?.id } + let current = Dictionary(uniqueKeysWithValues: passive.map { ($0.id, $0) }) + for (id, worker) in owner.aggregateRefreshWorkers where current[id] != worker.environment { + worker.task.cancel() + owner.aggregateRefreshWorkers[id] = nil + } + // Prune once per topology check, not once per shell result. + owner.reconcileEnvironmentLoads([], savedEnvironments: environments) + if owner.latestSnapshot?.environments != environments.map({ + owner.mapEnvironment($0, activeID: owner.activeEnvironment?.id) + }) { + owner.publishAggregateSnapshot(environments) + } + for environment in passive where owner.aggregateRefreshWorkers[environment.id] == nil { + let state = PassiveShellState() + state.needsHTTP = refreshImmediately || owner.shellsByEnvironmentID[environment.id] == nil + let task = Task { + await withTaskGroup(of: Void.self) { group in + group.addTask { await self.refreshShell(environment, state: state) } + group.addTask { await self.followShell(environment, state: state) } + group.addTask { await self.refreshCatalogue(environment) } + } } - let environments: [Environment] + owner.aggregateRefreshWorkers[environment.id] = (environment, task) + } + if let dirtyID = dirtyEnvironmentIDs.first { schedulePublication(dirtyID) } + return passive.isEmpty ? idleInterval : fastInterval + } + + /// Revalidate membership after suspension, including endpoint edits. + /// Persistence errors retry; removed or disabled peers end their work. + func currentEnvironments(for environment: Environment) async throws -> [Environment]? { + guard isCurrent else { return nil } + let environments = try await runtime.environments() + guard isCurrent, environments.contains(environment), environment.isEnabled, + owner?.activeEnvironment?.id != environment.id else { return nil } + return environments + } + + func refreshShell(_ environment: Environment, state: PassiveShellState) async { + var interval = fastInterval + while owns(environment) { do { - environments = try await loadEnvironments(self.runtime) + let wasLive = state.isLive + if wasLive || !state.needsHTTP { + let delay = interval + // A rejected credential outlives the open socket; stop the quiet cadence. + let reconciles = wasLive && !state.credentialRejected + await withTaskGroup(of: Void.self) { group in + group.addTask { + if reconciles { + try? await self.reconciliationSleep(environment.id, NativeFeatureClient.shellReconciliationInterval) + } else { + try? await self.peerSleep(environment.id, delay) + } + } + group.addTask { await state.waitForRepair(untilLive: !wasLive) } + _ = await group.next() + group.cancelAll() + } + } + guard owns(environment) else { return } + // A newly authoritative snapshot starts the quiet cadence; + // it must not turn a fallback wake-up into a duplicate read. + if !wasLive && state.isLive { continue } + state.needsHTTP = false + guard try await currentEnvironments(for: environment) != nil else { return } + let client = await runtime.client(for: environment) + let epoch = state.epoch + let authority = state.authorityRevision + let connectionID = await client.currentConnectionID() + guard owns(environment), epoch == state.epoch, + authority == state.authorityRevision else { continue } + let shell: OrchestrationShellSnapshot? + var hydrationError: (any Error)? + do { shell = try await client.shellSnapshot(timeoutInterval: shellTimeout) } + catch { shell = nil; hydrationError = error } + defer { refreshReceipt(.httpFinished(environmentID: environment.id)) } + let currentConnection = await client.currentConnectionID() + guard let environments = try await currentEnvironments(for: environment) else { return } + guard owns(environment), epoch == state.epoch, + authority == state.authorityRevision, connectionID == currentConnection else { continue } + // A rejected credential applies even while the old socket stays open. + if let error = hydrationError, error.isRejectedAuthorization { + state.credentialRejected = true + owner?.applyEnvironmentLoad(EnvironmentShellLoad( + environment: environment, client: client, shell: nil, config: nil, + credentialRejected: true, failureDetail: error.localizedDescription + )) + owner?.publishAggregateSnapshot(environments) + interval = .seconds(24 * 60 * 60) + continue + } + // Other HTTP failures say nothing about the connected stream. + if shell == nil && state.isLive { continue } + // A validated HTTP snapshot remains in this socket's cache + // epoch without claiming that the live stream is complete. + if shell != nil { owner?.shellConnectionIDsByEnvironmentID[environment.id] = connectionID } + // The first authoritative response in this subscription epoch + // may reset a cache left over from a restarted server. + interval = applyShell( + shell, client: client, environment: environment, saved: environments, + replacingEpoch: state.cacheEpoch != epoch + ) + if shell != nil { state.cacheEpoch = epoch } } catch is CancellationError where Task.isCancelled { return } catch { - // Persistence can be briefly unavailable while another - // actor atomically replaces the environment document. - // Back off while keeping the loop alive for recovery. - nextInterval = failureInterval - continue + interval = failureInterval } - guard !Task.isCancelled, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + } + } + + func followShell(_ environment: Environment, state: PassiveShellState) async { + while owns(environment) { + do { + guard try await currentEnvironments(for: environment) != nil else { return } + let client = await runtime.client(for: environment) + let subscription = try await client.shellEventsOnCurrentConnection() + guard try await currentEnvironments(for: environment) != nil, + owns(environment) else { return } + if state.connectionID != subscription.connectionID { + let needsRepair = state.connectionID != nil + || owner?.shellsByEnvironmentID[environment.id] == nil + state.connectionID = subscription.connectionID + state.epoch = UUID() + // A fresh bootstrap already read this peer. Its first + // socket attempt need not duplicate that HTTP request. + if needsRepair { state.requestRepair() } + } + let epoch = state.epoch + state.isLive = false + for try await item in subscription.events { + let connectionID = await client.currentConnectionID() + guard owns(environment), epoch == state.epoch, + connectionID == subscription.connectionID else { break } + guard applyStream(item, client: client, environment: environment, state: state) + else { break } + } + } catch is CancellationError where Task.isCancelled { return + } catch { + // HTTP recovery belongs to this peer's single fallback loop. } - let passiveEnvironments = environments.filter { - $0.isEnabled && $0.id != activeEnvironment.id - } - guard !passiveEnvironments.isEmpty else { - nextInterval = idleInterval - continue - } - let passiveIDs = Set(passiveEnvironments.map(\.id)) - failureBackoffs = failureBackoffs.reduce(into: [:]) { result, entry in - guard passiveIDs.contains(entry.key) else { return } - result[entry.key] = max(.zero, entry.value - elapsedInterval) + guard owns(environment) else { return } + state.isLive = false + // A read begun before the stream lost completeness cannot + // repair that gap, even if the socket identity is unchanged. + state.authorityRevision &+= 1 + markStreamPaused(environment, state: state) + state.requestRepair() + do { try await streamRetrySleep(environment.id, failureInterval) } catch { return } + } + } + + func markStreamPaused(_ environment: Environment, state: PassiveShellState) { + guard let owner, owns(environment) else { return } + owner.shellConnectionIDsByEnvironmentID[environment.id] = nil + // A rejected credential still needs pairing; keep Pair again visible. + guard !state.credentialRejected else { return } + owner.environmentConnectionStates[environment.id] = .reconnecting + owner.environmentConnectionDetails[environment.id] = "Live updates paused. Refreshing over HTTP." + schedulePublication(environment.id) + } + + func applyStream( + _ item: ShellStreamItem, client: T3Client, environment: Environment, state: PassiveShellState + ) -> Bool { + guard let owner, owns(environment) else { return false } + let current = owner.shellsByEnvironmentID[environment.id] + let shell: OrchestrationShellSnapshot + switch item { + case let .snapshot(snapshot): + if state.cacheEpoch == state.epoch, let current, + current.snapshotSequence > snapshot.snapshotSequence { + shell = current + } else { + shell = snapshot } - let refreshableEnvironments = passiveEnvironments.filter { - failureBackoffs[$0.id, default: .zero] <= .zero + let remainingIDs = Set(shell.threads.map(\.id)) + for removed in current?.threads ?? [] where !remainingIDs.contains(removed.id) { + owner.clearRemovedThreadDetail(FeatureScopedID.thread( + environmentID: environment.id, wireID: removed.id + )) } - guard !refreshableEnvironments.isEmpty else { - nextInterval = fastInterval - continue + owner.shellConnectionIDsByEnvironmentID[environment.id] = state.connectionID + state.authorityRevision &+= 1 + state.cacheEpoch = state.epoch + let wasLive = state.isLive + state.isLive = true + state.needsHTTP = false + // A rejected peer keeps its back-off even when a new stream seeds it. + if !wasLive && !state.credentialRejected { state.wakeWaiter() } + case .synchronized: + // A completion marker cannot make an arbitrary cached shell authoritative. + return state.isLive + case .refreshRequired: + return false + case .projectUpserted, .projectRemoved, .threadUpserted, .threadRemoved: + guard state.isLive, let current else { return false } + var projects = current.projects + var threads = current.threads + let sequence: Int + switch item { + case let .projectUpserted(nextSequence, project): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + if let index = projects.firstIndex(where: { $0.id == project.id }) { + projects[index] = project + } else { projects.append(project) } + case let .projectRemoved(nextSequence, id): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + projects.removeAll { $0.id == id } + case let .threadUpserted(nextSequence, thread): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + owner.archivedThreadsByEnvironmentID[environment.id]?.removeAll { + ($0.wireID ?? $0.id) == thread.id + } + if let index = threads.firstIndex(where: { $0.id == thread.id }) { + threads[index] = thread + } else { threads.append(thread) } + case let .threadRemoved(nextSequence, id): + sequence = nextSequence + guard sequence > current.snapshotSequence else { return true } + threads.removeAll { $0.id == id } + owner.clearRemovedThreadDetail(FeatureScopedID.thread( + environmentID: environment.id, wireID: id + )) + default: + return false } - let loads = await self.loadEnvironmentShells(refreshableEnvironments) - guard !Task.isCancelled, - self.aggregateRefreshID == refreshID, - self.isCurrentSession( - client: activeClient, - generation: generation - ) else { + shell = OrchestrationShellSnapshot( + snapshotSequence: sequence, projects: projects, threads: threads, updatedAt: current.updatedAt + ) + } + let membershipChanged = current?.projects.map(\.id) != shell.projects.map(\.id) + || current?.threads.map(\.id) != shell.threads.map(\.id) + owner.shellsByEnvironmentID[environment.id] = shell + owner.environmentClients[environment.id] = client + // Stream deltas still arrive on the old socket; keep showing the rejection. + if !state.credentialRejected { + owner.environmentConnectionStates[environment.id] = .connected + owner.environmentConnectionDetails[environment.id] = nil + } + if membershipChanged { owner.rebuildEntityIndexes(savedEnvironments) } + owner.synchronizeActiveDetail(with: shell, environment: environment) + refreshReceipt(.shellApplied(environmentID: environment.id, sequence: shell.snapshotSequence)) + schedulePublication(environment.id) + return true + } + + func schedulePublication(_ environmentID: String) { + guard let owner, isCurrent else { return } + dirtyEnvironmentIDs.insert(environmentID) + guard owner.aggregatePublishTask == nil else { return } + owner.aggregatePublishTask = Task { + do { try await self.publishSleep() } catch { return } + let environments: [Environment] + do { environments = try await self.runtime.environments() } + catch { + if self.isCurrent { self.owner?.aggregatePublishTask = nil } return } - let shellsChanged = loads.contains { load in - guard let shell = load.shell else { return false } - return shell != self.shellsByEnvironmentID[load.environment.id] + guard let owner = self.owner, self.isCurrent else { return } + self.savedEnvironments = environments + owner.reconcileEnvironmentLoads([], savedEnvironments: environments) + let dirty = self.dirtyEnvironmentIDs.filter { + owner.aggregateRefreshWorkers[$0] != nil } - let hasActiveWork = loads.contains { load in - load.shell.map(Self.shellNeedsFrequentAggregateRefresh) == true - } - for load in loads { - if load.credentialRejected { - // Only re-pairing changes this. Re-pairing replaces - // the runtime client and restarts this loop. - failureBackoffs[load.environment.id] = .seconds(24 * 60 * 60) - } else if load.shell == nil { - failureBackoffs[load.environment.id] = failureInterval - } else { - failureBackoffs[load.environment.id] = nil + self.dirtyEnvironmentIDs.removeAll() + owner.aggregatePublishTask = nil + guard !dirty.isEmpty else { return } + owner.publishAggregateSnapshot(self.savedEnvironments, changedEnvironmentIDs: dirty) + } + } + + func applyShell( + _ shell: OrchestrationShellSnapshot?, client: T3Client, + environment: Environment, saved: [Environment], replacingEpoch: Bool = false + ) -> Duration { + guard let owner, isCurrent else { return failureInterval } + let previous = owner.shellsByEnvironmentID[environment.id] + let changed = shell.map { $0 != previous } ?? false + let membershipChanged = shell.map { + $0.projects.map(\.id) != previous?.projects.map(\.id) + || $0.threads.map(\.id) != previous?.threads.map(\.id) + } ?? false + if replacingEpoch, let shell { owner.shellsByEnvironmentID[environment.id] = shell } + owner.applyEnvironmentLoad(EnvironmentShellLoad( + environment: environment, client: client, shell: shell, config: nil + )) + if membershipChanged { owner.rebuildEntityIndexes(saved) } + owner.publishAggregateSnapshot(saved, changedEnvironmentIDs: [environment.id]) + guard let shell else { return failureInterval } + return changed || NativeFeatureClient.shellNeedsFrequentAggregateRefresh(shell) + ? fastInterval : idleInterval + } + + func refreshCatalogue(_ environment: Environment) async { + // A rejected peer needs pairing; re-pairing starts a new worker. + while isCurrent && owner?.serverConfigsByEnvironmentID[environment.id] == nil + && owner?.environmentConnectionStates[environment.id] != .needsPairing { + do { + guard try await currentEnvironments(for: environment) != nil else { return } + // Never disconnect the shared client if this peer becomes selected. + let probe = await runtime.ephemeralClient(for: environment) + let config = try? await probe.serverConfig() + await probe.disconnect() + guard let environments = try await currentEnvironments(for: environment) else { return } + // A save or config event that landed during the probe is newer. + guard owner?.serverConfigsByEnvironmentID[environment.id] == nil else { return } + if let config { + applyCatalogue(config, environment: environment, saved: environments) + return } + } catch is CancellationError where Task.isCancelled { + return + } catch { + // Retry a transient environment-document read just like a failed probe. } - self.reconcileEnvironmentLoads(loads, savedEnvironments: environments) - let currentConnection = self.latestSnapshot?.connection - ?? FeatureConnection( - state: .disconnected, - environmentName: activeEnvironment.label, - endpoint: activeEnvironment.httpBaseURL.absoluteString - ) - let snapshot = self.makeSnapshot( - environments: environments, - activeEnvironment: activeEnvironment, - connectionState: currentConnection.state, - connectionDetail: currentConnection.detail - ) - self.publish(snapshot) - if shellsChanged || hasActiveWork { - nextInterval = fastInterval - } else { - nextInterval = idleInterval + do { try await Task.sleep(for: failureInterval) } catch { return } + } + } + + func applyCatalogue(_ config: ServerConfigSnapshot, environment: Environment, saved: [Environment]) { + guard let owner, isCurrent else { return } + owner.setServerConfig(config, environmentID: environment.id) + owner.publishAggregateSnapshot(saved, changedEnvironmentIDs: [environment.id]) + } + } + + @MainActor + private final class PassiveShellState { + var epoch = UUID() + var connectionID: UUID? + var cacheEpoch: UUID? + var authorityRevision = 0 + var isLive = false + var needsHTTP = false + /// Set when HTTP rejects this peer's credential; cleared by a new worker after re-pairing. + var credentialRejected = false + private var waiter: (id: UUID, continuation: CheckedContinuation)? + + func requestRepair() { + // Stream failures must not wake a rejected peer before its back-off. + guard !credentialRejected else { return } + needsHTTP = true + wakeWaiter() + } + + func wakeWaiter() { + waiter?.continuation.resume() + waiter = nil + } + + func waitForRepair(untilLive: Bool = false) async { + guard !needsHTTP, !(untilLive && isLive) else { return } + let id = UUID() + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + if Task.isCancelled || needsHTTP || (untilLive && isLive) { continuation.resume() } + else { waiter = (id, continuation) } + } + } onCancel: { + Task { @MainActor [weak self] in + guard self?.waiter?.id == id else { return } + self?.waiter?.continuation.resume() + self?.waiter = nil } } } } + private func publishAggregateSnapshot( + _ environments: [Environment], changedEnvironmentIDs: Set? = nil + ) { + guard let activeEnvironment else { return } + let connection = latestSnapshot?.connection + publish(makeSnapshot( + environments: environments, activeEnvironment: activeEnvironment, + connectionState: connection?.state ?? .disconnected, + connectionDetail: connection?.detail, + changedEnvironmentIDs: changedEnvironmentIDs + )) + } + nonisolated private static func shellNeedsFrequentAggregateRefresh( _ shell: OrchestrationShellSnapshot ) -> Bool { @@ -4154,35 +4812,6 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } - /// HTTP fallback refreshes data while preserving the socket's reconnecting - /// state. The generation travels through the awaited snapshot publish so a - /// task from a previous environment session cannot publish late results. - private func consumeFallbackShell( - shell: OrchestrationShellSnapshot, - client: T3Client, - generation: Int - ) async { - guard isCurrentSession(client: client, generation: generation), - shell.snapshotSequence >= (latestShell?.snapshotSequence ?? .min) else { - return - } - shellPublishTask?.cancel() - shellPublishTask = nil - latestShell = shell - shellsByEnvironmentID[client.environment.id] = shell - await emitSnapshot( - shell, - client: client, - expectedGeneration: generation, - markSourceConnected: false - ) - guard isCurrentSession(client: client, generation: generation), - let threadID = activeThreadID else { - return - } - scheduleDetailRefresh(threadID: threadID, client: client) - } - private func consume(deltas: [ShellStreamItem], client: T3Client, generation: Int) async { guard !deltas.isEmpty, !Task.isCancelled, isCurrentSession(client: client, generation: generation) else { return } @@ -4244,20 +4873,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, changedThreadIDs.insert(uiThreadID) shouldRefreshArchived = true threads.removeAll { $0.id == threadID } - latestDetails[uiThreadID] = nil - detailRenderCaches[uiThreadID] = nil - detailCacheRecency.removeAll { $0 == uiThreadID } - if activeThreadID == uiThreadID { - resetDetailRefresh() - resetDetailStream() - activeThreadID = nil - activeThreadEnvironmentID = nil - activeRawThread = nil - activeThreadSequence = nil - activeThreadPage = nil - threadHistoryEpoch &+= 1 - pendingOlderThreadPage = nil - } + clearRemovedThreadDetail(uiThreadID) case .snapshot, .synchronized, .refreshRequired: continue } @@ -4283,6 +4899,23 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } } + private func clearRemovedThreadDetail(_ uiThreadID: String) { + latestDetails[uiThreadID] = nil + detailRenderCaches[uiThreadID] = nil + detailCacheRecency.removeAll { $0 == uiThreadID } + if activeThreadID == uiThreadID { + resetDetailRefresh() + resetDetailStream() + activeThreadID = nil + activeThreadEnvironmentID = nil + activeRawThread = nil + activeThreadSequence = nil + activeThreadPage = nil + threadHistoryEpoch &+= 1 + pendingOlderThreadPage = nil + } + } + /// Shell streams can emit many metadata updates during one provider turn. /// Home only needs the newest row state, so publish at most four times per /// second while the selected transcript continues on its dedicated stream. @@ -4305,11 +4938,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, private func scheduleDetailRefresh( threadID: String, client: T3Client, - force: Bool = false + force: Bool = false, + reconcile: Bool = false ) { guard activeThreadID == threadID, activeThreadEnvironmentID == client.environment.id else { return } - guard force || detailStreamTask == nil else { return } + guard force || reconcile || detailStreamTask == nil else { return } if force { detailWasSynchronized = false // This required read owns recovery now. An older fallback must not @@ -4327,11 +4961,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detailRefreshGeneration &+= 1 let generation = detailRefreshGeneration let sessionGeneration = environmentGeneration + let streamGeneration = detailStreamGeneration detailRefreshTask = Task { [weak self] in do { // Shell updates can be coalesced. A required replacement cannot // apply more thread events until its snapshot arrives. - if !force { + if !force && !reconcile { try await Task.sleep(for: .milliseconds(250)) } } catch { @@ -4347,11 +4982,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, generation: sessionGeneration ) { do { - try await self.refreshThread(id: threadID, client: client) + try await self.refreshThread( + id: threadID, client: client, + expectedStreamGeneration: reconcile ? streamGeneration : nil, + reconcile: reconcile + ) } catch is CancellationError { // Closing a thread cancels its read without changing its status. } catch { - if !Task.isCancelled, + if !reconcile, !Task.isCancelled, self.detailRefreshGeneration == generation, self.activeThreadID == threadID, self.activeRawThread == nil || self.detailStreamTask == nil { @@ -4376,6 +5015,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, id: route.uiID, state: warmConnectionID != nil ? .live : .catchingUp )) ensureDetailCatchUpFallback(route, generation: streamGeneration) + startSelectedThreadReconciliation(route, generation: streamGeneration) let retryDelay = threadRetryDelay detailStreamTask = Task { [weak self] in var failedAttempts = 0 @@ -4500,6 +5140,60 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, continuation.yield(.threadSync(id: route.uiID, state: .failed(message))) } + /// The selected transcript gets a bounded HTTP check even when its socket + /// remains healthy but subscription delivery stops. Other threads are untouched. + private func startSelectedThreadReconciliation(_ route: NativeThreadRoute, generation: Int) { + stopSelectedThreadReconciliation() + guard isForeground else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() + let sessionGeneration = environmentGeneration + let sleep = selectedThreadReconciliationSleep + selectedThreadReconciliationTask = Task { [weak self] in + while !Task.isCancelled { + let remaining = self.map { owner in + max(.zero, .seconds(30) - (owner.selectedThreadLastProgressAt ?? owner.selectedThreadReconciliationNow()) + .duration(to: owner.selectedThreadReconciliationNow())) + } ?? .seconds(30) + do { try await sleep(remaining) } catch { return } + guard !Task.isCancelled, let self, self.isForeground, + self.environmentGeneration == sessionGeneration, + self.isCurrentDetail(route, generation: generation) else { return } + if let progress = self.selectedThreadLastProgressAt, + progress.duration(to: self.selectedThreadReconciliationNow()) < .seconds(30) { + self.selectedThreadReconciliationReceipt(.deferred(threadID: route.uiID)) + continue + } + guard self.detailRefreshTask == nil, self.detailCatchUpTask == nil, + self.acceptedCommandRefreshes[route.uiID] == nil, + self.activeRawThread != nil, + self.activeThreadPage?.isLoading != true || self.pendingOlderThreadPage != nil else { + self.selectedThreadLastProgressAt = self.selectedThreadReconciliationNow() + self.selectedThreadReconciliationReceipt(.deferred(threadID: route.uiID)) + continue + } + self.scheduleDetailRefresh(threadID: route.uiID, client: route.client, reconcile: true) + self.selectedThreadReconciliationRefreshGeneration = self.detailRefreshGeneration + let refresh = self.detailRefreshTask + await refresh?.value + guard !Task.isCancelled, self.isForeground, + self.environmentGeneration == sessionGeneration, + self.isCurrentDetail(route, generation: generation) else { return } + self.selectedThreadReconciliationRefreshGeneration = nil + self.selectedThreadLastProgressAt = self.selectedThreadReconciliationNow() + self.selectedThreadReconciliationReceipt(.finished(threadID: route.uiID)) + } + } + } + + private func stopSelectedThreadReconciliation() { + selectedThreadReconciliationTask?.cancel() + selectedThreadReconciliationTask = nil + if selectedThreadReconciliationRefreshGeneration == detailRefreshGeneration { + resetDetailRefresh() + } + selectedThreadReconciliationRefreshGeneration = nil + } + private func isCurrentDetail(_ route: NativeThreadRoute, generation: Int) -> Bool { detailStreamGeneration == generation && activeThreadID == route.uiID @@ -4616,6 +5310,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, subscriptionEpoch < requiredEpoch { return } guard snapshot.snapshotSequence >= (activeThreadSequence ?? 0), activeRawThread == nil || snapshot.snapshotSequence > (activeThreadSequence ?? 0) else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() beginWarmReplayIfNeeded(route) resetDetailRefresh() detailSnapshotRequiredAfterEpoch = nil @@ -4642,7 +5337,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, scheduleDetailRefresh(threadID: route.uiID, client: route.client, force: true) return } - let reduction = NativeThreadDetailReducer.apply(event, to: current) + let reduction = NativeThreadDetailReducer.apply( + event, to: current, afterSequence: activeThreadSequence ?? 0 + ) if reduction.sequence < 0 { threadHistoryEpoch &+= 1 detailSnapshotRequiredAfterEpoch = threadHistoryEpoch @@ -4653,6 +5350,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, return } guard reduction.sequence > (activeThreadSequence ?? 0) else { return } + selectedThreadLastProgressAt = selectedThreadReconciliationNow() beginWarmReplayIfNeeded(route) switch reduction.result { case let .updated(thread): @@ -4685,8 +5383,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pendingDetailRenderMutations.formUnion(mutation) guard detailPublishTask == nil else { return } let streamGeneration = detailStreamGeneration + let sleep = detailPublicationSleep detailPublishTask = Task { [weak self] in - try? await Task.sleep(for: .milliseconds(80)) + do { try await sleep() } catch { return } guard let self else { return } guard !Task.isCancelled, self.detailStreamGeneration == streamGeneration, @@ -4752,6 +5451,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func resetDetailStream() { + stopSelectedThreadReconciliation() detailStreamGeneration &+= 1 detailCompletionReceived = false detailWasSynchronized = false @@ -4849,6 +5549,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) { let savedIDs = Set(savedEnvironments.map(\.id)) environmentClients = environmentClients.filter { savedIDs.contains($0.key) } + shellConnectionIDsByEnvironmentID = shellConnectionIDsByEnvironmentID.filter { savedIDs.contains($0.key) } shellsByEnvironmentID = shellsByEnvironmentID.filter { savedIDs.contains($0.key) } shellProjectionCache = shellProjectionCache.filter { savedIDs.contains($0.key) } serverConfigsByEnvironmentID = serverConfigsByEnvironmentID.filter { @@ -4870,31 +5571,33 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, savedIDs.contains($0.key) } - for load in loads { - environmentClients[load.environment.id] = load.client - if let config = load.config { - setServerConfig(config, environmentID: load.environment.id) - if load.environment.id == activeEnvironment?.id { - latestServerConfig = config - } + for load in loads { applyEnvironmentLoad(load) } + rebuildEntityIndexes(savedEnvironments) + } + + private func applyEnvironmentLoad(_ load: EnvironmentShellLoad) { + environmentClients[load.environment.id] = load.client + if let config = load.config { + setServerConfig(config, environmentID: load.environment.id) + if load.environment.id == activeEnvironment?.id { + latestServerConfig = config } - if let shell = load.shell { - if shell.snapshotSequence - >= (shellsByEnvironmentID[load.environment.id]?.snapshotSequence ?? .min) { - shellsByEnvironmentID[load.environment.id] = shell - } - environmentConnectionStates[load.environment.id] = .connected - environmentConnectionDetails[load.environment.id] = nil - } else if load.credentialRejected && load.environment.kind != .managedDPoP { - environmentConnectionStates[load.environment.id] = .needsPairing - environmentConnectionDetails[load.environment.id] = Self.needsPairingDetail - } else { - environmentConnectionStates[load.environment.id] = .disconnected - environmentConnectionDetails[load.environment.id] = - load.failureDetail ?? "That server is currently unreachable." + } + if let shell = load.shell { + if shell.snapshotSequence + >= (shellsByEnvironmentID[load.environment.id]?.snapshotSequence ?? .min) { + shellsByEnvironmentID[load.environment.id] = shell } + environmentConnectionStates[load.environment.id] = .connected + environmentConnectionDetails[load.environment.id] = nil + } else if load.credentialRejected && load.environment.kind != .managedDPoP { + environmentConnectionStates[load.environment.id] = .needsPairing + environmentConnectionDetails[load.environment.id] = Self.needsPairingDetail + } else { + environmentConnectionStates[load.environment.id] = .disconnected + environmentConnectionDetails[load.environment.id] = + load.failureDetail ?? "That server is currently unreachable." } - rebuildEntityIndexes(savedEnvironments) } static let needsPairingDetail = "This computer no longer accepts the saved pairing. Pair again." @@ -4906,10 +5609,13 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, pollingTask?.cancel() fallbackPollingTask?.cancel() configurationTask?.cancel() + // Rejected credentials cannot hydrate; stop the quiet shell reads too. + shellReconciliationTask?.cancel() pollingTask = nil fallbackPollingTask = nil configurationTask = nil - lastShellEventAt = nil + shellReconciliationTask = nil + activeStreamIsAuthoritative = false if activeEnvironment?.kind == .managedDPoP { emitConnection(.disconnected, detail: detail) } else { @@ -5042,42 +5748,60 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, indexedProvisionalRoutes = provisionalThreadRoutes } + /// A missing expectation means an ordinary publication. A present value + /// retains even an absent source cache or socket as an exact expectation. + private struct RefreshPublicationGuard { + let bootstrapID: UUID + let sourceSequence: Int? + let connectionID: UUID? + let activeAuthorityRevision: Int? + } + private func refresh(client: T3Client, includeArchived: Bool = false) async throws { let environment = client.environment let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID + let sourceSequence = shellsByEnvironmentID[environment.id]?.snapshotSequence + let authorityRevision = activeHTTPAuthorityRevision + // Archive RPC can establish the first socket. Bind the subsequent shell + // read to that socket instead of treating its creation as replacement. + let archivedShell = includeArchived ? try? await client.archivedShellSnapshot() : nil + let connectionID = await client.currentConnectionID() + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID else { throw CancellationError() } let shell = try await client.shellSnapshot() - guard isKnownClient(client, environmentID: environment.id, generation: generation) else { + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, + isKnownClient(client, environmentID: environment.id, generation: generation) else { throw CancellationError() } - guard shell.snapshotSequence - >= (shellsByEnvironmentID[environment.id]?.snapshotSequence ?? .min) else { - return - } - shellsByEnvironmentID[environment.id] = shell - if activeEnvironment?.id == environment.id { - latestShell = shell - } - if includeArchived, - let archivedShell = try? await client.archivedShellSnapshot(), - isKnownClient(client, environmentID: environment.id, generation: generation) { - archivedThreadsByEnvironmentID[environment.id] = archivedShell.threads.map { - mapThread($0, environment: environment) - } - archivedShellThreadsByEnvironmentID[environment.id] = Dictionary( - uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } - ) + let currentConnectionID = await client.currentConnectionID() + guard !Task.isCancelled, bootstrapID == foregroundBootstrapID, + isKnownClient(client, environmentID: environment.id, generation: generation), + connectionID == currentConnectionID, + sourceSequence == shellsByEnvironmentID[environment.id]?.snapshotSequence, + activeEnvironment?.id != environment.id || authorityRevision == activeHTTPAuthorityRevision else { + throw CancellationError() } - await emitSnapshot(shell, client: client, expectedGeneration: generation) + guard shell.snapshotSequence + >= (shellsByEnvironmentID[environment.id]?.snapshotSequence ?? .min) else { return } + await emitSnapshot( + shell, client: client, expectedGeneration: generation, + refreshGuard: RefreshPublicationGuard( + bootstrapID: bootstrapID, sourceSequence: sourceSequence, connectionID: connectionID, + activeAuthorityRevision: activeEnvironment?.id == environment.id ? authorityRevision : nil + ), archivedShell: archivedShell + ) } private func scheduleArchivedRefresh(client: T3Client, environment: Environment) { archivedRefreshTask?.cancel() let generation = environmentGeneration + let bootstrapID = foregroundBootstrapID archivedRefreshTask = Task { [weak self] in - guard let self, - let archivedShell = try? await client.archivedShellSnapshot(), + guard let self else { return } + defer { self.aggregateRefreshReceipt(.archiveFinished(environmentID: environment.id)) } + guard let archivedShell = try? await client.archivedShellSnapshot(), !Task.isCancelled, - self.isCurrentSession(client: client, generation: generation) else { + self.isCurrentForeground(client, generation: generation, bootstrapID: bootstrapID) else { return } self.archivedThreadsByEnvironmentID[environment.id] = archivedShell.threads.map { @@ -5087,7 +5811,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } ) if let shell = self.latestShell { - await self.emitSnapshot(shell, client: client, expectedGeneration: generation) + await self.emitSnapshot(shell, client: client, expectedGeneration: generation, markSourceConnected: false) } } } @@ -5103,7 +5827,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } private func refreshThread( - id: String, client: T3Client, expectedStreamGeneration: Int? = nil + id: String, client: T3Client, expectedStreamGeneration: Int? = nil, reconcile: Bool = false, + isSuperseded: (() -> Bool)? = nil ) async throws { let route = try threadRoute(for: id) guard route.client === client else { @@ -5115,16 +5840,100 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let supportsPagination = serverConfigsByEnvironmentID[ environment.id ]?.threadSnapshotPagination == true - let snapshot = try await client.threadSnapshot( + let loadedUsers = reconcile ? activeRawThread?.messages.filter { $0.role == "user" } ?? [] : [] + let loadedAnchor = loadedUsers.first?.id + let pendingHistory = reconcile ? pendingOlderThreadPage : nil + if let pendingHistory { + guard pendingHistory.epoch == historyEpoch, pendingHistory.threadID == route.uiID, + pendingHistory.environmentID == route.environmentID, + (pendingHistory.snapshot.page?.threadSequence ?? 0) >= (activeThreadSequence ?? 0) else { return } + } + let loadedIDs = Set(loadedUsers.map(\.id)) + // Pending rows determine the requested extent only. The published data + // always comes from a new authoritative snapshot, never a cached union. + let pendingUsers = pendingHistory?.snapshot.thread.messages.filter { + $0.role == "user" && !loadedIDs.contains($0.id) + } ?? [] + let retainedUsers = pendingUsers + loadedUsers + let retainedIDs = Set(retainedUsers.map(\.id)) + let retainedPage = activeThreadPage + let retainedThreads = reconcile ? [activeRawThread, pendingHistory?.snapshot.thread].compactMap { $0 } : [] + let retainedMessageIDs = Set(retainedThreads.flatMap { $0.messages.map(\.id) }) + let retainedActivityIDs = Set(retainedThreads.flatMap { $0.activities.map(\.id) }) + let retainedCheckpointTurns = Set(retainedThreads.flatMap { $0.checkpoints.map(\.turnId) }) + let turnLimit = max(Self.initialThreadUserTurnLimit, retainedUsers.count) + + func ownsReconciliationExtent() -> Bool { + !reconcile || ( + pendingOlderThreadPage == pendingHistory + && threadHistoryEpoch == historyEpoch + && activeThreadPage == retainedPage + && (activeThreadPage?.isLoading != true || pendingHistory != nil) + && loadedAnchor == activeRawThread?.messages.first(where: { $0.role == "user" })?.id + ) + } + func coversRetainedHistory(_ thread: OrchestrationThread) -> Bool { + // A server page can hit its raw-turn cap before reaching even one + // user row. Preserve every loaded collection, not just user anchors. + retainedMessageIDs.isSubset(of: Set(thread.messages.map(\.id))) + && retainedActivityIDs.isSubset(of: Set(thread.activities.map(\.id))) + && retainedCheckpointTurns.isSubset(of: Set(thread.checkpoints.map(\.turnId))) + } + func additionalTailUsers(in thread: OrchestrationThread) -> Int? { + let users = thread.messages.filter { $0.role == "user" } + guard let lastKnown = users.lastIndex(where: { retainedIDs.contains($0.id) }) else { return nil } + return users.distance(from: users.index(after: lastKnown), to: users.endIndex) + } + var snapshot = try await client.threadSnapshot( id: route.wireID, - turnLimit: supportsPagination ? Self.initialThreadUserTurnLimit : nil, + turnLimit: supportsPagination ? turnLimit : nil, timeoutInterval: threadSnapshotTimeoutInterval ) + guard !Task.isCancelled, + expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true, + isKnownClient(client, environmentID: environment.id, generation: generation) else { + throw CancellationError() + } + guard ownsReconciliationExtent() else { return } + // A later delivered cursor does not prove that every earlier row arrived. + // Independent HTTP repair must compare content, even at an equal watermark. + if reconcile, pendingHistory == nil, let current = activeRawThread, + snapshot.snapshotSequence >= (activeThreadSequence ?? 0), + snapshot.thread == current { + return + } + if reconcile, snapshot.page?.hasMore == true, + !coversRetainedHistory(snapshot.thread) { + if let added = additionalTailUsers(in: snapshot.thread), added > 0 { + // One extra user turn needs one extra slot, not an automatic + // page of older history. Do not keep expanding a moving head. + snapshot = try await client.threadSnapshot( + id: route.wireID, turnLimit: turnLimit + added, + timeoutInterval: threadSnapshotTimeoutInterval + ) + guard !Task.isCancelled, ownsReconciliationExtent(), + expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { return } + guard additionalTailUsers(in: snapshot.thread) == added else { return } + } + if snapshot.page?.hasMore == true, + !coversRetainedHistory(snapshot.thread) { + // Missing retained rows or a raw-turn cap get one full read. Never + // resurrect old rows by unioning them into a rewound thread. + snapshot = try await client.threadSnapshot( + id: route.wireID, timeoutInterval: threadSnapshotTimeoutInterval + ) + } + } guard !Task.isCancelled, isKnownClient(client, environmentID: environment.id, generation: generation), expectedStreamGeneration.map({ isCurrentDetail(route, generation: $0) }) ?? true else { throw CancellationError() } + // A newer accepted command makes this snapshot stale; its owner reads again. + if isSuperseded?() == true { return } + guard ownsReconciliationExtent() else { return } + if let pendingWatermark = pendingHistory?.snapshot.page?.threadSequence, + snapshot.snapshotSequence < pendingWatermark { return } if activeThreadID == route.uiID { if activeRawThread == nil, historyEpoch != threadHistoryEpoch { return @@ -5167,7 +5976,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, wasSynchronized: false, connectionID: nil ) - if activeThreadID == route.uiID, + if !reconcile, activeThreadID == route.uiID, detailCompletionReceived || serverConfigsByEnvironmentID[environment.id]?.threadResumeCompletionMarker != true { markDetailSynchronized(route) @@ -5180,16 +5989,32 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, _ shell: OrchestrationShellSnapshot, client sourceClient: T3Client, expectedGeneration: Int, + refreshGuard: RefreshPublicationGuard? = nil, + archivedShell: OrchestrationShellSnapshot? = nil, markSourceConnected: Bool = true ) async { let sourceEnvironment = sourceClient.environment guard !Task.isCancelled, + refreshGuard.map({ + $0.bootstrapID == foregroundBootstrapID + && $0.sourceSequence == shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence + && ($0.activeAuthorityRevision.map { $0 == activeHTTPAuthorityRevision } ?? true) + }) ?? true, let environment = activeEnvironment, isKnownClient( sourceClient, environmentID: sourceEnvironment.id, generation: expectedGeneration ) else { return } let environments = (try? await runtime.environments()) ?? [environment] + // The metadata await can outlive the socket too. Read its identity last, + // then validate all expectations together before any cache mutation. + let currentConnectionID = refreshGuard == nil ? nil : await sourceClient.currentConnectionID() guard !Task.isCancelled, + refreshGuard.map({ $0.connectionID == currentConnectionID }) ?? true, + refreshGuard.map({ + $0.bootstrapID == foregroundBootstrapID + && $0.sourceSequence == shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence + && ($0.activeAuthorityRevision.map { $0 == activeHTTPAuthorityRevision } ?? true) + }) ?? true, isKnownClient( sourceClient, environmentID: sourceEnvironment.id, generation: expectedGeneration ), @@ -5199,6 +6024,17 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, >= (shellsByEnvironmentID[sourceEnvironment.id]?.snapshotSequence ?? .min) else { return } + if let archivedShell { + archivedThreadsByEnvironmentID[sourceEnvironment.id] = archivedShell.threads.map { + mapThread($0, environment: sourceEnvironment) + } + archivedShellThreadsByEnvironmentID[sourceEnvironment.id] = Dictionary( + uniqueKeysWithValues: archivedShell.threads.map { ($0.id, $0) } + ) + } + if let refreshGuard { + shellConnectionIDsByEnvironmentID[sourceEnvironment.id] = refreshGuard.connectionID + } shellsByEnvironmentID[sourceEnvironment.id] = shell if markSourceConnected { environmentConnectionStates[sourceEnvironment.id] = .connected @@ -5206,6 +6042,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } if sourceEnvironment.id == environment.id { latestShell = shell + if markSourceConnected { + activeHasHydrated = true + activeShellEpochHasSnapshot = true + activeHTTPAuthorityRevision &+= 1 + } } rebuildEntityIndexes(environments) synchronizeActiveDetail( @@ -5215,8 +6056,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let connectionState: FeatureConnection.State let connectionDetail: String? if sourceEnvironment.id == environment.id, markSourceConnected { - connectionState = .connected - connectionDetail = nil + connectionState = activeStreamIsAuthoritative ? .connected : .reconnecting + connectionDetail = activeStreamIsAuthoritative ? nil + : latestSnapshot?.connection.detail ?? "Live updates reconnecting. Refreshing over HTTP." } else { connectionState = latestSnapshot?.connection.state ?? environmentConnectionStates[environment.id] @@ -5241,10 +6083,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, guard activeThreadEnvironmentID == environment.id, let threadID = activeThreadID, let wireID = threadWireIDs[threadID], - let shellThread = shell.threads.first(where: { $0.id == wireID }), - var detail = latestDetails[threadID] else { + let shellThread = shell.threads.first(where: { $0.id == wireID }) else { return } + repairMissingCompletedTurn(shellThread, sequence: shell.snapshotSequence, threadID: threadID) + guard var detail = latestDetails[threadID] else { return } let backgroundLiveness = shellThread.backgroundLiveness let backgroundWorkIsActive = backgroundLiveness == .working @@ -5255,22 +6098,26 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, detail.thread.supportsTitleRegeneration = capabilities?.threadTitleRegeneration detail.thread.supportsPullRequestLinking = capabilities?.threadPullRequestLinking detail.thread.supportsMultiplePullRequests = capabilities?.threadPullRequests - let sessionIsLive = shellThread.session?.status == "starting" - || shellThread.session?.status == "running" - detail.thread.state = Self.resolveThreadState( - latestTurn: shellThread.latestTurn, - session: shellThread.session, - hasApprovals: !detail.approvals.isEmpty, - hasUserInput: !detail.userInputs.isEmpty, - backgroundLiveness: backgroundLiveness - ) - detail.thread.workingStartedAt = workingStartedAt( - latestTurn: shellThread.latestTurn, - session: shellThread.session, - backgroundWorkIsActive: backgroundWorkIsActive, - fallbackUpdatedAt: shellThread.updatedAt - ) - if shell.snapshotSequence >= (activeThreadSequence ?? .min) { + let shellMetadataIsCurrent = shell.snapshotSequence >= (activeThreadSequence ?? .min) + let latestTurn = shellMetadataIsCurrent ? shellThread.latestTurn : activeRawThread?.latestTurn + let session = shellMetadataIsCurrent ? shellThread.session : activeRawThread?.session + let sessionIsLive = session?.status == "starting" || session?.status == "running" + if shellMetadataIsCurrent || activeRawThread != nil { + detail.thread.state = Self.resolveThreadState( + latestTurn: latestTurn, + session: session, + hasApprovals: !detail.approvals.isEmpty, + hasUserInput: !detail.userInputs.isEmpty, + backgroundLiveness: backgroundLiveness + ) + detail.thread.workingStartedAt = workingStartedAt( + latestTurn: latestTurn, + session: session, + backgroundWorkIsActive: backgroundWorkIsActive, + fallbackUpdatedAt: shellThread.updatedAt + ) + } + if shellMetadataIsCurrent { applyShellMetadataAuthority(from: shellThread, to: &detail.thread) if let compaction = detailRenderCaches[threadID]?.compaction { detail.isCompacting = compaction.isActive( @@ -5288,6 +6135,44 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, publish(detail, threadID: threadID, renderCacheIsSource: true) } + private func repairMissingCompletedTurn( + _ shellThread: OrchestrationThreadShell, sequence: Int, threadID: String + ) { + guard let rawThread = activeRawThread, + sequence >= (activeThreadSequence ?? .min), + let completed = shellThread.latestTurn, completed.state == "completed", + let route = try? threadRoute(for: threadID) else { return } + if let messageID = completed.assistantMessageId { + if let message = rawThread.messages.first(where: { $0.id == messageID }), + !message.streaming { + flushDetailPublish(route) + return + } + } else if rawThread.latestTurn?.turnId == completed.turnId, + rawThread.latestTurn?.state == completed.state, + rawThread.latestTurn?.completedAt == completed.completedAt, + rawThread.messages.contains(where: { + $0.role == "assistant" && $0.turnId == completed.turnId && !$0.streaming + }), + !rawThread.messages.contains(where: { + $0.role == "assistant" && $0.turnId == completed.turnId && $0.streaming + }) { + flushDetailPublish(route) + return + } + + // Equal-cursor completion may flush final content, but only a newer + // shell can prove that detail is missing and require HTTP repair. + guard sequence > (activeThreadSequence ?? .min) else { return } + + // Keep rendered partial text while requiring a snapshot that includes + // the shell's completion. Later detail events advance this same floor. + flushDetailPublish(route) + activeRawThread = nil + activeThreadSequence = sequence + scheduleDetailRefresh(threadID: threadID, client: route.client, force: true) + } + /// Thread-only shell changes stay granular so Home does not replace and /// diff the aggregate snapshot for every active turn update. Structural /// changes retain the canonical snapshot event as a safe fallback. @@ -5510,14 +6395,29 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environments: [Environment], activeEnvironment: Environment, connectionState: FeatureConnection.State, - connectionDetail: String? = nil + connectionDetail: String? = nil, + changedEnvironmentIDs: Set? = nil ) -> FeatureSnapshot { let enabledEnvironments = environments.filter(\.isEnabled) let enabledIDs = Set(enabledEnvironments.map(\.id)) shellProjectionCache = shellProjectionCache.filter { enabledIDs.contains($0.key) } var threads: [FeatureThread] = [] var projects: [FeatureProject] = [] + let previousThreads = changedEnvironmentIDs == nil ? [:] + : Dictionary(grouping: latestSnapshot?.threads ?? [], by: \.environmentID) + let previousProjects = changedEnvironmentIDs == nil ? [:] + : Dictionary(grouping: latestSnapshot?.projects ?? [], by: \.environmentID) + let previousEnvironments = Dictionary(uniqueKeysWithValues: + (latestSnapshot?.environments ?? []).map { ($0.id, $0) } + ) for environment in enabledEnvironments { + if let changedEnvironmentIDs, !changedEnvironmentIDs.contains(environment.id), + previousEnvironments[environment.id] + == mapEnvironment(environment, activeID: activeEnvironment.id) { + threads.append(contentsOf: previousThreads[environment.id] ?? []) + projects.append(contentsOf: previousProjects[environment.id] ?? []) + continue + } // Take ownership while updating so the cache does not copy its // retained arrays when one row changes. var projection = shellProjectionCache.removeValue(forKey: environment.id) @@ -5597,6 +6497,11 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let providersByEnvironment = enabledEnvironments.reduce( into: [String: [FeatureProvider]]() ) { catalogues, environment in + if let changedEnvironmentIDs, !changedEnvironmentIDs.contains(environment.id), + let previous = latestSnapshot?.providersByEnvironment?[environment.id] { + catalogues[environment.id] = previous + return + } guard let shell = shellsByEnvironmentID[environment.id] else { return } catalogues[environment.id] = mapProviders( environmentID: environment.id, @@ -6696,6 +7601,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, providerCatalogCache[environmentID] = nil } serverConfigsByEnvironmentID[environmentID] = config + aggregateRefreshReceipt(.configurationApplied(environmentID: environmentID)) } private func mapProviders( @@ -7704,7 +8610,8 @@ struct NativeThreadDetailReduction: Equatable { enum NativeThreadDetailReducer { static func apply( _ event: JSONValue, - to thread: OrchestrationThread + to thread: OrchestrationThread, + afterSequence: Int? = nil ) -> NativeThreadDetailReduction { guard case let .object(object) = event, let type = object["type"]?.stringValue, @@ -7719,6 +8626,13 @@ enum NativeThreadDetailReducer { ) } + // Validate ownership and the common envelope before skipping replayed events. + if let afterSequence, sequence >= 0, sequence <= afterSequence { + return NativeThreadDetailReduction( + sequence: sequence, result: .unchanged, renderMutation: .none + ) + } + let result: NativeThreadDetailReductionResult var renderMutation = NativeDetailRenderMutation.metadata switch type { @@ -8338,7 +9252,7 @@ private struct NativeProjectRoute { let client: T3Client } -private struct PendingOlderThreadPage { +private struct PendingOlderThreadPage: Equatable { let snapshot: OrchestrationThreadDetailSnapshot let epoch: Int let threadID: String @@ -8489,3 +9403,11 @@ private enum NativeFeatureClientError: LocalizedError { } } } + +/// Deterministic receipts at the passive transport/publication boundary. +enum NativePassiveShellReceipt: Sendable { + case shellApplied(environmentID: String, sequence: Int) + case httpFinished(environmentID: String) + case configurationApplied(environmentID: String) + case archiveFinished(environmentID: String) +} diff --git a/apps/swift-ios/Core/T3Client.swift b/apps/swift-ios/Core/T3Client.swift index 0c8659ac4e58..902c5fa081e5 100644 --- a/apps/swift-ios/Core/T3Client.swift +++ b/apps/swift-ios/Core/T3Client.swift @@ -638,6 +638,29 @@ public actor T3Client { ) } + /// Passive shells start with a full snapshot on each socket subscription. + /// Registering its identity atomically prevents late socket results from + /// acquiring authority over a replacement connection. + public func shellEventsOnCurrentConnection() async throws -> ( + events: AsyncThrowingStream, connectionID: UUID + ) { + try await rpc.subscribeOnCurrentConnection( + RPCMethod.subscribeShell.rawValue, + payload: .object(["requestCompletionMarker": .bool(true)]), + as: ShellStreamItem.self + ) + } + + public func shellEventBatchesOnCurrentConnection() async throws -> ( + events: AsyncThrowingStream<[ShellStreamItem], Error>, connectionID: UUID + ) { + try await rpc.subscribeBatchesOnCurrentConnection( + RPCMethod.subscribeShell.rawValue, + payload: .object(["requestCompletionMarker": .bool(true)]), + as: ShellStreamItem.self + ) + } + public func threadEventBatches( threadID: String, after sequence: Int? = nil, diff --git a/apps/swift-ios/Features/Chat/ThreadDetailView.swift b/apps/swift-ios/Features/Chat/ThreadDetailView.swift index 6483b15c0ef4..d6a545b7f4f2 100644 --- a/apps/swift-ios/Features/Chat/ThreadDetailView.swift +++ b/apps/swift-ios/Features/Chat/ThreadDetailView.swift @@ -14,6 +14,7 @@ public struct ThreadDetailView: View { let submitMessage: (FeatureMessageSubmission) async -> Bool let onNavigateBack: () -> Void private let draftStore: FeatureComposerDraftStore + private let managesThreadPresentation: Bool @State private var draft = "" @State private var composerContext: OrchestrationMessageContext? @@ -52,20 +53,22 @@ public struct ThreadDetailView: View { thread: FeatureThread, submitMessage: @escaping (FeatureMessageSubmission) async -> Bool, onNavigateBack: @escaping () -> Void = {}, - draftStore: FeatureComposerDraftStore = .shared + draftStore: FeatureComposerDraftStore = .shared, + managesThreadPresentation: Bool = true ) { self.model = model self.thread = thread self.submitMessage = submitMessage self.onNavigateBack = onNavigateBack self.draftStore = draftStore + self.managesThreadPresentation = managesThreadPresentation } private var threadContent: some View { Group { if let detail { timeline(detail) - } else if isLoading { + } else if isOpening { FeatureThreadOpeningView() } else { ContentUnavailableView { @@ -102,16 +105,27 @@ public struct ThreadDetailView: View { } } .task(id: thread.id) { + guard managesThreadPresentation else { return } isLoading = true - _ = await model.detail(for: thread.id, force: true) - isLoading = false + await model.runThreadPresentation(id: thread.id) { + isLoading = false + } } .task(id: thread.id) { // A cached thread can already show its composer while the server // is catching up. Local drafts must not wait for that request. - await model.checkRewindRecovery(for: currentThread) - guard !didRestoreDraft else { return } - await restoreDraft(from: composerDraft, key: draftKey) + // When a thread is reopened in the compact split view, SwiftUI can + // report this view as disappeared right after it appears while it + // stays on screen. That cancels this task and it never re-runs, so + // the composer would stay busy. Finish the restore outside the + // view's task lifetime; each thread has its own view identity, so + // a late restore cannot reach another thread's composer. + let restore = Task { @MainActor in + await model.checkRewindRecovery(for: currentThread) + guard !didRestoreDraft else { return } + await restoreDraft(from: composerDraft, key: draftKey) + } + await restore.value } .task(id: pullRequestObservationID) { await observeThreadPullRequest() @@ -133,7 +147,7 @@ public struct ThreadDetailView: View { .onChange(of: threadConnectionState) { _, state in if state == .connected, case .failed = model.detailLoadStates[thread.id], - !isLoading { + !isOpening { reloadThread() } } @@ -143,7 +157,6 @@ public struct ThreadDetailView: View { } } .onDisappear { - model.releaseThread(thread.id) persistDraftBeforeLeaving() } .sheet(item: $toolSurface) { surface in @@ -277,6 +290,12 @@ public struct ThreadDetailView: View { } } + private var isOpening: Bool { + if managesThreadPresentation { return isLoading } + if case .failed = model.detailLoadStates[thread.id] { return false } + return detail == nil || model.detailLoadStates[thread.id] == .loading + } + private var detail: FeatureThreadDetail? { model.details[thread.id] } @@ -683,10 +702,8 @@ public struct ThreadDetailView: View { } private func reloadThread() { - isLoading = true - Task { - _ = await model.detail(for: thread.id, force: true, fresh: true) - isLoading = false + if model.refreshThreadPresentation(id: thread.id, onLoaded: { isLoading = false }) { + isLoading = true } } @@ -699,7 +716,7 @@ public struct ThreadDetailView: View { ThreadRefreshPresentation.resolve( loadState: model.detailLoadStates[thread.id], connectionState: threadConnectionState, - isOpening: isLoading, + isOpening: isOpening, syncState: model.threadSyncStates[thread.id] ) } diff --git a/apps/swift-ios/Features/Root/FeatureRootModel.swift b/apps/swift-ios/Features/Root/FeatureRootModel.swift index cb91af596570..3ad36c6bc102 100644 --- a/apps/swift-ios/Features/Root/FeatureRootModel.swift +++ b/apps/swift-ios/Features/Root/FeatureRootModel.swift @@ -90,11 +90,19 @@ public final class FeatureRootModel { draftStore: draftStore ) private var pendingSubmissionsByID: [String: FeatureQueuedSubmission] = [:] + /// Accepted messages not yet present in a server transcript, by message ID. + private var deliveredAwaitingDetail: [String: (threadID: String, message: FeatureMessage)] = [:] + /// Message IDs in each thread's latest server transcript. + private var serverMessageIDs: [String: Set] = [:] private var activeSubmissionCounts: [String: Int] = [:] private var pendingThreadsByID: [String: FeatureThread] = [:] private var pendingSettlementMutations: [String: PendingSettlementMutation] = [:] private var pendingCompletionSubmissionIDs: Set = [] private var pendingDiscardSubmissionIDs: Set = [] + @ObservationIgnored private var activeDetailPresentation: ( + threadID: String, owner: UUID, lifetime: AsyncStream.Continuation, + refreshTask: Task? + )? private var detailRecency: [String] = [] private var detailLoadGeneration: UInt64 = 0 private var detailLoadRevisions: [String: UInt64] = [:] @@ -149,6 +157,7 @@ public final class FeatureRootModel { func applicationDidEnterBackground(at date: Date = .now) { backgroundedAt = date + client.suspendForBackground() } func applicationDidBecomeActive(at date: Date = .now) async { @@ -721,6 +730,55 @@ public final class FeatureRootModel { } } + /// Holds selected-thread transport for the owning presentation task, including after loading. + func runThreadPresentation(id: String, onLoaded: @MainActor () -> Void = {}) async { + guard !Task.isCancelled else { return } + if let previous = activeDetailPresentation { + releaseThread(previous.threadID) + } + let owner = UUID() + let lifetime = AsyncStream.makeStream() + activeDetailPresentation = (id, owner, lifetime.continuation, nil) + await withTaskCancellationHandler { + defer { releaseThreadPresentation(id: id, owner: owner) } + _ = await detail(for: id, force: true) + guard !Task.isCancelled, + activeDetailPresentation?.owner == owner else { return } + onLoaded() + for await _ in lifetime.stream {} + } onCancel: { + // Cancellation must release even when the detail read is still suspended. + Task { @MainActor [weak self] in + self?.releaseThreadPresentation(id: id, owner: owner) + } + } + } + + /// Retries belong to the current presentation and cannot outlive a close or replacement. + @discardableResult + func refreshThreadPresentation( + id: String, fresh: Bool = true, onLoaded: @escaping @MainActor () -> Void = {} + ) -> Bool { + guard let presentation = activeDetailPresentation, presentation.threadID == id else { return false } + presentation.refreshTask?.cancel() + let owner = presentation.owner + activeDetailPresentation?.refreshTask = Task { [weak self] in + guard let self, !Task.isCancelled, + self.activeDetailPresentation?.owner == owner else { return } + _ = await self.detail(for: id, force: true, fresh: fresh) + guard !Task.isCancelled, + self.activeDetailPresentation?.owner == owner else { return } + onLoaded() + } + return true + } + + private func releaseThreadPresentation(id: String, owner: UUID) { + guard activeDetailPresentation?.threadID == id, + activeDetailPresentation?.owner == owner else { return } + releaseThread(id) + } + public func detail(for id: String, force: Bool = false, fresh: Bool = false) async -> FeatureThreadDetail? { if !force, let cached = details[id] { return cached @@ -799,6 +857,11 @@ public final class FeatureRootModel { /// Ends any selected-thread transport work when its detail view closes. public func releaseThread(_ id: String) { + if let presentation = activeDetailPresentation, presentation.threadID == id { + activeDetailPresentation = nil + presentation.refreshTask?.cancel() + presentation.lifetime.finish() + } client.releaseThread(id: id) markDetailRecentlyUsed(id) evictOldThreadDetailsIfNeeded() @@ -1317,6 +1380,7 @@ public final class FeatureRootModel { } private func removeThread(id: String) { + forgetDeliveredMessages(threadID: id) guard let index = snapshot.threads.firstIndex(where: { $0.id == id }) else { return } let projectID = snapshot.threads[index].projectID snapshot.threads.remove(at: index) @@ -1449,15 +1513,18 @@ public final class FeatureRootModel { } } + /// `isServerTranscript` is false for details built locally, such as a + /// restored outbox or a pending creation; they cannot confirm delivery. private func store( _ incoming: FeatureThreadDetail, - invalidatesInFlightLoad: Bool = true + invalidatesInFlightLoad: Bool = true, + isServerTranscript: Bool = true ) { var incoming = retainingLocalAttachmentPreviews(in: incoming) incoming.thread = retainingPendingSettlement(in: incoming.thread) let id = incoming.thread.id acknowledgeDeliveredMessages(incoming.messages) - let prepared = addingPendingMessages(to: incoming) + let prepared = addingPendingMessages(to: incoming, isServerTranscript: isServerTranscript) let next = details[id].map { current in FeatureThreadDetail( thread: prepared.thread, @@ -1484,7 +1551,7 @@ public final class FeatureRootModel { incoming.thread = retainingPendingSettlement(in: incoming.thread) let id = incoming.thread.id acknowledgeDeliveredMessages(incoming.messages) - let next = addingPendingMessages(to: incoming) + let next = addingPendingMessages(to: incoming, isServerTranscript: true) details[id] = next markDetailRecentlyUsed(id) bumpDetailLoadRevision(id: id) @@ -1523,7 +1590,18 @@ public final class FeatureRootModel { return true } - private func removeDetail(id: String) { + private func forgetDeliveredMessages(threadID: String) { + deliveredAwaitingDetail = deliveredAwaitingDetail.filter { $0.value.threadID != threadID } + serverMessageIDs[threadID] = nil + } + + private func removeDetail(id: String, forgettingDeliveredMessages: Bool = true) { + if forgettingDeliveredMessages { + forgetDeliveredMessages(threadID: id) + } else { + // The next server transcript records these again; keep the cache bounded. + serverMessageIDs[id] = nil + } if details.removeValue(forKey: id) != nil { detailRecency.removeAll { $0 == id } } @@ -1536,6 +1614,8 @@ public final class FeatureRootModel { } private func clearDetails() { + deliveredAwaitingDetail.removeAll() + serverMessageIDs.removeAll() detailLoadGeneration &+= 1 detailLoadRevisions.removeAll() storedDetailLoadRequestRevisions.removeAll() @@ -1571,7 +1651,8 @@ public final class FeatureRootModel { while details.count > Self.maximumRetainedThreadDetails, let candidate = detailRecency.first(where: { !protected.contains($0) }) { detailRecency.removeAll { $0 == candidate } - removeDetail(id: candidate) + // Eviction only drops the cache; accepted messages must survive a reopen. + removeDetail(id: candidate, forgettingDeliveredMessages: false) } } @@ -1614,7 +1695,7 @@ public final class FeatureRootModel { if snapshot.threads.contains(where: { $0.id == submission.threadID }) { pendingSubmissionsByID[submission.id] = submission if let detail = details[submission.threadID] { - store(addingPendingMessages(to: detail)) + store(detail, isServerTranscript: false) } continue } @@ -1645,7 +1726,7 @@ public final class FeatureRootModel { } pendingSubmissionsByID[submission.id] = submission if let detail = details[submission.threadID] { - store(addingPendingMessages(to: detail)) + store(detail, isServerTranscript: false) } } } @@ -1708,7 +1789,7 @@ public final class FeatureRootModel { store(FeatureThreadDetail( thread: thread, messages: [queuedMessage(for: submission)] - )) + ), isServerTranscript: false) } private func provider(id: String?, environmentID: String) -> FeatureProvider? { @@ -1737,16 +1818,30 @@ public final class FeatureRootModel { ) } - private func addingPendingMessages(to incoming: FeatureThreadDetail) -> FeatureThreadDetail { + private func addingPendingMessages( + to incoming: FeatureThreadDetail, + isServerTranscript: Bool + ) -> FeatureThreadDetail { + let existing = Set(incoming.messages.map(\.id)) + // A delivered message stays visible until a server transcript includes it. + // Its refresh no longer blocks the send, so an older read can arrive first. + if isServerTranscript { + serverMessageIDs[incoming.thread.id] = existing + for (id, delivered) in deliveredAwaitingDetail + where delivered.threadID == incoming.thread.id && existing.contains(id) { + deliveredAwaitingDetail.removeValue(forKey: id) + } + } + let delivered = deliveredAwaitingDetail.values + .filter { $0.threadID == incoming.thread.id && !existing.contains($0.message.id) } + .map(\.message) let queued = pendingSubmissionsByID.values - .filter { $0.threadID == incoming.thread.id } - .sorted { $0.identity.createdAt < $1.identity.createdAt } - guard !queued.isEmpty else { return incoming } + .filter { $0.threadID == incoming.thread.id && !existing.contains($0.identity.messageID) } + .map(queuedMessage(for:)) + guard !queued.isEmpty || !delivered.isEmpty else { return incoming } var result = incoming - let existing = Set(result.messages.map(\.id)) - result.messages.append(contentsOf: queued.lazy - .filter { !existing.contains($0.identity.messageID) } - .map(queuedMessage(for:))) + // A newer send can be delivered while an older one waits to retry; keep send order. + result.messages.append(contentsOf: (delivered + queued).sorted { $0.createdAt < $1.createdAt }) return result } @@ -1823,6 +1918,12 @@ public final class FeatureRootModel { } pendingCompletionSubmissionIDs.remove(submission.id) pendingSubmissionsByID.removeValue(forKey: submission.id) + // A transcript that already includes it confirmed delivery; do not retain it. + if serverMessageIDs[submission.threadID]?.contains(submission.identity.messageID) != true { + var message = queuedMessage(for: submission) + message.state = .complete + deliveredAwaitingDetail[submission.identity.messageID] = (submission.threadID, message) + } setAttachmentOutboxOwnership(false, for: submission) pendingThreadsByID.removeValue(forKey: submission.threadID) markQueuedMessageDelivered(submission) diff --git a/apps/swift-ios/Features/Shared/FeatureClient.swift b/apps/swift-ios/Features/Shared/FeatureClient.swift index 5b791cca92cf..8a99278d612a 100644 --- a/apps/swift-ios/Features/Shared/FeatureClient.swift +++ b/apps/swift-ios/Features/Shared/FeatureClient.swift @@ -10,6 +10,7 @@ public protocol FeatureClient: AnyObject { func backgroundSnapshot() async throws -> FeatureSnapshot func events() -> AsyncStream func resumeAfterBackground(reconnect: Bool) async + func suspendForBackground() func preuploadAttachment( _ attachment: FeatureUploadAttachment, @@ -314,6 +315,8 @@ public extension FeatureClient { func resumeAfterBackground(reconnect: Bool) async {} + func suspendForBackground() {} + func preuploadAttachment( _ attachment: FeatureUploadAttachment, environmentID: String diff --git a/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift b/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift new file mode 100644 index 000000000000..70c524635174 --- /dev/null +++ b/apps/swift-ios/Features/Workspace/WorkspaceThreadSelection.swift @@ -0,0 +1,25 @@ +import Foundation + +struct WorkspaceThreadSelection: Equatable { + private(set) var selectedID: String? + private(set) var lastOpenedID: String? + + var highlightedID: String? { selectedID ?? lastOpenedID } + + mutating func open(_ id: String) { + selectedID = id + lastOpenedID = id + } + + func presentedID(isCompact: Bool, showsDetailColumn: Bool) -> String? { + isCompact && !showsDetailColumn ? nil : selectedID + } + + mutating func reconcilePresentation(isCompact: Bool, showsDetailColumn: Bool) { + if isCompact && !showsDetailColumn { close() } + } + + mutating func close() { + selectedID = nil + } +} diff --git a/apps/swift-ios/Features/Workspace/WorkspaceView.swift b/apps/swift-ios/Features/Workspace/WorkspaceView.swift index 2a0d3a448149..9e13c977dda4 100644 --- a/apps/swift-ios/Features/Workspace/WorkspaceView.swift +++ b/apps/swift-ios/Features/Workspace/WorkspaceView.swift @@ -18,24 +18,9 @@ struct FeatureWorkspaceNavigationRequest: Equatable, Sendable { } } -struct WorkspaceThreadSelection: Equatable { - private(set) var selectedID: String? - private(set) var lastOpenedID: String? - - var highlightedID: String? { selectedID ?? lastOpenedID } - - mutating func open(_ id: String) { - selectedID = id - lastOpenedID = id - } - - mutating func close() { - selectedID = nil - } -} - public struct WorkspaceView: View { @SwiftUI.Environment(\.dynamicTypeSize) private var dynamicTypeSize + @SwiftUI.Environment(\.horizontalSizeClass) private var horizontalSizeClass @Bindable var model: FeatureRootModel private let navigationRequest: FeatureWorkspaceNavigationRequest? @@ -81,8 +66,8 @@ public struct WorkspaceView: View { self.submitMessage = { submission in await model.sendMessage(submission) } } - public var body: some View { - NavigationSplitView(preferredCompactColumn: $preferredCompactColumn) { + private var workspaceNavigation: some View { + NavigationSplitView(preferredCompactColumn: compactColumnBinding) { sidebar .navigationSplitViewColumnWidth( min: T3Metrics.minimumSidebarWidth, @@ -93,6 +78,17 @@ public struct WorkspaceView: View { detail } .navigationSplitViewStyle(.balanced) + .task(id: presentedThreadID) { [id = presentedThreadID] in + guard let id else { return } + await model.runThreadPresentation(id: id) + } + .onChange(of: horizontalSizeClass) { _, _ in + reconcileThreadPresentation() + } + } + + public var body: some View { + workspaceNavigation .sheet(isPresented: $showingNewTask) { NewThreadView( model: model, @@ -297,7 +293,8 @@ public struct WorkspaceView: View { model: model, thread: thread, submitMessage: submitMessage, - onNavigateBack: closeSelectedThread + onNavigateBack: closeSelectedThread, + managesThreadPresentation: false ) .id(id) } else { @@ -600,6 +597,31 @@ public struct WorkspaceView: View { return model.snapshot.projects.contains { $0.id == selectedProjectID } } + private var compactColumnBinding: Binding { + Binding( + get: { preferredCompactColumn }, + set: { column in + preferredCompactColumn = column + reconcileThreadPresentation() + } + ) + } + + private var presentedThreadID: String? { + guard let id = threadSelection.presentedID( + isCompact: horizontalSizeClass == .compact, + showsDetailColumn: preferredCompactColumn == .detail + ), model.snapshot.threads.contains(where: { $0.id == id }) else { return nil } + return id + } + + private func reconcileThreadPresentation() { + threadSelection.reconcilePresentation( + isCompact: horizontalSizeClass == .compact, + showsDetailColumn: preferredCompactColumn == .detail + ) + } + private func openThread(_ id: String) { threadSelection.open(id) preferredCompactColumn = .detail diff --git a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift index 241354476b03..9e0704ff076e 100644 --- a/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/FeatureRootModelTests.swift @@ -9,6 +9,163 @@ import XCTest @MainActor @Suite("Feature root model") struct FeatureRootModelTests { + @Test(arguments: [["a", "b", "a"], ["a", "a"]]) + func presentationTaskReplacementAndCloseReleaseExactlyOnce(threadIDs: [String]) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + var loads = loaded.stream.makeAsyncIterator() + var previous: Task? + for id in threadIDs { + let next = Task { + await model.runThreadPresentation(id: id) { loaded.continuation.yield(id) } + } + #expect(await loads.next() == id) + // Replacement finishes the old lifetime even before SwiftUI cancels its task. + await previous?.value + previous?.cancel() + previous = next + } + #expect(client.loadedThreadIDs == threadIDs) + #expect(client.releasedThreadIDs == Array(threadIDs.dropLast())) + previous?.cancel() + await previous?.value + #expect(client.releasedThreadIDs == threadIDs) + loaded.continuation.finish() + } + + @Test + func cancelledPresentationTaskCannotReplaceVisibleThread() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let current = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + let cancelled = Task { @MainActor in + withUnsafeCurrentTask { $0?.cancel() } + await model.runThreadPresentation(id: "a") + } + await cancelled.value + #expect(client.loadedThreadIDs == ["a"]) + #expect(client.releasedThreadIDs.isEmpty) + current.cancel() + await current.value + #expect(client.releasedThreadIDs == ["a"]) + loaded.continuation.finish() + } + + @Test(arguments: [false, true]) + func cancellationReleasesSuspendedPresentationBeforeItsLoadReturns(cancelBeforeReplacement: Bool) async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let started = AsyncStream.makeStream() + let released = AsyncStream.makeStream() + let loaded = AsyncStream.makeStream() + client.onReleaseThread = { released.continuation.yield($0) } + var pendingLoad: CheckedContinuation? + client.loadThreadHandler = { _ in + try await withCheckedThrowingContinuation { continuation in + pendingLoad = continuation + started.continuation.yield(()) + } + } + var oldLoaded = false + let old = Task { await model.runThreadPresentation(id: "a") { oldLoaded = true } } + var starts = started.stream.makeAsyncIterator() + await starts.next() + var releases = released.stream.makeAsyncIterator() + if cancelBeforeReplacement { + old.cancel() + #expect(await releases.next() == "a") + } + #expect(pendingLoad != nil) + #expect(!oldLoaded) + + client.loadThreadHandler = nil + let replacement = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + if !cancelBeforeReplacement { + // The old cancellation handler runs only after the same-ID replacement owns transport. + old.cancel() + #expect(await releases.next() == "a") + } + pendingLoad?.resume(returning: FeatureThreadDetail(thread: client.createdThread)) + pendingLoad = nil + await old.value + #expect(!oldLoaded) + #expect(client.releasedThreadIDs == ["a"]) + replacement.cancel() + await replacement.value + #expect(client.releasedThreadIDs == ["a", "a"]) + #expect(client.loadedThreadIDs == ["a", "a"]) + started.continuation.finish() + released.continuation.finish() + loaded.continuation.finish() + } + + @Test + func presentationCloseStillReleasesAfterConnectionReload() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + await model.reloadAfterConnection() + presentation.cancel() + await presentation.value + #expect(client.releasedThreadIDs == ["a"]) + loaded.continuation.finish() + } + + @Test + func presentationCloseCancelsRetryAndRejectsLateRetry() async { + let client = FeatureClientStub() + let model = FeatureRootModel(client: client) + let loaded = AsyncStream.makeStream() + let started = AsyncStream.makeStream() + let cancelled = AsyncStream.makeStream() + let presentation = Task { + await model.runThreadPresentation(id: "a") { loaded.continuation.yield(()) } + } + var loads = loaded.stream.makeAsyncIterator() + await loads.next() + var pendingRetry: CheckedContinuation? + client.loadThreadHandler = { _ in + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + pendingRetry = continuation + started.continuation.yield(()) + } + } onCancel: { + cancelled.continuation.yield(()) + } + } + #expect(model.refreshThreadPresentation(id: "a")) + var retries = started.stream.makeAsyncIterator() + await retries.next() + presentation.cancel() + await presentation.value + var cancellations = cancelled.stream.makeAsyncIterator() + await cancellations.next() + #expect(client.releasedThreadIDs == ["a"]) + #expect(!model.refreshThreadPresentation(id: "a")) + pendingRetry?.resume(throwing: CancellationError()) + pendingRetry = nil + #expect(client.loadedThreadIDs == ["a", "a"]) + loaded.continuation.finish() + started.continuation.finish() + cancelled.continuation.finish() + } + @Test func rewindLocksSendingAndSavesRecoveredInputAfterLeavingTheThread() async throws { let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) @@ -287,6 +444,7 @@ struct FeatureRootModelTests { await model.applicationDidBecomeActive(at: start.addingTimeInterval(10)) await model.applicationDidBecomeActive(at: start.addingTimeInterval(11)) #expect(client.foregroundReconnects == [false, true]) + #expect(client.backgroundSuspends == 2) } @Test @@ -1683,6 +1841,157 @@ struct FeatureRootModelTests { #expect(model.details[thread.id]?.messages.last?.state == .complete) } + @Test + func deliveredMessageSurvivesAnOlderDetailUntilTheServerIncludesIt() async throws { + let client = FeatureClientStub() + let thread = FeatureThread( + id: "thread-1", + projectID: "project-1", + environmentID: "environment-1", + title: "Thread" + ) + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", + name: "Studio", + endpoint: "https://studio.example", + isActive: true, + connectionState: .connected + ), + ], + threads: [thread] + ) + client.threadDetail = FeatureThreadDetail(thread: thread) + let model = testRootModel(client: client) + await model.reload() + _ = await model.detail(for: thread.id) + + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + // Its detail refresh no longer blocks the send, so an older read can land first. + _ = await model.detail(for: thread.id, force: true) + let delivered = try #require(model.details[thread.id]?.messages.last) + #expect(delivered.text == "ship it") + #expect(delivered.state == .complete) + + client.threadDetail = FeatureThreadDetail( + thread: thread, + messages: [FeatureMessage(id: delivered.id, role: .user, text: "ship it")] + ) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.filter { $0.id == delivered.id }.count == 1) + } + + private func deliveredMessageModel() async -> (FeatureClientStub, FeatureRootModel, FeatureThread) { + let client = FeatureClientStub() + let thread = FeatureThread( + id: "thread-1", projectID: "project-1", environmentID: "environment-1", title: "Thread" + ) + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", name: "Studio", endpoint: "https://studio.example", + isActive: true, connectionState: .connected + ), + ], + threads: [thread] + ) + client.threadDetail = FeatureThreadDetail(thread: thread) + let model = testRootModel(client: client) + await model.reload() + _ = await model.detail(for: thread.id) + return (client, model, thread) + } + + @Test + func confirmedDeliveredMessageDoesNotReturnAfterARewind() async throws { + let (client, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + client.threadDetail = FeatureThreadDetail( + thread: thread, messages: [FeatureMessage(id: id, role: .user, text: "ship it")] + ) + _ = await model.detail(for: thread.id, force: true) + // A rewind removes the message from the server transcript. + client.threadDetail = FeatureThreadDetail(thread: thread) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) + } + + @Test + func clearedDetailsDropRetainedDeliveredMessages() async throws { + let (_, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + await model.reloadAfterConnection() + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == false) + } + + @Test + func evictedDetailKeepsItsDeliveredMessageOnReopen() async throws { + let (client, model, thread) = await deliveredMessageModel() + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "ship it", selection: nil))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + client.loadThreadHandler = { id in + FeatureThreadDetail(thread: FeatureThread( + id: id, projectID: "project-1", environmentID: "environment-1", title: id + )) + } + // Opening and closing other threads evicts this detail from the bounded cache. + for index in 0..<6 { _ = await model.detail(for: "other-\(index)") } + model.releaseThread("other-5") + #expect(model.details[thread.id] == nil) + // The server transcript has not caught up yet. + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) + } + + @Test + func localCopiesKeepSendOrderWhenANewerSendIsDeliveredFirst() async throws { + let (client, model, thread) = await deliveredMessageModel() + client.sendMessageError = URLError(.notConnectedToInternet) + _ = await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "first", selection: nil)) + client.sendMessageError = nil + #expect(await model.sendMessage(FeatureMessageSubmission(threadID: thread.id, text: "second", selection: nil))) + // An older read that includes neither message. + _ = await model.detail(for: thread.id, force: true) + let texts = model.details[thread.id]?.messages.map(\.text) ?? [] + #expect(texts.filter { $0 == "first" || $0 == "second" } == ["first", "second"]) + } + + @Test + func startedTaskKeepsItsPromptUntilTheServerIncludesIt() async throws { + let client = FeatureClientStub() + client.snapshot = FeatureSnapshot( + connection: .init(state: .connected), + environments: [ + .init( + id: "environment-1", name: "Studio", endpoint: "https://studio.example", + isActive: true, connectionState: .connected + ), + ], + projects: [.init(id: "project-1", environmentID: "environment-1", name: "Native", path: "/native")] + ) + let model = testRootModel(client: client) + await model.reload() + // The server creates the thread under the ID the app queued it with. + client.beforeStartTask = { @MainActor in + if let pending = model.snapshot.threads.first { client.createdThread = pending } + } + let thread = try #require(await model.startTask(NewTaskRequest( + projectID: "project-1", prompt: "Ship it", selection: nil, + runtimeMode: .fullAccess, interactionMode: .standard + ))) + let id = try #require(model.details[thread.id]?.messages.last?.id) + // An older read that does not include the accepted prompt yet. + client.threadDetail = FeatureThreadDetail(thread: thread) + _ = await model.detail(for: thread.id, force: true) + #expect(model.details[thread.id]?.messages.contains { $0.id == id } == true) + } + @Test func sendPreservesTheThreadAutomaticPermission() async { let client = FeatureClientStub() @@ -3772,7 +4081,17 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { try await prepareRecovery(.init(message: message, attachments: rewindAttachments)) _ = try await rewindHandler(threadID, messageID) } + + var loadedThreadIDs: [String] = [] + var releasedThreadIDs: [String] = [] + var onReleaseThread: ((String) -> Void)? + func releaseThread(id: String) { + releasedThreadIDs.append(id) + onReleaseThread?(id) + } var foregroundReconnects: [Bool] = [] + var backgroundSuspends = 0 + func suspendForBackground() { backgroundSuspends += 1 } func resumeAfterBackground(reconnect: Bool) async { foregroundReconnects.append(reconnect) } private let eventStream: AsyncStream private let eventContinuation: AsyncStream.Continuation @@ -3950,6 +4269,7 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { } func loadThread(id: String, fresh: Bool) async throws -> FeatureThreadDetail { + loadedThreadIDs.append(id) if let loadThreadError { throw loadThreadError } @@ -3974,7 +4294,7 @@ private final class FeatureClientStub: FeatureClient, T3ConnectCapable { selection: FeatureSelection?, runtimeMode: FeatureRuntimeMode, attachments _: [FeatureUploadAttachment], - identity _: FeatureSubmissionIdentity, + identity: FeatureSubmissionIdentity, context: OrchestrationMessageContext? = nil ) async throws { sentRuntimeModes.append(runtimeMode) diff --git a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift index cc188815bb14..7c5e2f9be863 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeMultiEnvironmentTests.swift @@ -1,4 +1,6 @@ import Foundation +import Darwin +import Observation import Testing import XCTest @testable import T3Code @@ -24,7 +26,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { projectID: "project-shared", threadID: "thread-shared", title: "Remote files", workspaceRoot: #"C:\work\project"#, worktreePath: #"D:\worktrees\feature"# ), host: "two.example") - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let root = try await fixture.client.listFiles(threadID: thread.id, path: nil) @@ -56,7 +58,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { rpcConnectionWaitTimeout: .seconds(2) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "one" }) let root = try await fixture.client.listFiles(threadID: thread.id, path: nil) @@ -87,17 +89,19 @@ final class NativeMultiEnvironmentTests: XCTestCase { ])]] ) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) XCTAssertEqual(project.defaultSelection?.modelID, environmentDefault.model) _ = try await fixture.client.createThread(projectID: project.id, title: "Task", selection: nil) - let creates = await fixture.transport.dispatchRecords().filter { $0.command["type"] == .string("thread.create") } - XCTAssertEqual(creates.last?.command["modelSelection"], try JSONValue.encode(environmentDefault)) + // The configured peer keeps its socket, so the command crosses it rather than HTTP. + let creates = await server.dispatchedCommands(host: "two.example").filter { $0["type"] == .string("thread.create") } + XCTAssertEqual(creates.last?["modelSelection"], try JSONValue.encode(environmentDefault)) await fixture.client.disconnect() } @@ -117,12 +121,13 @@ final class NativeMultiEnvironmentTests: XCTestCase { ]] ) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) XCTAssertEqual(project.defaultSelection?.modelID, environmentDefault.model) XCTAssertEqual(project.supportsProjectSettingsOverrides, true) @@ -139,7 +144,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .responseStreamingMode, value: .string("turn") )) - let updated = try await fixture.client.initialSnapshot() + let updated = try await fixture.hydratedSnapshot() let updatedProject = try XCTUnwrap(updated.projects.first { $0.id == project.id }) XCTAssertEqual(updatedProject.defaultSelection?.modelID, projectDefault.model) XCTAssertEqual(updatedProject.defaultWorkspaceMode, .worktree) @@ -162,12 +167,12 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .defaultModelSelection, value: .null )) - let noDefault = try await fixture.client.initialSnapshot() + let noDefault = try await fixture.hydratedSnapshot() XCTAssertNil(noDefault.projects.first { $0.id == project.id }?.defaultSelection) try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( key: .defaultModelSelection, value: nil )) - let inherited = try await fixture.client.initialSnapshot() + let inherited = try await fixture.hydratedSnapshot() XCTAssertEqual(inherited.projects.first { $0.id == project.id }?.defaultSelection?.modelID, environmentDefault.model) await fixture.client.disconnect() @@ -194,7 +199,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { snapshotSequence: source.snapshotSequence, projects: [wireProject], threads: source.threads, updatedAt: source.updatedAt ), host: "two.example") - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) let preferences = try await fixture.client.projectPreferences(projectID: project.id) XCTAssertEqual(preferences.environment.defaultModelSelection?.model, "environment-model") @@ -212,7 +217,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap(snapshot.projects.first { $0.environmentID == "two" }) do { try await fixture.client.updateProjectPreferences(projectID: project.id, change: .init( @@ -325,7 +330,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(Set(snapshot.projects.map(\.environmentID)), ["one", "two"]) XCTAssertEqual(Set(snapshot.threads.compactMap(\.wireID)), ["thread-one", "thread-two"]) @@ -383,19 +388,19 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveProviderRefreshKeepsActiveThreadsAndAcceptsTheirNextSequence() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let providers = try await fixture.client.refreshProviders(environmentID: "two") XCTAssertEqual(providers.map(\.id), ["codex-two.example"]) - let refreshed = try await fixture.client.initialSnapshot() + let refreshed = try await fixture.hydratedSnapshot() XCTAssertEqual( refreshed.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -421,7 +426,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let updated = try await fixture.client.initialSnapshot() + let updated = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(updated.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID)), ["thread-one", "thread-new"] @@ -435,20 +440,20 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveEnvironmentSettingsDoNotReplaceActiveThreads() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() try await fixture.client.updateServerPreferences( environmentID: "two", change: .environmentIcon("mac-mini") ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -465,6 +470,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testMachineModelDefaultsRefreshCachedProjectsWithoutChangingOtherEnvironments() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), @@ -479,7 +485,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { snapshotSequence: source.snapshotSequence, projects: [project], threads: source.threads, updatedAt: source.updatedAt ), host: "two.example") - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() XCTAssertNil(initial.projects.first { $0.environmentID == "two" }?.defaultSelection) let localDefault = initial.projects.first { $0.environmentID == "one" }?.defaultSelection @@ -488,13 +494,13 @@ final class NativeMultiEnvironmentTests: XCTestCase { try await fixture.client.updateServerPreferences(environmentID: "two", change: .sharedPreferences(.object([ "defaultModelSelection": try JSONValue.encode(selection), ]))) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.projects.first { $0.environmentID == "two" }?.defaultSelection?.modelID, model) XCTAssertEqual(snapshot.projects.first { $0.environmentID == "one" }?.defaultSelection, localDefault) } await fixture.transport.setShell(source, host: "two.example") - let overridden = try await fixture.client.initialSnapshot() + let overridden = try await fixture.hydratedSnapshot() XCTAssertEqual(overridden.projects.first { $0.environmentID == "two" }?.defaultSelection?.modelID, "gpt-5.6-sol") await fixture.client.disconnect() } @@ -502,20 +508,20 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testSharedSettingsFanOutDoesNotReplaceActiveThreads() async throws { let server = MultiEnvironmentConfigurationServer() let fixture = try await Self.makeFixture( - passiveSequence: 5_000, + requireConfiguration: true, passiveSequence: 5_000, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() try await fixture.client.updateServerPreferences( environmentID: "one", change: .defaultThreadEnvMode(.worktree) ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.filter { $0.environmentID == "one" }.compactMap(\.wireID), ["thread-one"] @@ -532,13 +538,14 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testRestartPreferenceOnlyReachesComputersThatSupportIt() async throws { let server = MultiEnvironmentConfigurationServer(restartSupportHosts: ["one.example"]) let fixture = try await Self.makeFixture( + requireConfiguration: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(1), fallbackPollingInitialDelay: .seconds(60), aggregateRefreshInterval: .seconds(60) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.preferencesByEnvironment?["one"]?.continueThreadsAfterServerUpdate, false) XCTAssertNil(snapshot.preferencesByEnvironment?["two"]?.continueThreadsAfterServerUpdate) @@ -583,7 +590,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap( snapshot.threads.first(where: { $0.wireID == "thread-one" }) ) @@ -618,7 +625,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let detail = try await fixture.client.loadThread(id: thread.id) @@ -650,7 +657,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) let detail = try await fixture.client.loadThread(id: thread.id) @@ -696,7 +703,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "two.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap(snapshot.threads.first { $0.environmentID == "two" }) XCTAssertTrue(thread.isRegeneratingTitle) @@ -724,7 +731,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture(repositoryIdentity: identity) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let groups = DailyUXCreationContext.projectGroups(in: snapshot) XCTAssertEqual(Set(snapshot.projects.compactMap(\.repositoryIdentity?.canonicalKey)), [ @@ -739,15 +746,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await fixture.transport.setReachable(false, host: "two.example") - let passiveFailure = try await fixture.client.initialSnapshot() + let passiveFailure = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(passiveFailure.threads.compactMap(\.wireID)), ["thread-one", "thread-two"] ) - XCTAssertEqual(passiveFailure.connection.state, .connected) + XCTAssertEqual(passiveFailure.environments.first { $0.id == "one" }?.connectionState, .connected) XCTAssertEqual( passiveFailure.environments.first(where: { $0.id == "two" })?.connectionState, .disconnected @@ -756,7 +763,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { await fixture.transport.setReachable(false, host: "one.example") await fixture.transport.setReachable(true, host: "two.example") - let activeFailure = try await fixture.client.initialSnapshot() + let activeFailure = try await fixture.hydratedSnapshot() XCTAssertEqual( Set(activeFailure.threads.compactMap(\.wireID)), ["thread-one", "thread-two"] @@ -773,7 +780,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testCachedShellRowsApplySettlementAndRemoveDeletedRoutes() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() let original = try XCTUnwrap(initial.threads.first { $0.environmentID == "two" }) let updated = multiEnvironmentShell( projectID: "project-two", threadID: "thread-two", title: "Remote work", @@ -782,7 +789,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { settledOverride: "settled", settledAt: "2026-07-31T12:01:00.000Z" ) await fixture.transport.setShell(updated, host: "two.example") - let refreshed = try await fixture.client.initialSnapshot() + let refreshed = try await fixture.hydratedSnapshot() let settled = try XCTUnwrap(refreshed.threads.first { $0.id == original.id }) XCTAssertEqual(settled.updatedAt, original.updatedAt) XCTAssertTrue(settled.isSettled) @@ -796,7 +803,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "two.example" ) - let removed = try await fixture.client.initialSnapshot() + let removed = try await fixture.hydratedSnapshot() XCTAssertFalse(removed.threads.contains { $0.id == original.id }) XCTAssertEqual(removed.projects.first { $0.environmentID == "two" }?.threadCount, 0) do { @@ -811,7 +818,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testOlderHTTPSnapshotCannotReplaceNewerEnvironmentState() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let newer = multiEnvironmentShell( projectID: "project-one", @@ -827,7 +834,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let older = multiEnvironmentShell( projectID: "project-one", @@ -844,7 +851,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { host: "one.example" ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.first(where: { $0.environmentID == "one" })?.title, @@ -856,7 +863,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testThreadCreationCannotReplaceNewerEnvironmentStateWithAnOlderShell() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let newer = multiEnvironmentShell( projectID: "project-one", @@ -872,7 +879,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - let current = try await fixture.client.initialSnapshot() + let current = try await fixture.hydratedSnapshot() let project = try XCTUnwrap( current.projects.first(where: { $0.environmentID == "one" }) ) @@ -897,7 +904,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { title: "Another task", selection: nil ) - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual( snapshot.threads.first(where: { $0.wireID == "thread-one" })?.title, @@ -969,7 +976,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await loader.waitForCallCount(2) let retryCallCount = await loader.callCount @@ -990,7 +997,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { _ = try await fixture.client.initialSnapshot() await loader.waitForCallCount(1) - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() await loader.waitForFirstLoadCancellation() await loader.waitForCallCount(2) @@ -1003,7 +1010,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture(duplicateIDs: true) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() XCTAssertEqual(snapshot.projects.count, 2) XCTAssertEqual(snapshot.threads.count, 2) XCTAssertEqual(Set(snapshot.projects.map(\.id)).count, 2) @@ -1026,7 +1033,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let remoteProject = try XCTUnwrap( snapshot.projects.first(where: { $0.environmentID == "two" }) ) @@ -1059,7 +1066,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testPassiveCreateRecoversACommittedThreadAfterItsReplyIsLost() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let snapshot = try await fixture.client.initialSnapshot() + let snapshot = try await fixture.hydratedSnapshot() let project = try XCTUnwrap( snapshot.projects.first(where: { $0.environmentID == "two" }) ) @@ -1084,32 +1091,17 @@ final class NativeMultiEnvironmentTests: XCTestCase { func testUnarchiveImmediatelyRestoresLiveThreadWhenRefreshIsUnavailable() async throws { let fixture = try await Self.makeFixture() defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() + let initial = try await fixture.hydratedSnapshot() let thread = try XCTUnwrap( initial.threads.first(where: { $0.environmentID == "one" }) ) - let events = fixture.client.events() - var iterator = events.makeAsyncIterator() + let recorder = try XCTUnwrap(fixture.recorder) await fixture.transport.setShellReadsEnabled(false, host: "one.example") - try await fixture.client.setThreadArchived(id: thread.id, archived: true) - while let event = await iterator.next() { - if case let .thread(candidate) = event, - candidate.id == thread.id, - candidate.isArchived { - break - } - } + _ = try await recorder.wait { $0.threads.contains { $0.id == thread.id && $0.isArchived } } try await fixture.client.setThreadArchived(id: thread.id, archived: false) - var restored: FeatureThread? - while let event = await iterator.next() { - if case let .thread(candidate) = event, - candidate.id == thread.id, - !candidate.isArchived { - restored = candidate - break - } - } + let restoredSnapshot = try await recorder.wait { $0.threads.contains { $0.id == thread.id && !$0.isArchived } } + let restored = restoredSnapshot.threads.first { $0.id == thread.id } XCTAssertEqual(restored?.id, thread.id) XCTAssertEqual(restored?.isArchived, false) @@ -1122,7 +1114,7 @@ final class NativeMultiEnvironmentTests: XCTestCase { fallbackPollingInterval: .seconds(2) ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - _ = try await fixture.client.initialSnapshot() + _ = try await fixture.hydratedSnapshot() let current = multiEnvironmentShell( projectID: "project-one", threadID: "thread-one", @@ -1148,23 +1140,15 @@ final class NativeMultiEnvironmentTests: XCTestCase { ), host: "one.example" ) - let events = fixture.client.events() - var iterator = events.makeAsyncIterator() - var refreshed: FeatureSnapshot? - while let event = await iterator.next() { - if case let .snapshot(snapshot) = event, - snapshot.projects.contains(where: { $0.wireID == addedProject.id }) { - refreshed = snapshot - break - } - } - - XCTAssertEqual(refreshed?.connection.state, .reconnecting) + let recorder = try XCTUnwrap(fixture.recorder) + let refreshed = try await recorder.wait { $0.projects.contains { $0.wireID == addedProject.id } } + XCTAssertEqual(refreshed.connection.state, .reconnecting) await fixture.client.disconnect() } fileprivate static func makeFixture( duplicateIDs: Bool = false, + requireConfiguration: Bool = false, passiveSequence: Int = 1, includeThirdEnvironment: Bool = false, repositoryIdentity: RepositoryIdentity? = nil, @@ -1173,12 +1157,25 @@ final class NativeMultiEnvironmentTests: XCTestCase { rpcConnectionWaitTimeout: Duration = .milliseconds(5), fallbackPollingInitialDelay: Duration = .seconds(3), fallbackPollingInterval: Duration = .seconds(2), + shellReconciliationSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, aggregateRefreshInterval: Duration = NativeFeatureClient.defaultAggregateRefreshInterval, aggregateIdleRefreshInterval: Duration = NativeFeatureClient.defaultAggregateIdleRefreshInterval, aggregateFailureRefreshInterval: Duration = NativeFeatureClient.defaultAggregateFailureRefreshInterval, aggregateRefreshSleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }, + aggregatePeerRefreshSleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: @escaping @Sendable (String, Duration) async throws -> Void = { _, interval in + try await Task.sleep(for: interval) + }, + aggregatePublishSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(250)) + }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, aggregateEnvironmentLoader: @escaping @Sendable (EnvironmentRuntime) async throws -> [Environment] = { try await $0.environments() } @@ -1273,20 +1270,30 @@ final class NativeMultiEnvironmentTests: XCTestCase { let settings = UserDefaults( suiteName: "t3-native-multi-\(UUID().uuidString)" )! + let receipts = PassiveLiveReceipts() return MultiEnvironmentFixture( directory: directory, transport: transport, + runtime: runtime, client: NativeFeatureClient( runtime: runtime, settingsStore: settings, fallbackPollingInitialDelay: fallbackPollingInitialDelay, fallbackPollingInterval: fallbackPollingInterval, + shellReconciliationSleep: shellReconciliationSleep, aggregateRefreshInterval: aggregateRefreshInterval, aggregateIdleRefreshInterval: aggregateIdleRefreshInterval, aggregateFailureRefreshInterval: aggregateFailureRefreshInterval, aggregateRefreshSleep: aggregateRefreshSleep, + aggregatePeerRefreshSleep: aggregatePeerRefreshSleep, + aggregateStreamRetrySleep: aggregateStreamRetrySleep, + aggregatePublishSleep: aggregatePublishSleep, + aggregateRefreshReceipt: { receipt in + receipts.record(receipt) + aggregateRefreshReceipt(receipt) + }, aggregateEnvironmentLoader: aggregateEnvironmentLoader - ) + ), receipts: receipts, requireConfiguration: requireConfiguration ) } } @@ -1369,11 +1376,13 @@ struct NativeGitHubRoutingTests { private func configuredFixture(_ server: MultiEnvironmentConfigurationServer, includeThird: Bool = false) async throws -> MultiEnvironmentFixture { let fixture = try await NativeMultiEnvironmentTests.makeFixture( + requireConfiguration: true, includeThirdEnvironment: includeThird, pullRequestsAvailable: true, webSocketConnector: MultiEnvironmentConfigurationConnector(server: server), rpcConnectionWaitTimeout: .seconds(2) ) - _ = try await fixture.client.initialSnapshot() + // Peers hydrate independently; routing reads each peer's configuration. + _ = try await fixture.hydratedSnapshot() for id in includeThird ? ["one", "two", "three"] : ["one", "two"] { try await fixture.client.setGitHubRoutingPermission(environmentID: id, permission: .readWrite) } @@ -1391,19 +1400,17 @@ struct NativePassiveThreadRefreshTests { func passiveThreadEventsArriveWithinFiveSecondsAndStayFastAfterChanges() async throws { let refreshSleep = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { _, interval in + try await refreshSleep.sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } - let initial = try await fixture.client.initialSnapshot() - let thread = try #require( - initial.threads.first(where: { $0.environmentID == "two" }) - ) + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") let updatedTitle = "Passive work updated automatically" let eventProbe = ThreadTitleEventProbe( events: fixture.client.events(), - threadID: thread.id, + threadID: threadID, title: updatedTitle ) eventProbe.start() @@ -1433,8 +1440,8 @@ struct NativePassiveThreadRefreshTests { func passiveRefreshUsesTenSecondsWhenWorkIsUnchanged() async throws { let refreshSleep = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { _, interval in + try await refreshSleep.sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } @@ -1448,52 +1455,199 @@ struct NativePassiveThreadRefreshTests { await fixture.client.disconnect() } - @Test("A failed passive environment backs off without slowing an active peer") + @Test("A failed passive environment backs off without slowing an active peer", .timeLimit(.minutes(1))) func failedPassiveEnvironmentBacksOffWithoutSlowingActivePeer() async throws { - let refreshSleep = ControllableAggregateRefreshSleep() + let healthyClock = ControllableAggregateRefreshSleep() + let failedClock = ControllableAggregateRefreshSleep() let fixture = try await NativeMultiEnvironmentTests.makeFixture( includeThirdEnvironment: true, - aggregateRefreshSleep: { - try await refreshSleep.sleep(for: $0) + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? healthyClock : failedClock).sleep(for: interval) } ) defer { try? FileManager.default.removeItem(at: fixture.directory) } _ = try await fixture.client.initialSnapshot() - await fixture.transport.setShell( - multiEnvironmentShell( - projectID: "project-two", - threadID: "thread-two", - title: "Remote work", - providerID: "claudeAgent", - modelID: "claude-opus-4-1", - backgroundLiveness: .working - ), - host: "two.example" - ) + _ = await healthyClock.waitUntilRequested(count: 1) + _ = await failedClock.waitUntilRequested(count: 1) await fixture.transport.setReachable(false, host: "three.example") + await failedClock.resume() + let failureCadence = await failedClock.waitUntilRequested(count: 2) + #expect(failureCadence == .seconds(20)) + for count in 2...4 { + await healthyClock.resume() + _ = await healthyClock.waitUntilRequested(count: count) + let failedReads = await fixture.transport.shellReadCount(host: "three.example") + #expect(failedReads == 2) + } + await failedClock.resume() + _ = await failedClock.waitUntilRequested(count: 3) + let retriedReads = await fixture.transport.shellReadCount(host: "three.example") + #expect(retriedReads == 3) + await fixture.client.disconnect() + } - let firstCadence = await refreshSleep.waitUntilRequested(count: 1) - #expect(firstCadence == .seconds(5)) - await refreshSleep.resume() - let secondCadence = await refreshSleep.waitUntilRequested(count: 2) - #expect(secondCadence == .seconds(5)) - let initialFailedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(initialFailedReadCount == 2) + @Test("Healthy rows publish twice while a peer shell and optional catalogue remain held", .timeLimit(.minutes(1))) + func healthyRowsPublishTwiceWhilePeerAndCatalogueAreHeld() async throws { + let healthyClock = ControllableAggregateRefreshSleep() + let slowClock = ControllableAggregateRefreshSleep() + let connector = GatedPassiveCatalogueConnector() + let topologyGate = PassiveRequestGate() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + webSocketConnector: connector, + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? healthyClock : slowClock).sleep(for: interval) + }, + aggregateEnvironmentLoader: { runtime in + await topologyGate.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let healthyID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + let heldShell = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "three.example", gate: heldShell) + await connector.holdPassiveConnections() + await topologyGate.release() + await connector.gate.waitUntilEntered() + _ = await healthyClock.waitUntilRequested(count: 1) + await heldShell.waitUntilEntered() + for update in 1...2 { + let title = "Healthy update \(update)" + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: healthyID, title: title) + probe.start() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: title, + snapshotSequence: update + 1 + ), host: "two.example") + await healthyClock.resume() + await probe.waitUntilObserved() + #expect(probe.didObserveTitle()) + _ = await healthyClock.waitUntilRequested(count: update + 1) + #expect(await heldShell.isHeld) + #expect(await connector.gate.isHeld) + } + await fixture.client.disconnect() + await heldShell.release() + await connector.release() + } + @Test("A cancelled refresh generation cannot overwrite a restarted peer", .timeLimit(.minutes(1))) + func cancelledGenerationCannotOverwriteRestartedPeer() async throws { + let clock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + aggregatePeerRefreshSleep: { _, interval in try await clock.sleep(for: interval) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + _ = await clock.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let gate = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Stale generation", + snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: gate) + await clock.resume() + await gate.waitUntilEntered() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Current generation", + snapshotSequence: 2 + ), host: "two.example") + _ = try await fixture.client.initialSnapshot() + #expect(oldWorker.isCancelled) + await gate.release() + await oldWorker.value + // A stale 999 would survive this read of sequence 2 if it reached the cache. + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Current generation") + await fixture.client.disconnect() + } - for requestCount in 2...4 { - await refreshSleep.resume() - let cadence = await refreshSleep.waitUntilRequested(count: requestCount + 1) - #expect(cadence == .seconds(5)) - let failedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(failedReadCount == 2) + @Test("Failed environment writes preserve the passive worker", arguments: [false, true]) + func failedEnvironmentWritePreservesWorker(removal: Bool) async throws { + let clock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + aggregatePeerRefreshSleep: { _, interval in try await clock.sleep(for: interval) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + _ = await clock.waitUntilRequested(count: 1) + let worker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let catalog = fixture.directory.appendingPathComponent("environments.json") + let data = try Data(contentsOf: catalog) + try FileManager.default.removeItem(at: catalog) + try FileManager.default.createDirectory(at: catalog, withIntermediateDirectories: false) + do { + if removal { try await fixture.client.removeEnvironment(id: "two") } + else { try await fixture.client.setEnvironmentEnabled(id: "two", enabled: false) } + Issue.record("The blocked catalog write unexpectedly succeeded") + } catch { + #expect(!worker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] != nil) } + try FileManager.default.removeItem(at: catalog) + try data.write(to: catalog) + #expect(try await fixture.runtime.environments().contains { $0.id == "two" && $0.isEnabled }) + let probe = ThreadTitleEventProbe( + events: fixture.client.events(), + threadID: FeatureScopedID.thread(environmentID: "two", wireID: "thread-two"), + title: "Still updating" + ) + probe.start() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Still updating", snapshotSequence: 2 + ), host: "two.example") + await clock.resume() + await probe.waitUntilObserved() + #expect(probe.didObserveTitle()) + await fixture.client.disconnect() + } - await refreshSleep.resume() - _ = await refreshSleep.waitUntilRequested(count: 6) - let retriedReadCount = await fixture.transport.shellReadCount(host: "three.example") - #expect(retriedReadCount == 3) + @Test("Removal cancels a held peer and its late response never returns a row", .timeLimit(.minutes(1))) + func removedPeerCannotPublishLateResponse() async throws { + let removedClock = ControllableAggregateRefreshSleep() + let healthyClock = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + aggregatePeerRefreshSleep: { id, interval in + try await (id == "two" ? removedClock : healthyClock).sleep(for: interval) + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let healthyID = FeatureScopedID.thread(environmentID: "three", wireID: "thread-three") + let probe = ThreadTitleEventProbe( + events: fixture.client.events(), threadID: healthyID, title: "Removal verified" + ) + probe.start() + _ = await removedClock.waitUntilRequested(count: 1) + _ = await healthyClock.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let gate = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Ghost removed row", + snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: gate) + await removedClock.resume() + await gate.waitUntilEntered() + try await fixture.client.removeEnvironment(id: "two") + #expect(oldWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + await gate.release() + await oldWorker.value + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-three", threadID: "thread-three", title: "Removal verified", + snapshotSequence: 2 + ), host: "three.example") + await healthyClock.resume() + await probe.waitUntilObserved() + #expect(!probe.sawThreadTitle("Ghost removed row")) + #expect(!probe.lastEnvironmentIDs.contains("two")) await fixture.client.disconnect() } + } private actor FailOnceAggregateEnvironmentLoader { @@ -1678,6 +1832,8 @@ private final class ThreadTitleEventProbe { private let threadID: String private let title: String private var observed = false + private var observedTitles = Set() + private(set) var lastEnvironmentIDs = Set() private var observedWaiters: [CheckedContinuation] = [] private var task: Task? @@ -1693,8 +1849,11 @@ private final class ThreadTitleEventProbe { for await event in events { switch event { case let .thread(thread): + observedTitles.insert(thread.title) observed = thread.id == threadID && thread.title == title case let .snapshot(snapshot): + observedTitles.formUnion(snapshot.threads.map(\.title)) + lastEnvironmentIDs = Set(snapshot.environments.map(\.id)) observed = snapshot.threads.contains { $0.id == self.threadID && $0.title == self.title } @@ -1710,6 +1869,8 @@ private final class ThreadTitleEventProbe { } } + func sawThreadTitle(_ title: String) -> Bool { observedTitles.contains(title) } + func didObserveTitle() -> Bool { observed } @@ -1727,6 +1888,7 @@ private final class ThreadTitleEventProbe { } private actor ControllableAggregateRefreshSleep { + var requestCount: Int { requestedCadences.count } private var requestedCadences: [Duration] = [] private var requestWaiters: [( count: Int, @@ -1768,10 +1930,47 @@ private actor ControllableAggregateRefreshSleep { } } -private struct MultiEnvironmentFixture { +@MainActor +private final class MultiEnvironmentFixture { let directory: URL let transport: MultiEnvironmentHTTPTransport + let runtime: EnvironmentRuntime let client: NativeFeatureClient + let receipts: PassiveLiveReceipts + let requireConfiguration: Bool + private(set) var recorder: BootstrapSnapshotRecorder? + + init(directory: URL, transport: MultiEnvironmentHTTPTransport, runtime: EnvironmentRuntime, + client: NativeFeatureClient, receipts: PassiveLiveReceipts, requireConfiguration: Bool) { + self.directory = directory + self.transport = transport + self.runtime = runtime + self.client = client + self.receipts = receipts + self.requireConfiguration = requireConfiguration + } + + func hydratedSnapshot() async throws -> FeatureSnapshot { + let ids = try await runtime.environments().filter(\.isEnabled).map(\.id) + let targets = Dictionary(uniqueKeysWithValues: ids.map { ($0, receipts.httpCount($0) + 1) }) + let seed = try await client.initialSnapshot() + if let recorder { recorder.record(seed) } + else { recorder = BootstrapSnapshotRecorder(seed: seed, events: client.events()) } + for id in ids { + try await receipts.waitForHTTP(id, count: targets[id]!) + if requireConfiguration { try await receipts.waitForConfiguration(id) } + } + let needsPublishedConfiguration = requireConfiguration + return try await recorder!.wait { snapshot in + ids.allSatisfy { id in + // The configuration receipt precedes publication. Wait for + // the same event consumer to observe settings in its snapshot. + if needsPublishedConfiguration && snapshot.preferencesByEnvironment?[id] == nil { return false } + guard let state = snapshot.environments.first(where: { $0.id == id })?.connectionState else { return false } + return state == .connected || state == .disconnected + } + } + } } private actor MultiEnvironmentHTTPTransport: HTTPTransport { @@ -1780,12 +1979,15 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { private var detailData: [String: [String: Data]] = [:] private var reachableHosts: Set private var shellReadsEnabledHosts: Set + private var rejectedShellHosts: Set = [] private var shellReadCounts: [String: Int] = [:] private var dispatched: [MultiEnvironmentDispatchRecord] = [] private var hostsDroppingNextCreateReply = Set() private var diffRequests: [(host: String, input: JSONValue)] = [] func pullRequestDiffRequests() -> [(host: String, input: JSONValue)] { diffRequests } + private var nextShellGates: [String: PassiveRequestGate] = [:] + private var nextDispatchGates: [String: PassiveRequestGate] = [:] init(shells: [String: OrchestrationShellSnapshot]) { self.shells = shells @@ -1802,6 +2004,11 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { } } + /// Answers this host's shell reads with 401, as after a revoked credential. + func rejectShellReads(host: String) { + rejectedShellHosts.insert(host) + } + func setShellReadsEnabled(_ enabled: Bool, host: String) { if enabled { shellReadsEnabledHosts.insert(host) @@ -1837,11 +2044,39 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { hostsDroppingNextCreateReply.insert(host) } - func data(for request: URLRequest) throws -> (Data, HTTPURLResponse) { + func holdNextDispatch(host: String, gate: PassiveRequestGate) { nextDispatchGates[host] = gate } + + func holdNextShell(host: String, gate: PassiveRequestGate) { + nextShellGates[host] = gate + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { let host = request.url?.host ?? "" let path = request.url?.path ?? "" + if path == "/.well-known/t3/environment" { + let id = host == "one.example" ? "one" : "two" + return (try JSONEncoder.t3.encode(multiEnvironmentDescriptor( + environmentID: id, label: id == "one" ? "Left Book" : "Steam Box", pullRequestsAvailable: true + )), multiEnvironmentResponse(request)) + } + if path == "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/oauth/token" { + return (Data(#"{"access_token":"paired-token","issued_token_type":"urn:ietf:params:oauth:token-type:access_token","token_type":"Bearer","expires_in":3600,"scope":"read write"}"#.utf8), multiEnvironmentResponse(request)) + } if path == "/api/orchestration/shell" { shellReadCounts[host, default: 0] += 1 + if rejectedShellHosts.contains(host) { + return ( + Data(#"{"error":"Unauthorized"}"#.utf8), + HTTPURLResponse( + url: request.url!, statusCode: 401, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + )! + ) + } + if let gate = nextShellGates.removeValue(forKey: host), let data = shellData[host] { + await gate.enter() + return (data, multiEnvironmentResponse(request)) + } } guard reachableHosts.contains(host) else { throw URLError(.cannotConnectToHost) @@ -1876,6 +2111,7 @@ private actor MultiEnvironmentHTTPTransport: HTTPTransport { dispatched.append( MultiEnvironmentDispatchRecord(host: host, command: command) ) + if let gate = nextDispatchGates.removeValue(forKey: host) { await gate.enter() } if command["type"]?.stringValue == "thread.create", hostsDroppingNextCreateReply.remove(host) != nil, let projectID = command["projectId"]?.stringValue, @@ -1935,6 +2171,7 @@ private actor MultiEnvironmentConfigurationServer { private let routingIdentityHosts: Set private let failPullRequestWrites: Bool private var prRequests: [(host: String, method: String, input: JSONValue)] = [] + private var dispatches: [(host: String, command: JSONValue)] = [] init( restartSupportHosts: Set = [], @@ -1962,6 +2199,7 @@ private actor MultiEnvironmentConfigurationServer { func settings(host: String) -> [String: JSONValue] { settingsByHost[host] ?? [:] } func fileRequests() -> [(host: String, input: JSONValue)] { directoryRequests } func pullRequestRequests() -> [(host: String, method: String, input: JSONValue)] { prRequests } + func dispatchedCommands(host: String) -> [JSONValue] { dispatches.filter { $0.host == host }.map(\.command) } func response(to request: JSONValue, host: String) throws -> JSONValue? { guard let tag = request["tag"]?.stringValue, @@ -2031,6 +2269,9 @@ private actor MultiEnvironmentConfigurationServer { } settingsByHost[host] = updated value = .object(settingsByHost[host] ?? [:]) + case RPCMethod.dispatchCommand.rawValue: + dispatches.append((host, request["payload"] ?? .object([:]))) + value = try JSONValue.encode(DispatchResult(sequence: 2)) case RPCMethod.getArchivedShellSnapshot.rawValue: value = try JSONValue.encode(OrchestrationShellSnapshot( snapshotSequence: 0, projects: [], threads: [], updatedAt: "2026-09-04T12:00:00.000Z" @@ -2333,3 +2574,1531 @@ private func multiEnvironmentResponse(_ request: URLRequest) -> HTTPURLResponse headerFields: ["Content-Type": "application/json"] )! } + +/// Deliberately ignores cancellation to model a transport completing an old read. +private actor PassiveRequestGate { + private var cancellableEntryWaiters: [UUID: CheckedContinuation] = [:] + private var entered = false + private var released = false + private var entryWaiters: [CheckedContinuation] = [] + private var releaseWaiters: [CheckedContinuation] = [] + var isHeld: Bool { entered && !released } + + func enter() async { + entered = true + let pending = cancellableEntryWaiters.values + cancellableEntryWaiters.removeAll() + pending.forEach { $0.resume() } + entryWaiters.forEach { $0.resume() } + entryWaiters.removeAll() + guard !released else { return } + await withCheckedContinuation { releaseWaiters.append($0) } + } + + func waitUntilEnteredCancellable() async throws { + try Task.checkCancellation() + guard !entered else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { cancellableEntryWaiters[id] = $0 } + } onCancel: { + Task { await self.cancelEntryWaiter(id) } + } + } + + private func cancelEntryWaiter(_ id: UUID) { + cancellableEntryWaiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } + + func waitUntilEntered() async { + guard !entered else { return } + await withCheckedContinuation { entryWaiters.append($0) } + } + + func release() { + released = true + releaseWaiters.forEach { $0.resume() } + releaseWaiters.removeAll() + } +} + +private actor GatedPassiveCatalogueConnector: WebSocketConnecting { + private var shouldHold = false + let gate = PassiveRequestGate() + + func holdPassiveConnections() { shouldHold = true } + + func connect(to url: URL) async throws -> any WebSocketConnection { + if shouldHold && url.host == "two.example" { + return GatedPassiveCatalogueConnection(gate: gate) + } + throw URLError(.cannotConnectToHost) + } + + func release() async { await gate.release() } +} + +private actor GatedPassiveCatalogueConnection: WebSocketConnection { + let gate: PassiveRequestGate + private var receiver: CheckedContinuation? + private var closed = false + + init(gate: PassiveRequestGate) { self.gate = gate } + + func send(_ data: Data) async throws { + let request = try JSONDecoder.t3.decode(JSONValue.self, from: data) + if request["tag"]?.stringValue != nil { await gate.enter() } + } + + func receive() async throws -> Data { + if closed { throw CancellationError() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + closed = true + receiver?.resume(throwing: CancellationError()) + receiver = nil + } +} + +@Suite("Native passive live shells") +@MainActor +struct NativePassiveLiveShellTests { + @Test("A live peer whose credential is rejected stops quiet reconciliation", .timeLimit(.minutes(1))) + func livePeerRejectedCredentialStopsReconciling() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clock = SilentShellClock() + let (peerSleeps, peerSleepRecorder) = AsyncStream.makeStream() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .zero, + fallbackPollingInterval: .milliseconds(1), + shellReconciliationSleep: { id, interval in + if id == "two" { try await clock.sleep(interval) } + else { try await Task.sleep(for: interval) } + }, + aggregatePeerRefreshSleep: { id, interval in + if id == "two" { peerSleepRecorder.yield(interval) } + try await Task.sleep(for: interval) + }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + try await receipts.waitForHTTP("two", count: 1) + await server.waitForSubscriptions(host: "two.example", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Live seed", snapshotSequence: 10 + ), host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + try await clock.waitForRequest() + await fixture.transport.rejectShellReads(host: "two.example") + await clock.advance() + // The socket is still open, but the credential is gone: show it and back off. + _ = try await recorder.wait { + $0.environments.first { $0.id == "two" }?.connectionState == .needsPairing + } + // The next wait is the long rejected-credential back-off, not another quiet read. + var sleeps = peerSleeps.makeAsyncIterator() + while let next = await sleeps.next(), next != .seconds(24 * 60 * 60) {} + #expect(await clock.intervals.count == 1) + // Updates still arriving on the open socket must not hide the rejection. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "After rejection", snapshotSequence: 11 + ), host: "two.example") + let afterUpdate = try await recorder.wait { $0.threads.contains { $0.title == "After rejection" } } + #expect(afterUpdate.environments.first { $0.id == "two" }?.connectionState == .needsPairing) + // A closed stream and its fresh snapshot must neither hide the rejection nor end the back-off. + let reads = await fixture.transport.shellReadCount(host: "two.example") + try await server.finish(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 2) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "After reconnect", snapshotSequence: 12 + ), host: "two.example") + let afterReconnect = try await recorder.wait { $0.threads.contains { $0.title == "After reconnect" } } + #expect(afterReconnect.environments.first { $0.id == "two" }?.connectionState == .needsPairing) + // A woken fallback read would land before this later update is published. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Still backed off", snapshotSequence: 13 + ), host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Still backed off" } } + #expect(await fixture.transport.shellReadCount(host: "two.example") == reads) + await fixture.client.disconnect() + } + + @Test("Silent connected shells reconcile through bounded HTTP", arguments: ["one", "two"], [false, true]) + func silentConnectedShellReconciles(environmentID: String, disconnect: Bool) async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clock = SilentShellClock() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .zero, + fallbackPollingInterval: .milliseconds(1), + shellReconciliationSleep: { id, interval in + if id == environmentID { try await clock.sleep(interval) } + else { try await Task.sleep(for: interval) } + }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + let host = environmentID + ".example" + let threadID = "thread-" + environmentID + try await receipts.waitForHTTP(environmentID, count: 1) + await server.waitForSubscriptions(host: host, count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "Live seed", snapshotSequence: 10 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Live seed" } } + let reads = await fixture.transport.shellReadCount(host: host) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "HTTP repaired silent shell", snapshotSequence: 11 + ), host: host) + let held = PassiveRequestGate() + let operation = Task { @MainActor in + try await clock.waitForRequest() + #expect(await clock.intervals == [.seconds(30)]) + #expect(await fixture.transport.shellReadCount(host: host) == reads) + await clock.advance() + _ = try await recorder.wait { $0.threads.contains { $0.title == "HTTP repaired silent shell" } } + #expect(await fixture.transport.shellReadCount(host: host) == reads + 1) + try await clock.waitForRequest(2) + await fixture.transport.holdNextShell(host: host, gate: held) + await clock.advance() + try await held.waitUntilEnteredCancellable() + #expect(await clock.intervals.count == 2) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 2) + let completions = receipts.httpCount(environmentID) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-" + environmentID, threadID: threadID, + title: "Newer live shell", snapshotSequence: 20 + ), host: host) + _ = try await recorder.wait { $0.threads.contains { $0.title == "Newer live shell" } } + await held.release() + try await receipts.waitForHTTP(environmentID, count: completions + 1) + try await clock.waitForRequest(3) + #expect(recorder.history.last?.threads.contains { $0.title == "Newer live shell" } == true) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 2) + // A failed quiet check neither disconnects the live stream nor + // retries until another full reconciliation interval has elapsed. + await fixture.transport.setShellReadsEnabled(false, host: host) + await clock.advance() + try await clock.waitForRequest(4) + #expect(await fixture.transport.shellReadCount(host: host) == reads + 3) + #expect(recorder.history.last?.environments.first { $0.id == environmentID }?.connectionState == .connected) + #expect(await clock.intervals == Array(repeating: .seconds(30), count: 4)) + if disconnect { await fixture.client.disconnect() } + else { fixture.client.suspendForBackground() } + try await clock.waitForCancellation() + #expect(await fixture.transport.shellReadCount(host: host) == reads + 3) + } + // Failure watchdog only; successful verification advances an explicit clock. + let watchdog = Task { + try? await Task.sleep(for: .seconds(3)) + if !Task.isCancelled { operation.cancel() } + } + do { try await operation.value } + catch { Issue.record("Silent connected shell never scheduled or applied its HTTP repair: \(error)") } + watchdog.cancel() + await held.release() + recorder.stop() + await fixture.client.disconnect() + } + + @Test("Live peer bursts publish together without polling", .timeLimit(.minutes(1))) + func livePeerBurstsPublishTogetherWithoutPolling() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let topologyClock = ControllableAggregateRefreshSleep() + let clocks = PassiveClockBank() + let publication = ControllableAggregateRefreshSleep() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregateRefreshSleep: { try await topologyClock.sleep(for: $0) }, + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: { try await publication.sleep(for: .zero) }, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threeID = FeatureScopedID.thread(environmentID: "three", wireID: "thread-three") + let heldTwo = PassiveRequestGate() + let heldThree = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "two.example", gate: heldTwo) + await fixture.transport.holdNextShell(host: "three.example", gate: heldThree) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await server.waitForSubscriptions(host: "three.example", count: 1) + await heldTwo.waitUntilEntered() + await heldThree.waitUntilEntered() + let shellTwo = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Live base", snapshotSequence: 10) + let shellThree = multiEnvironmentShell(projectID: "project-three", threadID: "thread-three", title: "Third base", snapshotSequence: 20) + try await server.snapshot(shellTwo, host: "two.example") + try await server.snapshot(shellThree, host: "three.example") + try await receipts.waitForShells("two", count: 1) + try await receipts.waitForShells("three", count: 1) + let finalTwo = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Burst final", snapshotSequence: 12) + let finalThree = multiEnvironmentShell(projectID: "project-three", threadID: "thread-three", title: "Third final", snapshotSequence: 21) + try await server.upsert(finalTwo.threads[0], sequence: 11, host: "two.example") + try await server.upsert(finalTwo.threads[0], sequence: 12, host: "two.example") + try await server.upsert(finalThree.threads[0], sequence: 21, host: "three.example") + try await receipts.waitForShells("two", count: 3) + try await receipts.waitForShells("three", count: 2) + _ = await publication.waitUntilRequested(count: 1) + #expect(await publication.requestCount == 1) + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: threeID, title: "Third final") + probe.start() + await publication.resume() + await probe.waitUntilObserved() + #expect(probe.sawThreadTitle("Burst final")) + #expect(await heldTwo.isHeld) + #expect(await heldThree.isHeld) + await heldTwo.release() + await heldThree.release() + try await receipts.waitForHTTP("two", count: 1) + try await receipts.waitForHTTP("three", count: 1) + let reads = await fixture.transport.shellReadCount(host: "two.example") + #expect(reads == 1) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + await fixture.client.disconnect() + } + + @Test("A replacement socket accepts a low sequence and rejects old HTTP", .timeLimit(.minutes(1))) + func replacementSocketRejectsOldHTTP() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let clocks = PassiveClockBank() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let oldHTTP = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Old HTTP", snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: oldHTTP) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await oldHTTP.waitUntilEntered() + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Old socket", snapshotSequence: 100 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + await server.closeLatest(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 2) + let fresh = multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Replacement socket", snapshotSequence: 2 + ) + await fixture.transport.setShell(fresh, host: "two.example") + try await server.snapshot(fresh, host: "two.example") + try await receipts.waitForShells("two", count: 2) + await oldHTTP.release() + try await receipts.waitForHTTP("two", count: 1) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Replacement socket") + #expect(receipts.sequences["two"] == [100, 2]) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + await fixture.client.disconnect() + } + + @Test("Unknown live events trigger one immediate HTTP repair and recover", .timeLimit(.minutes(1))) + func unknownEventRepairsWithoutPollingDelay() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + await server.waitForSubscriptions(host: "two.example", count: 1) + // Drain the initial repair before making the stream authoritative. + try await receipts.waitForHTTP("two", count: 1) + let live = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Live", snapshotSequence: 10) + try await server.snapshot(live, host: "two.example") + try await receipts.waitForShells("two", count: 1) + let initialReadCount = await fixture.transport.shellReadCount(host: "two.example") + let initialReceiptCount = receipts.httpCount("two") + let held = PassiveRequestGate() + let repaired = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "HTTP repaired", snapshotSequence: 11) + await fixture.transport.setShell(repaired, host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: held) + try await server.emit([.object(["kind": .string("future-shell-event")])], host: "two.example") + await held.waitUntilEntered() + await server.waitForSubscriptions(host: "two.example", count: 2) + let readCount = await fixture.transport.shellReadCount(host: "two.example") + #expect(readCount == initialReadCount + 1) + let probe = ThreadTitleEventProbe(events: fixture.client.events(), threadID: threadID, title: "HTTP repaired") + probe.start() + await held.release() + try await receipts.waitForHTTP("two", count: initialReceiptCount + 1) + await probe.waitUntilObserved() + try await server.snapshot(repaired, host: "two.example") + try await receipts.waitForShells("two", count: 2) + #expect(await fixture.transport.shellReadCount(host: "two.example") == initialReadCount + 1) + await fixture.client.disconnect() + } + + @Test("Background cancels passive work and foreground replaces subscriptions", .timeLimit(.minutes(1))) + func backgroundAndRemovalCancelOwnedSubscriptions() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let publication = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: { try await publication.sleep(for: .zero) }, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Cached live peer", snapshotSequence: 3 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + _ = await publication.waitUntilRequested(count: 1) + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + fixture.client.suspendForBackground() + await oldWorker.value + await server.waitForInterrupts(host: "two.example", count: 1) + #expect(fixture.client.aggregateRefreshWorkers.isEmpty) + #expect(oldWorker.isCancelled) + await fixture.client.resumeAfterBackground(reconnect: false) + await server.waitForSubscriptions(host: "two.example", count: 2) + let payload = await server.lastPayload(host: "two.example") + #expect(payload?["afterSequence"] == nil) + let nextWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + try await fixture.client.removeEnvironment(id: "two") + await nextWorker.value + // Successful removal closes the transport before cancelling the worker. + // There is no live socket on which to deliver another Interrupt frame. + #expect(await server.latestConnectionIsClosed(host: "two.example")) + #expect(nextWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(!snapshot.environments.contains { $0.id == "two" }) + await fixture.client.disconnect() + } + @Test("Repeated snapshot then stream end waits twenty seconds between subscriptions", .timeLimit(.minutes(1))) + func repeatedStreamEndsBackOffWhileHTTPRepairs() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, interval in try await retries.sleep(for: interval) }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + for attempt in 1...3 { + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Snapshot \(attempt)", snapshotSequence: 10 + attempt + ), host: "two.example") + try await receipts.waitForShells("two", count: attempt) + try await server.finish(host: "two.example") + let cadence = await retries.waitUntilRequested(count: attempt) + #expect(cadence == .seconds(20)) + try await receipts.waitForHTTP("two", count: attempt + 1) + #expect(await server.subscriptionCount(host: "two.example") == attempt) + if attempt < 3 { + await retries.resume() + await server.waitForSubscriptions(host: "two.example", count: attempt + 1) + } + } + await fixture.client.disconnect() + } + + @Test("Selecting a passive environment transfers shell ownership", .timeLimit(.minutes(1))) + func activeSwitchTransfersShellOwnership() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Selected peer", snapshotSequence: 10 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + // A same-socket HTTP refresh must retain the epoch association even + // when its complete snapshot is identical to the last socket snapshot. + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Selected peer", snapshotSequence: 10 + ), host: "two.example") + let readsBeforeRename = await fixture.transport.shellReadCount(host: "two.example") + try await fixture.client.renameThread( + id: FeatureScopedID.thread(environmentID: "two", wireID: "thread-two"), title: "Selected peer" + ) + #expect(await fixture.transport.shellReadCount(host: "two.example") == readsBeforeRename + 1) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Remote work", snapshotSequence: 1 + ), host: "two.example") + let oldWorker = try #require(fixture.client.aggregateRefreshWorkers["two"]?.task) + let adoptedHTTPCount = receipts.httpCount("two") + 1 + _ = try await fixture.runtime.activate(id: "two") + let selected = try await fixture.client.initialSnapshot() + await oldWorker.value + await server.waitForSubscriptions(host: "two.example", count: 2) + try await receipts.waitForHTTP("two", count: adoptedHTTPCount) + #expect(oldWorker.isCancelled) + #expect(fixture.client.aggregateRefreshWorkers["two"] == nil) + #expect(selected.environments.first { $0.isActive }?.id == "two") + #expect(selected.threads.first { $0.environmentID == "two" }?.title == "Selected peer") + await server.waitForSubscriptions(host: "one.example", count: 2) + try await server.snapshot(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "New passive peer", snapshotSequence: 2 + ), host: "one.example") + try await receipts.waitForShells("one", count: 2) + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Selected peer") + #expect(snapshot.threads.first { $0.environmentID == "one" }?.title == "New passive peer") + + // Retaining the adopted socket's sequence must not pin a genuinely + // replacement socket to that old high watermark after a server restart. + let restarted = multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Restarted active peer", snapshotSequence: 2 + ) + await fixture.transport.setShell(restarted, host: "two.example") + await server.closeLatest(host: "two.example") + await server.waitForSubscriptions(host: "two.example", count: 3) + try await server.snapshot(restarted, host: "two.example") + let recovered = try await recorder.wait { + $0.connection.state == .connected + && $0.threads.first { $0.environmentID == "two" }?.title == "Restarted active peer" + } + #expect(recovered.threads.first { $0.environmentID == "two" }?.title == "Restarted active peer") + await fixture.client.disconnect() + } + + @Test("HTTP begun before an unknown same-socket event cannot repair the gap", .timeLimit(.minutes(1))) + func sameSocketGapRejectsPreGapHTTP() async throws { + let server = PassiveLiveServer() + let topology = PassiveRequestGate() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, interval in try await retries.sleep(for: interval) }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) }, + aggregateEnvironmentLoader: { runtime in + await topology.enter() + return try await runtime.environments() + } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let oldRead = PassiveRequestGate() + let freshRead = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Pre-gap stale HTTP", snapshotSequence: 999 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: oldRead) + await topology.release() + await server.waitForSubscriptions(host: "two.example", count: 1) + await oldRead.waitUntilEntered() + try await server.emit([.object(["kind": .string("unknown-before-baseline")])], host: "two.example") + _ = await retries.waitUntilRequested(count: 1) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Post-gap authoritative HTTP", snapshotSequence: 2 + ), host: "two.example") + await fixture.transport.holdNextShell(host: "two.example", gate: freshRead) + await oldRead.release() + try await receipts.waitForHTTP("two", count: 1) + await freshRead.waitUntilEntered() + #expect(await fixture.transport.shellReadCount(host: "two.example") == 2) + // If the old 999 was installed, this independent sequence-2 read + // cannot undo it and exposes the poisoned cache deterministically. + let snapshot = try await fixture.client.backgroundSnapshot() + #expect(snapshot.threads.first { $0.environmentID == "two" }?.title == "Post-gap authoritative HTTP") + await freshRead.release() + try await receipts.waitForHTTP("two", count: 2) + await fixture.client.disconnect() + } + + @Test("An authoritative peer snapshot removes its selected detail subscription", .timeLimit(.minutes(1))) + func fullSnapshotRemovalStopsSelectedPeerDetail() async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let clocks = PassiveClockBank() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, + fallbackPollingInitialDelay: .seconds(60), + aggregatePeerRefreshSleep: { try await clocks.sleep(id: $0, interval: $1) }, + aggregateStreamRetrySleep: { _, _ in }, + aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.client.initialSnapshot() + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + await server.waitForSubscriptions(host: "two.example", count: 1) + try await receipts.waitForHTTP("two", count: 1) + // HTTP completion can mean a rejected read after the socket epoch + // changed. Establish an applied baseline before using its detail route. + try await server.snapshot(multiEnvironmentShell( + projectID: "project-two", threadID: "thread-two", title: "Before removal", snapshotSequence: 9 + ), host: "two.example") + try await receipts.waitForShells("two", count: 1) + _ = try await fixture.client.loadThread(id: threadID) + await server.waitForSubscriptions(host: "two.example#detail", count: 1) + let baseline = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "Removed", snapshotSequence: 10) + try await server.snapshot(OrchestrationShellSnapshot( + snapshotSequence: 10, projects: baseline.projects, threads: [], updatedAt: baseline.updatedAt + ), host: "two.example") + try await receipts.waitForShells("two", count: 2) + await server.waitForInterrupts(host: "two.example#detail", count: 1) + #expect(await server.subscriptionCount(host: "one.example") == 1) + await fixture.client.disconnect() + } + +} + +@MainActor +private final class PassiveLiveReceipts { + private(set) var sequences: [String: [Int]] = [:] + private var counts: [String: Int] = [:] + private var waiters: [UUID: (String, Int, CheckedContinuation)] = [:] + + func record(_ receipt: NativePassiveShellReceipt) { + let key: String + switch receipt { + case let .shellApplied(id, sequence): + sequences[id, default: []].append(sequence) + key = "shell:" + id + case let .httpFinished(id): key = "http:" + id + case let .configurationApplied(id): key = "config:" + id + case let .archiveFinished(id): key = "archive:" + id + } + counts[key, default: 0] += 1 + let ready = waiters.filter { counts[$0.value.0, default: 0] >= $0.value.1 } + for (id, waiter) in ready { + waiters[id] = nil + waiter.2.resume() + } + } + + func httpCount(_ id: String) -> Int { counts["http:" + id, default: 0] } + func waitForShells(_ id: String, count: Int) async throws { try await wait("shell:" + id, count: count) } + func waitForHTTP(_ id: String, count: Int) async throws { try await wait("http:" + id, count: count) } + func waitForConfiguration(_ id: String) async throws { try await wait("config:" + id, count: 1) } + func waitForArchive(_ id: String) async throws { try await wait("archive:" + id, count: 1) } + + private func wait(_ key: String, count: Int) async throws { + try Task.checkCancellation() + guard counts[key, default: 0] < count else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (key, count, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.2.resume(throwing: CancellationError()) + } + } + } +} + +private actor PassiveClockBank { + private var clocks: [String: ControllableAggregateRefreshSleep] = [:] + + private func clock(_ id: String) -> ControllableAggregateRefreshSleep { + if let clock = clocks[id] { return clock } + let clock = ControllableAggregateRefreshSleep() + clocks[id] = clock + return clock + } + + func sleep(id: String, interval: Duration) async throws { + try await clock(id).sleep(for: interval) + } + + func resumeInitial(_ id: String) async { + let clock = clock(id) + _ = await clock.waitUntilRequested(count: 1) + await clock.resume() + } +} + +private actor PassiveLiveServer: WebSocketConnecting { + struct Subscription: Sendable { + let connection: PassiveLiveConnection + let requestID: Int + let payload: JSONValue? + } + private let configs = MultiEnvironmentConfigurationServer() + private var archiveGates: [String: PassiveRequestGate] = [:] + func holdArchive(host: String, gate: PassiveRequestGate) { archiveGates[host] = gate } + private var subscriptions: [String: [Subscription]] = [:] + private var interrupts: [String: Int] = [:] + private var waiters: [(String, Int, Bool, CheckedContinuation)] = [] + + func connect(to url: URL) -> any WebSocketConnection { + PassiveLiveConnection(host: url.host ?? "", server: self) + } + + func request(_ value: JSONValue, host: String, connection: PassiveLiveConnection) async throws -> JSONValue? { + if let tag = value["tag"]?.stringValue, + tag == RPCMethod.subscribeShell.rawValue || tag == RPCMethod.subscribeThread.rawValue, + case let .number(id)? = value["id"] { + let key = tag == RPCMethod.subscribeThread.rawValue ? host + "#detail" : host + subscriptions[key, default: []].append(Subscription(connection: connection, requestID: Int(id), payload: value["payload"])) + resumeWaiters() + return nil + } + if value["_tag"]?.stringValue == "Interrupt", case let .number(id)? = value["requestId"] { + for key in [host, host + "#detail"] where subscriptions[key, default: []].contains(where: { + $0.connection === connection && $0.requestID == Int(id) + }) { + interrupts[key, default: 0] += 1 + } + resumeWaiters() + } + if value["_tag"]?.stringValue == "Ping" { return .object(["_tag": .string("Pong")]) } + if value["tag"]?.stringValue == RPCMethod.dispatchCommand.rawValue, + ["thread.meta.update", "thread.unarchive"].contains(value["payload"]?["type"]?.stringValue ?? ""), + case let .number(id)? = value["id"] { + return .object([ + "_tag": .string("Exit"), "requestId": .number(id), + "exit": .object(["_tag": .string("Success"), "value": try JSONValue.encode(DispatchResult(sequence: 10))]), + ]) + } + if value["tag"]?.stringValue == RPCMethod.getArchivedShellSnapshot.rawValue, + let gate = archiveGates.removeValue(forKey: host) { await gate.enter() } + return try await configs.response(to: value, host: host) + } + + func lastPayload(host: String) -> JSONValue? { subscriptions[host]?.last?.payload } + + func waitForSubscriptions(host: String, count: Int) async { await wait(host: host, count: count, interrupt: false) } + func waitForInterrupts(host: String, count: Int) async { await wait(host: host, count: count, interrupt: true) } + + private func count(_ host: String, interrupt: Bool) -> Int { + interrupt ? interrupts[host, default: 0] : subscriptions[host, default: []].count + } + + private func wait(host: String, count: Int, interrupt: Bool) async { + guard self.count(host, interrupt: interrupt) < count else { return } + await withCheckedContinuation { waiters.append((host, count, interrupt, $0)) } + } + + private func resumeWaiters() { + let ready = waiters.filter { count($0.0, interrupt: $0.2) >= $0.1 } + waiters.removeAll { count($0.0, interrupt: $0.2) >= $0.1 } + ready.forEach { $0.3.resume() } + } + + func emit(_ values: [JSONValue], host: String) async throws { + guard let subscription = subscriptions[host]?.last else { throw URLError(.notConnectedToInternet) } + try await subscription.connection.enqueue(.object([ + "_tag": .string("Chunk"), "requestId": .number(Double(subscription.requestID)), "values": .array(values), + ])) + } + + func snapshot(_ shell: OrchestrationShellSnapshot, host: String) async throws { + try await emit([.object(["kind": .string("snapshot"), "snapshot": try JSONValue.encode(shell)])], host: host) + } + + func upsert(_ thread: OrchestrationThreadShell, sequence: Int, host: String) async throws { + try await emit([.object([ + "kind": .string("thread-upserted"), "sequence": .number(Double(sequence)), "thread": try JSONValue.encode(thread), + ])], host: host) + } + + func finish(host: String) async throws { + guard let subscription = subscriptions[host]?.last else { throw URLError(.notConnectedToInternet) } + try await subscription.connection.enqueue(.object([ + "_tag": .string("Exit"), "requestId": .number(Double(subscription.requestID)), + "exit": .object(["_tag": .string("Success"), "value": .null]), + ])) + } + + func subscriptionCount(host: String) -> Int { subscriptions[host, default: []].count } + + func latestConnectionIsClosed(host: String) async -> Bool { + await subscriptions[host]?.last?.connection.isClosed ?? false + } + + func closeLatest(host: String) async { await subscriptions[host]?.last?.connection.close() } +} + +private actor PassiveLiveConnection: WebSocketConnection { + var isClosed: Bool { closed } + private let host: String + private let server: PassiveLiveServer + private var responses: [Data] = [] + private var receiver: CheckedContinuation? + private var closed = false + + init(host: String, server: PassiveLiveServer) { self.host = host; self.server = server } + + func send(_ data: Data) async throws { + guard !closed else { throw URLError(.networkConnectionLost) } + let value = try JSONDecoder.t3.decode(JSONValue.self, from: data) + if let response = try await server.request(value, host: host, connection: self) { try enqueue(response) } + } + + func enqueue(_ value: JSONValue) throws { + let data = try JSONEncoder.t3.encode(value) + if let receiver { self.receiver = nil; receiver.resume(returning: data) } + else { responses.append(data) } + } + + func receive() async throws -> Data { + guard !closed else { throw URLError(.networkConnectionLost) } + if !responses.isEmpty { return responses.removeFirst() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + closed = true + receiver?.resume(throwing: URLError(.networkConnectionLost)) + receiver = nil + } +} + +@Suite("Native incremental bootstrap") +@MainActor +struct NativeIncrementalBootstrapTests { + @Test("Environment-store failures remain visible to the root model") + func environmentStoreFailureIsNotCancellation() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let catalog = directory.appendingPathComponent("environments.json") + try Data("invalid catalog".utf8).write(to: catalog) + let runtime = EnvironmentRuntime(environmentStore: EnvironmentStore(fileURL: catalog), + credentialStore: InMemoryCredentialStore()) + let client = NativeFeatureClient(runtime: runtime) + do { + _ = try await client.initialSnapshot() + Issue.record("The invalid catalog unexpectedly loaded") + } catch { + #expect(error is DecodingError) + #expect(!(error is CancellationError)) + } + let model = FeatureRootModel(client: client, + outboxStore: FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")), + draftStore: FeatureComposerDraftStore(fileURL: directory.appendingPathComponent("drafts.json"))) + await model.reload() + #expect(model.errorMessage?.isEmpty == false) + await model.disconnect() + } + + @Test("Pairing resets the previous environment's hydration and archive lifetime", .timeLimit(.minutes(1)), arguments: [false, true]) + func pairingReplacesHydrationLifetime(previousHydrated: Bool) async throws { + let server = PassiveLiveServer() + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + if !previousHydrated { await fixture.transport.holdNextShell(host: "one.example", gate: held) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + do { + if previousHydrated { try await receipts.waitForArchive("one") } + else { try await held.waitUntilEnteredCancellable() } + await server.waitForSubscriptions(host: "two.example", count: 1) + try await fixture.client.pair(endpoint: "https://two.example", token: "fixture-bootstrap") + await server.waitForSubscriptions(host: "two.example", count: 2) + let shell = multiEnvironmentShell(projectID: "project-two", threadID: "thread-two", title: "New active hydration", snapshotSequence: 20) + await fixture.transport.setShell(shell, host: "two.example") + try await server.upsert(shell.threads[0], sequence: 20, host: "two.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "New active hydration" } } + try await receipts.waitForArchive("two") + if !previousHydrated { #expect(await held.isHeld) } + } catch { + await held.release() + await fixture.client.disconnect() + throw error + } + await held.release() + await fixture.client.disconnect() + } + + @Test("Foreground restores bootstrap before a client has been adopted", .timeLimit(.minutes(1))) + func foregroundRestartsUnadoptedClient() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let recorder = BootstrapSnapshotRecorder( + seed: FeatureSnapshot(connection: .init(state: .disconnected)), events: fixture.client.events() + ) + defer { recorder.stop() } + fixture.client.suspendForBackground() + await fixture.client.resumeAfterBackground(reconnect: false) + let restored = try await recorder.wait { $0.threads.contains { $0.environmentID == "one" } } + #expect(restored.environments.contains { $0.id == "one" }) + await fixture.client.disconnect() + } + + @Test("An archive refresh reads the current shell after the archive RPC settles", .timeLimit(.minutes(1))) + func archiveRefreshReadsShellAfterArchive() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + try await receipts.waitForArchive("one") + let archive = PassiveRequestGate() + await server.holdArchive(host: "one.example", gate: archive) + let refresh = Task { try await fixture.client.setThreadArchived(id: "thread-one", archived: false) } + do { + try await archive.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "Current after archive", snapshotSequence: 20 + ), host: "one.example") + await archive.release() + try await refresh.value + #expect(recorder.history.last?.threads.contains { $0.title == "Current after archive" } == true) + } catch { + await archive.release() + refresh.cancel() + _ = try? await refresh.value + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("Metadata returns while active HTTP and catalogue are held; a healthy peer publishes", .timeLimit(.minutes(1))) + func heldActiveDoesNotBlockHealthyPeer() async throws { + let connector = BootstrapHeldConnector() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + includeThirdEnvironment: true, webSocketConnector: connector, + rpcConnectionWaitTimeout: .seconds(60), fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let active = PassiveRequestGate() + let passive = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: active) + await fixture.transport.holdNextShell(host: "two.example", gate: passive) + let seed = try await fixture.client.initialSnapshot() + #expect(Set(seed.environments.map(\.id)) == ["one", "two", "three"]) + #expect(seed.threads.isEmpty) + #expect(seed.environments.allSatisfy { $0.connectionState != .connected }) + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + try await active.waitUntilEnteredCancellable() + try await passive.waitUntilEnteredCancellable() + try await connector.gate.waitUntilEnteredCancellable() + let healthy = try await recorder.wait { snapshot in + snapshot.threads.contains { $0.environmentID == "three" } + && snapshot.environments.first { $0.id == "three" }?.connectionState == .connected + } + #expect(await active.isHeld) + #expect(await passive.isHeld) + #expect(await connector.gate.isHeld) + #expect(healthy.environments.first { $0.id == "one" }?.connectionState != .connected) + await active.release() + await passive.release() + await connector.gate.release() + await fixture.client.disconnect() + } + + @Test("Catalog and synchronized markers cannot authorize an empty outbox view", .timeLimit(.minutes(1))) + func authorityAndRowsPublishTogether() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: held) + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + let submission = FeatureQueuedSubmission( + environmentID: "one", identity: .init(threadID: "thread-one"), + threadID: FeatureScopedID.thread(environmentID: "one", wireID: "thread-one"), + text: "Queued before hydration", selection: nil, runtimeMode: .fullAccess, + interactionMode: .standard, attachments: [] + ) + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: seed) == .wait) + try await held.waitUntilEnteredCancellable() + await server.waitForSubscriptions(host: "one.example", count: 1) + try await server.emit([.object(["kind": .string("synchronized")])], host: "one.example") + try await receipts.waitForConfiguration("one") + let shell = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Authoritative", snapshotSequence: 10) + try await server.snapshot(shell, host: "one.example") + let ready = try await recorder.wait { $0.threads.contains { $0.title == "Authoritative" } } + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: ready) == .send) + for snapshot in recorder.history { + let connected = snapshot.environments.first { $0.id == "one" }?.connectionState == .connected + #expect(!connected || snapshot.threads.contains { $0.id == submission.threadID }) + #expect(FeatureOutboxPolicy.decision(for: submission, snapshot: snapshot) != .discard) + } + #expect(await held.isHeld) + await held.release() + await fixture.client.disconnect() + } + + @Test("Same-client reload rejects a late old HTTP response and retains cached rows", .timeLimit(.minutes(1))) + func reloadRetainsCacheAndRejectsOldRead() async throws { + let receipts = PassiveLiveReceipts() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {}, + aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { $0.threads.count == 2 } + let old = PassiveRequestGate() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Stale old request", snapshotSequence: 999), host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: old) + let cached = try await fixture.client.initialSnapshot() + #expect(cached.threads.count == 2) + #expect(cached.environments.allSatisfy { $0.connectionState != .connected }) + try await old.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Current reload", snapshotSequence: 2), host: "one.example") + _ = try await fixture.client.initialSnapshot() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Current reload" } } + await old.release() + try await receipts.waitForHTTP("one", count: 3) + #expect(!recorder.history.contains { $0.threads.contains { $0.title == "Stale old request" } }) + await fixture.client.disconnect() + } + + @Test("A current HTTP command refresh keeps the live header reconnecting while its rows are ready", .timeLimit(.minutes(1))) + func currentHTTPCommandPreservesReconnectingHeader() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + do { + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + await fixture.transport.setShell(multiEnvironmentShell( + projectID: "project-one", threadID: "thread-one", title: "HTTP command result", snapshotSequence: 2 + ), host: "one.example") + try await fixture.client.renameThread(id: "thread-one", title: "HTTP command result") + let updated = try await recorder.wait { $0.threads.contains { $0.title == "HTTP command result" } } + #expect(updated.connection.state == .reconnecting) + #expect(updated.environments.first { $0.id == "one" }?.connectionState == .connected) + #expect(await fixture.transport.dispatchRecords().count == 1) + } catch { + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("A public rename's old optional shell cannot authorize a replacement bootstrap", .timeLimit(.minutes(1))) + func oldCommandRefreshCannotAuthorizeReload() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + _ = try await fixture.hydratedSnapshot() + let recorder = try #require(fixture.recorder) + let oldRead = PassiveRequestGate() + let replacementRead = PassiveRequestGate() + let stale = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Old command shell", snapshotSequence: 999) + await fixture.transport.setShell(stale, host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: oldRead) + let command = Task { try await fixture.client.renameThread(id: "thread-one", title: "Rename once") } + do { + try await oldRead.waitUntilEnteredCancellable() + await fixture.transport.setShell(multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Replacement bootstrap", snapshotSequence: 2), host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: replacementRead) + let seed = try await fixture.client.initialSnapshot() + recorder.record(seed) + try await replacementRead.waitUntilEnteredCancellable() + await oldRead.release() + try await command.value + #expect(recorder.history.last?.environments.first { $0.id == "one" }?.connectionState != .connected) + #expect(!recorder.history.contains { $0.threads.contains { $0.title == "Old command shell" } }) + await replacementRead.release() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Replacement bootstrap" } } + } catch { + await oldRead.release() + await replacementRead.release() + command.cancel() + _ = try? await command.value + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + + @Test("A cancelled old archive cannot authorize a same-client reload", .timeLimit(.minutes(1))) + func oldArchiveCannotAuthorizeReload() async throws { + let receipts = PassiveLiveReceipts() + let server = PassiveLiveServer() + let archive = PassiveRequestGate() + await server.holdArchive(host: "one.example", gate: archive) + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregatePublishSleep: {}, aggregateRefreshReceipt: { receipts.record($0) } + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { $0.threads.contains { $0.environmentID == "one" } } + try await archive.waitUntilEnteredCancellable() + let heldShell = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: heldShell) + let reload = try await fixture.client.initialSnapshot() + recorder.record(reload) + let startIndex = recorder.history.count - 1 + try await heldShell.waitUntilEnteredCancellable() + await archive.release() + try await receipts.waitForArchive("one") + for snapshot in recorder.history.dropFirst(startIndex) { + #expect(snapshot.environments.first { $0.id == "one" }?.connectionState != .connected) + } + #expect(await heldShell.isHeld) + await heldShell.release() + await fixture.client.disconnect() + } + + @Test("Unknown active events repair once, then a replacement stream delivers deltas", .timeLimit(.minutes(1))) + func unknownActiveStreamRepairsAndResubscribes() async throws { + let server = PassiveLiveServer() + let retries = ControllableAggregateRefreshSleep() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: server, fallbackPollingInitialDelay: .seconds(60), + aggregateStreamRetrySleep: { id, interval in + if id == "one" { try await retries.sleep(for: interval) } + }, aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + await server.waitForSubscriptions(host: "one.example", count: 1) + let baseline = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active live baseline", snapshotSequence: 10) + try await server.snapshot(baseline, host: "one.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active live baseline" } } + let held = PassiveRequestGate() + let repaired = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active HTTP repaired", snapshotSequence: 20) + await fixture.transport.setShell(repaired, host: "one.example") + await fixture.transport.holdNextShell(host: "one.example", gate: held) + try await server.emit([.object(["kind": .string("unknown-active-event")])], host: "one.example") + try await held.waitUntilEnteredCancellable() + #expect(await retries.waitUntilRequested(count: 1) == .seconds(20)) + // These are delivered to the discarded subscription and cannot poison repair. + for sequence in 21...23 { try await server.upsert(baseline.threads[0], sequence: sequence, host: "one.example") } + await held.release() + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active HTTP repaired" } } + #expect(await server.subscriptionCount(host: "one.example") == 1) + await retries.resume() + await server.waitForSubscriptions(host: "one.example", count: 2) + try await server.snapshot(repaired, host: "one.example") + let delta = multiEnvironmentShell(projectID: "project-one", threadID: "thread-one", title: "Active delta recovered", snapshotSequence: 24) + try await server.upsert(delta.threads[0], sequence: 24, host: "one.example") + _ = try await recorder.wait { $0.threads.contains { $0.title == "Active delta recovered" } } + await fixture.client.disconnect() + } + + @Test("A failed initial socket attempt cannot invalidate its independent HTTP read", .timeLimit(.minutes(1))) + func failedSocketDoesNotStarveHTTP() async throws { + let connector = FailedBootstrapAttemptConnector() + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + webSocketConnector: connector, fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let held = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "one.example", gate: held) + let seed = try await fixture.client.initialSnapshot() + let recorder = BootstrapSnapshotRecorder(seed: seed, events: fixture.client.events()) + defer { recorder.stop() } + do { + try await held.waitUntilEnteredCancellable() + try await connector.waitForFailure() + await held.release() + let ready = try await recorder.wait { + $0.environments.first { $0.id == "one" }?.connectionState == .connected + } + #expect(ready.threads.contains { $0.environmentID == "one" }) + #expect(await fixture.transport.shellReadCount(host: "one.example") == 1) + } catch { + await held.release() + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } +} + +/// One collector per fixture; any number of predicates observe the same snapshots. +@MainActor +final class BootstrapSnapshotRecorder { + private(set) var history: [FeatureSnapshot] + private var task: Task? + private var waiters: [UUID: (@MainActor (FeatureSnapshot) -> Bool, CheckedContinuation)] = [:] + private var finished = false + + init(seed: FeatureSnapshot, events: AsyncStream) { + history = [seed] + task = Task { [weak self] in + for await event in events { + guard !Task.isCancelled else { break } + guard let self else { break } + switch event { + case let .snapshot(snapshot): self.record(snapshot) + case let .thread(thread): + guard var snapshot = self.history.last else { continue } + if let index = snapshot.threads.firstIndex(where: { $0.id == thread.id }) { + snapshot.threads[index] = thread + } else { snapshot.threads.append(thread) } + self.record(snapshot) + case let .threadRemoved(id): + guard var snapshot = self.history.last else { continue } + snapshot.threads.removeAll { $0.id == id } + self.record(snapshot) + default: break + } + } + self?.stop() + } + } + + func record(_ snapshot: FeatureSnapshot) { + history.append(snapshot) + let ready = waiters.filter { $0.value.0(snapshot) } + for (id, waiter) in ready { + waiters[id] = nil + waiter.1.resume(returning: snapshot) + } + } + + func wait(_ predicate: @escaping @MainActor (FeatureSnapshot) -> Bool) async throws -> FeatureSnapshot { + try Task.checkCancellation() + if let snapshot = history.last, predicate(snapshot) { return snapshot } + guard !finished else { throw CancellationError() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (predicate, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + } + } + + func stop() { + finished = true + task?.cancel() + task = nil + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.1.resume(throwing: CancellationError()) } + } + + deinit { task?.cancel() } +} + +private actor BootstrapHeldConnector: WebSocketConnecting { + let gate = PassiveRequestGate() + func connect(to url: URL) async throws -> any WebSocketConnection { + if url.host == "one.example" { await gate.enter() } + throw URLError(.cannotConnectToHost) + } +} + +@Suite("Native bootstrap durable outbox") +@MainActor +struct NativeBootstrapRootOutboxTests { + @Test("A persisted follow-up waits through another peer's update and sends once after its owner hydrates", .timeLimit(.minutes(1))) + func restoredFollowUpWaitsForOwningShell() async throws { + let fixture = try await NativeMultiEnvironmentTests.makeFixture( + fallbackPollingInitialDelay: .seconds(60), aggregatePublishSleep: {} + ) + defer { try? FileManager.default.removeItem(at: fixture.directory) } + let outbox = FeatureOutboxStore(fileURL: fixture.directory.appendingPathComponent("root-outbox.json")) + let drafts = FeatureComposerDraftStore(fileURL: fixture.directory.appendingPathComponent("root-drafts.json")) + let threadID = FeatureScopedID.thread(environmentID: "two", wireID: "thread-two") + let identity = FeatureSubmissionIdentity(threadID: "thread-two", commandID: "bootstrap-root-command", messageID: "bootstrap-root-message") + let submission = FeatureQueuedSubmission( + environmentID: "two", identity: identity, threadID: threadID, + text: "Persisted before the peer loaded", selection: nil, + runtimeMode: .fullAccess, interactionMode: .standard, attachments: [] + ) + try await outbox.enqueue(submission) + let heldShell = PassiveRequestGate() + let heldDispatch = PassiveRequestGate() + await fixture.transport.holdNextShell(host: "two.example", gate: heldShell) + await fixture.transport.holdNextDispatch(host: "two.example", gate: heldDispatch) + let model = FeatureRootModel(client: fixture.client, outboxStore: outbox, draftStore: drafts) + let run = Task { await model.start() } + do { + try await heldShell.waitUntilEnteredCancellable() + let healthy = RootObservationWaiter { + !model.isLoading && model.snapshot.threads.contains { $0.environmentID == "one" } + } + try await healthy.wait() + #expect(model.snapshot.environments.first { $0.id == "two" }?.connectionState != .connected) + #expect(try await outbox.submissions().map(\.id) == [submission.id]) + #expect(await fixture.transport.dispatchRecords().isEmpty) + #expect(await heldShell.isHeld) + + await heldShell.release() + try await heldDispatch.waitUntilEnteredCancellable() + // Load the real detail while acceptance is held; the root attaches its + // restored queued message, giving an observable completion after removal. + _ = await model.detail(for: threadID) + #expect(model.details[threadID]?.messages.first { $0.id == identity.messageID }?.state == .queued) + let completed = RootObservationWaiter { + model.details[threadID]?.messages.first { $0.id == identity.messageID }?.state == .complete + } + await heldDispatch.release() + try await completed.wait() + // Message acceptance may precede durable deletion. Wait for the actual + // owned outbox file change, independent of that presentation ordering. + try await OwnedOutboxRemovalWaiter(fileURL: fixture.directory.appendingPathComponent("root-outbox.json")).wait() + #expect(try await outbox.submissions().isEmpty) + let commands = await fixture.transport.dispatchRecords() + #expect(commands.count == 1) + #expect(commands.first?.host == "two.example") + #expect(commands.first?.command["commandId"]?.stringValue == identity.commandID) + #expect(commands.first?.command["threadId"]?.stringValue == identity.threadID) + #expect(commands.first?.command["message"]?["messageId"]?.stringValue == identity.messageID) + #expect(model.errorMessage == nil) + } catch { + await heldShell.release() + await heldDispatch.release() + run.cancel() + await fixture.client.disconnect() + await run.value + throw error + } + await heldShell.release() + await heldDispatch.release() + run.cancel() + await fixture.client.disconnect() + await run.value + } +} + +/// Observe the root's applied state without taking a second client event iterator. +@MainActor +private final class RootObservationWaiter { + private let predicate: @MainActor () -> Bool + private var continuation: CheckedContinuation? + + init(_ predicate: @escaping @MainActor () -> Bool) { self.predicate = predicate } + + func wait() async throws { + try Task.checkCancellation() + if predicate() { return } + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor [weak self] in + self?.finish(.failure(CancellationError())) + } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { predicate() } onChange: { [weak self] in + Task { @MainActor in self?.observe() } + } + if ready { finish(.success(())) } + } + + private func finish(_ result: Result) { + let pending = continuation + continuation = nil + pending?.resume(with: result) + } +} + + +/// The test watches only its owned directory. Atomic store writes replace the +/// file, so observing the directory avoids retaining an obsolete file inode. +@MainActor +private final class OwnedOutboxRemovalWaiter { + private let fileURL: URL + private var source: DispatchSourceFileSystemObject? + private var continuation: CheckedContinuation? + + init(fileURL: URL) { self.fileURL = fileURL } + + func wait() async throws { + try Task.checkCancellation() + let descriptor = open(fileURL.deletingLastPathComponent().path, O_EVTONLY) + guard descriptor >= 0 else { throw POSIXError(.EIO) } + let source = DispatchSource.makeFileSystemObjectSource( + fileDescriptor: descriptor, eventMask: [.write, .rename], queue: .global() + ) + self.source = source + source.setCancelHandler { close(descriptor) } + source.setEventHandler { [weak self] in + Task { @MainActor in self?.check() } + } + source.resume() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + check() + } + } onCancel: { + Task { @MainActor [weak self] in self?.finish(.failure(CancellationError())) } + } + } + + private func check() { + guard continuation != nil else { return } + do { + let document = try JSONSerialization.jsonObject(with: Data(contentsOf: fileURL)) as? [String: Any] + guard let submissions = document?["submissions"] as? [Any] else { throw URLError(.cannotParseResponse) } + if submissions.isEmpty { finish(.success(())) } + } catch { finish(.failure(error)) } + } + + private func finish(_ result: Result) { + let pending = continuation + continuation = nil + source?.cancel() + source = nil + pending?.resume(with: result) + } +} + + +/// Observe the failed connector attempt itself: the RPC subscription deliberately +/// keeps waiting for its first usable socket instead of failing with a UI header. +private actor FailedBootstrapAttemptConnector: WebSocketConnecting { + private let failures = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + + func connect(to url: URL) async throws -> any WebSocketConnection { + if url.host == "one.example" { failures.continuation.yield(()) } + throw URLError(.cannotConnectToHost) + } + + func waitForFailure() async throws { + try Task.checkCancellation() + var iterator = failures.stream.makeAsyncIterator() + guard await iterator.next() != nil else { throw CancellationError() } + try Task.checkCancellation() + } +} + +private actor SilentShellClock { + private(set) var intervals: [Duration] = [] + private let cancelled = PassiveRequestGate() + private var continuation: CheckedContinuation? + private var requestWaiters: [UUID: (Int, CheckedContinuation)] = [:] + + func sleep(_ interval: Duration) async throws { + intervals.append(interval) + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else { + self.continuation = continuation + let ready = requestWaiters.filter { intervals.count >= $0.value.0 } + for (id, waiter) in ready { + requestWaiters[id] = nil + waiter.1.resume() + } + } + } + } onCancel: { Task { await self.cancel() } } + } + func waitForRequest(_ count: Int = 1) async throws { + try Task.checkCancellation() + guard intervals.count < count else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { requestWaiters[id] = (count, $0) } + } onCancel: { Task { await self.cancelWaiter(id) } } + } + private func cancelWaiter(_ id: UUID) { + requestWaiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + func advance() { continuation?.resume(); continuation = nil } + func waitForCancellation() async throws { try await cancelled.waitUntilEnteredCancellable() } + func cancel() async { + continuation?.resume(throwing: CancellationError()) + continuation = nil + await cancelled.release() + await cancelled.enter() + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift index 0ce3e6a1a4f7..fd0bda3de996 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeRetryIdentityTests.swift @@ -1,4 +1,5 @@ import Foundation +import Observation import XCTest @testable import T3Code @@ -71,6 +72,258 @@ final class NativeRetryIdentityTests: XCTestCase { } } + + func testRejectedStopPropagatesRemoteError() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + await fixture.transport.holdFollowups() + let before = await fixture.transport.snapshotReadCount + await fixture.socket.rejectInterruptAcknowledgements() + do { + try await fixture.client.cancelTurn(threadID: fixture.threadID) + XCTFail("Rejected interrupt must remain a command error") + } catch let error as RPCError { + guard case .remote("Stop rejected by fixture") = error else { throw error } + } + // These counters cover the command-return boundary only; they are + // not a proof that no mistakenly queued future task could read later. + let after = await fixture.transport.snapshotReadCount + let pending = await fixture.transport.heldCount + XCTAssertEqual(after, before) + XCTAssertEqual(pending, 0) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["type"]?.stringValue, "thread.turn.interrupt") + } + + func testStopDuringHeldSendDetailPreservesDetailAndShellRecovery() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Send before Stop", selection: nil) + let firstRead = await reads.next() + let detail = try XCTUnwrap(firstRead) + XCTAssertTrue(detail.path.hasPrefix("/api/orchestration/threads/")) + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let count = await fixture.transport.heldCount + XCTAssertEqual(count, 1) + let detailStillPending = await fixture.transport.isPending(detail.id) + XCTAssertTrue(detailStillPending) + await fixture.transport.release(detail.id) + let nextRead = await reads.next() + let shell = try XCTUnwrap(nextRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + await fixture.transport.release(shell.id) + let trailingRead = await reads.next() + let trailing = try XCTUnwrap(trailingRead) + XCTAssertTrue( + trailing.path.hasPrefix("/api/orchestration/threads/"), + "Stop superseded the in-flight send detail, so the send's repair reads detail again" + ) + let marker = "Send detail recovered after Stop" + await fixture.transport.release(trailing.id, detailMessage: marker) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: marker + ).wait() + let finalRead = await reads.next() + let final = try XCTUnwrap(finalRead) + XCTAssertEqual(final.path, "/api/orchestration/shell") + await fixture.client.disconnect() + let cancelled = await cancellations.next() + XCTAssertEqual(cancelled, final.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2) + } + + func testAcceptedStopCompletesBeforeOptionalShellReadFinishes() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let completed = expectation(description: "Accepted Stop completes without snapshot reads") + let stopping = Task { + await fixture.model.cancelTurn(threadID: fixture.threadID) + completed.fulfill() + } + let read = await reads.next() + XCTAssertEqual(read?.path, "/api/orchestration/shell") + await fulfillment(of: [completed], timeout: 2) + XCTAssertFalse(fixture.model.isPerformingAction) + + await fixture.transport.failFollowups() + await stopping.value + XCTAssertNil(fixture.model.errorMessage) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(commands.first?["type"]?.stringValue, "thread.turn.interrupt") + } + + func testSendDuringStopRefreshRetainsItsDetailRefresh() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let shellRead = await reads.next() + let shell = try XCTUnwrap(shellRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "After Stop", selection: nil) + let heldCount = await fixture.transport.heldCount + XCTAssertEqual(heldCount, 1) + await fixture.transport.release(shell.id) + let detailRead = await reads.next() + let detail = try XCTUnwrap(detailRead) + XCTAssertTrue(detail.path.hasPrefix("/api/orchestration/threads/")) + + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, detail.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2) + } + + func testDisconnectCancelsStopRefreshWithoutAnotherInterrupt() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.cancelTurn(threadID: fixture.threadID) + let read = await reads.next() + let held = try XCTUnwrap(read) + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + let pending = await fixture.transport.heldCount + XCTAssertEqual(pending, 0) + } + + func testAcceptedSendCompletesOutboxBeforeOptionalReadsFinish() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups() + let completed = expectation(description: "Accepted send completes without snapshot reads") + let sending = Task { + let sent = await fixture.model.sendMessage(threadID: fixture.threadID, text: "Accepted now", selection: nil) + completed.fulfill() + return sent + } + _ = await reads.next() + await fulfillment(of: [completed], timeout: 2) + let queued = try await fixture.outbox.submissions() + XCTAssertTrue(queued.isEmpty, "Acknowledgement must retire the outbox while optional reads are held.") + XCTAssertFalse(fixture.model.isPerformingAction) + + await fixture.transport.failFollowups() + let sent = await sending.value + XCTAssertTrue(sent, "Optional read failures cannot turn acceptance into a failed send.") + XCTAssertNil(fixture.model.errorMessage) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + XCTAssertEqual(fixture.model.details[fixture.threadID]?.messages.last?.state, .complete) + await fixture.client.disconnect() + } + + func testAcceptedSendRefreshesCoalesceAndCancelWhenThreadCloses() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "First", selection: nil) + let firstRead = await reads.next() + let first = try XCTUnwrap(firstRead) + XCTAssertTrue(first.path.hasPrefix("/api/orchestration/threads/")) + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Second", selection: nil) + let heldCount = await fixture.transport.heldCount + XCTAssertEqual(heldCount, 1, "A second acceptance must coalesce behind the current read.") + + await fixture.transport.release(first.id) + let shellRead = await reads.next() + let shell = try XCTUnwrap(shellRead) + XCTAssertEqual(shell.path, "/api/orchestration/shell") + await fixture.transport.release(shell.id) + let trailingRead = await reads.next() + let trailing = try XCTUnwrap(trailingRead) + XCTAssertTrue(trailing.path.hasPrefix("/api/orchestration/threads/")) + fixture.client.releaseThread(id: fixture.threadID) + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, trailing.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 2, "Read reconciliation must never redispatch a command.") + await fixture.transport.failFollowups() + await fixture.client.disconnect() + } + + func testSupersededAcceptedSendRefreshDoesNotPublishItsOlderDetail() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + await fixture.transport.holdFollowups(includeShell: false) + let history = AcceptedSendDetailHistory(model: fixture.model, threadID: fixture.threadID) + history.start() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "First", selection: nil) + let firstRead = await reads.next() + let first = try XCTUnwrap(firstRead) + XCTAssertTrue(first.path.hasPrefix("/api/orchestration/threads/")) + // The second acceptance lands while the first read is still in flight. + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Second", selection: nil) + let stale = "Read started before the second send was accepted" + await fixture.transport.release(first.id, detailMessage: stale) + let followUpRead = await reads.next() + let followUp = try XCTUnwrap(followUpRead) + XCTAssertTrue(followUp.path.hasPrefix("/api/orchestration/threads/")) + let fresh = "Read after the second send was accepted" + await fixture.transport.release(followUp.id, detailMessage: fresh) + try await AcceptedSendDetailReceipt( + model: fixture.model, threadID: fixture.threadID, expectedMessage: fresh + ).wait() + XCTAssertFalse(history.seen.contains(stale), "A superseded read must not replace newer accepted state.") + await fixture.client.disconnect() + } + + func testDeletingThreadCancelsItsAcceptedSendRefresh() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Accepted before delete", selection: nil) + let heldRead = await reads.next() + let held = try XCTUnwrap(heldRead) + let deletion = Task { try await fixture.client.deleteThread(id: fixture.threadID) } + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id, "A deleted thread's pending refresh must not publish it again.") + await fixture.transport.failFollowups() + _ = try? await deletion.value + await fixture.client.disconnect() + } + + func testDisconnectCancelsAcceptedSendRefreshWithoutAnotherCommand() async throws { + let fixture = try await AcceptedSendFixture.make() + addTeardownBlock { await fixture.cleanUp() } + var reads = fixture.transport.heldReads.makeAsyncIterator() + var cancellations = fixture.transport.cancellations.makeAsyncIterator() + await fixture.transport.holdFollowups() + try await fixture.client.sendMessage(threadID: fixture.threadID, text: "Accepted before disconnect", selection: nil) + let heldRead = await reads.next() + let held = try XCTUnwrap(heldRead) + await fixture.client.disconnect() + let cancelledID = await cancellations.next() + XCTAssertEqual(cancelledID, held.id) + let commands = await fixture.socket.commands + XCTAssertEqual(commands.count, 1) + let pending = await fixture.transport.heldCount + XCTAssertEqual(pending, 0) + } + func testSavedSettingsSurviveAConnectionRepublish() async throws { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("t3-native-settings-republish-\(UUID().uuidString)") @@ -100,32 +353,21 @@ final class NativeRetryIdentityTests: XCTestCase { let settingsStore = UserDefaults(suiteName: settingsSuite)! defer { settingsStore.removePersistentDomain(forName: settingsSuite) } let client = NativeFeatureClient(runtime: runtime, settingsStore: settingsStore) - let initial = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + let initial = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() var updated = initial.settings updated.textSize = FeatureTextSizeAdjustment(steps: 2) updated.codeSize = FeatureTextSizeAdjustment(steps: -1) try await client.saveSettings(updated) - var events = client.events().makeAsyncIterator() - await connection.failReceive() - - var receivedReconnect = false - while let event = await events.next() { - guard case let .connection(state, _) = event, - state.state == .reconnecting else { - continue - } - receivedReconnect = true - break - } - XCTAssertTrue(receivedReconnect) - // A reconnect patches the connection instead of republishing the - // snapshot. The next snapshot the client builds must still carry the - // saved sizes. - let republished = try await client.backgroundSnapshot() - XCTAssertEqual(republished.settings.textSize.steps, 2) - XCTAssertEqual(republished.settings.codeSize.steps, -1) + let snapshot = try await recorder.wait { + $0.connection.state == .reconnecting && $0.settings.textSize.steps == 2 + } + XCTAssertEqual(snapshot.settings.textSize.steps, 2) + XCTAssertEqual(snapshot.settings.codeSize.steps, -1) await client.disconnect() } @@ -160,7 +402,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-concurrent-retry-\(UUID().uuidString)" )! ) - _ = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() await transport.rejectShellReads() @@ -226,7 +471,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-retry-\(UUID().uuidString)" )! let client = NativeFeatureClient(runtime: runtime, settingsStore: settings) - let initial = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + let initial = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } XCTAssertEqual(initial.threads.first?.runtimeMode, .approvalRequired) XCTAssertEqual(initial.threads.first?.interactionMode, .standard) await connection.waitUntilConnected() @@ -341,7 +589,10 @@ final class NativeRetryIdentityTests: XCTestCase { suiteName: "t3-native-partial-\(UUID().uuidString)" )! let client = NativeFeatureClient(runtime: runtime, settingsStore: settings) - _ = try await client.initialSnapshot() + let seed = try await client.initialSnapshot() + let recorder = AcceptedCommandSnapshotRecorder(seed: seed, events: client.events()) + defer { recorder.stop() } + _ = try await recorder.wait { !$0.projects.isEmpty && !$0.threads.isEmpty } await connection.waitUntilConnected() let identity = FeatureSubmissionIdentity( @@ -923,3 +1174,435 @@ private func retryDispatchCommand(from request: URLRequest) throws -> JSONValue } return try JSONDecoder.t3.decode(JSONValue.self, from: body) } + +@MainActor +private struct AcceptedSendFixture { + let modelTask: Task + let directory: URL + let threadID: String + let settingsName: String + let client: NativeFeatureClient + let model: FeatureRootModel + let outbox: FeatureOutboxStore + let transport: AcceptedSendHTTPTransport + let socket: AcceptedSendSocket + let runtime: EnvironmentRuntime + + static func make(includePeer: Bool = false) async throws -> Self { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("t3-accepted-send-\(UUID().uuidString)") + let environment = Environment( + id: "accepted-send", label: "Accepted send", + httpBaseURL: URL(string: "https://accepted-send.example")!, + webSocketBaseURL: URL(string: "wss://accepted-send.example")! + ) + let store = EnvironmentStore(fileURL: directory.appendingPathComponent("environments.json")) + let peer = Environment( + id: "accepted-peer", label: "Accepted peer", + httpBaseURL: URL(string: "https://accepted-peer.example")!, + webSocketBaseURL: URL(string: "wss://accepted-peer.example")! + ) + try await store.save(includePeer ? [environment, peer] : [environment]) + try await store.setActiveEnvironment(id: environment.id) + let transport = AcceptedSendHTTPTransport() + let socket = AcceptedSendSocket() + let runtime = EnvironmentRuntime( + environmentStore: store, + credentialStore: InMemoryCredentialStore(credentials: [ + environment.id: EnvironmentCredential(accessToken: "fixture-token"), + peer.id: EnvironmentCredential(accessToken: "fixture-peer-token"), + ]), + httpTransport: transport, + webSocketConnector: AcceptedSendConnector(socket: socket, peer: AcceptedSendSocket()) + ) + let settingsName = "t3-accepted-send-\(UUID().uuidString)" + let client = NativeFeatureClient( + runtime: runtime, settingsStore: UserDefaults(suiteName: settingsName)!, + fallbackPollingInitialDelay: .seconds(3600), aggregateRefreshInterval: .seconds(3600), + aggregateIdleRefreshInterval: .seconds(3600) + ) + let outbox = FeatureOutboxStore(fileURL: directory.appendingPathComponent("outbox.json")) + let model = FeatureRootModel(client: client, outboxStore: outbox) + let modelTask = Task { await model.start() } + do { + try await AcceptedSendRootReadiness(model: model).wait() + let threadID = FeatureScopedID.thread(environmentID: environment.id, wireID: "thread-existing") + _ = await model.detail(for: threadID) + return Self(modelTask: modelTask, directory: directory, threadID: threadID, + settingsName: settingsName, client: client, model: model, + outbox: outbox, transport: transport, socket: socket, runtime: runtime) + } catch { + modelTask.cancel() + await client.disconnect() + await modelTask.value + UserDefaults.standard.removePersistentDomain(forName: settingsName) + try? FileManager.default.removeItem(at: directory) + throw error + } + } + + func cleanUp() async { + modelTask.cancel() + await client.disconnect() + await modelTask.value + UserDefaults.standard.removePersistentDomain(forName: settingsName) + try? FileManager.default.removeItem(at: directory) + } +} + +@MainActor +private final class AcceptedSendRootReadiness { + private let model: FeatureRootModel + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel) { self.model = model } + + func wait() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor in self.finish(CancellationError()) } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + !model.isLoading + && model.snapshot.projects.contains { $0.id == FeatureScopedID.project(environmentID: "accepted-send", wireID: "project-1") } + && model.snapshot.threads.contains { $0.id == FeatureScopedID.thread(environmentID: "accepted-send", wireID: "thread-existing") } + && model.snapshot.environments.first(where: { $0.id == "accepted-send" })?.connectionState == .connected + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { finish(nil) } + } + + private func finish(_ error: Error?) { + guard let pending = continuation else { return } + continuation = nil + if let error { pending.resume(throwing: error) } else { pending.resume() } + } +} + +/// Records every last-message text the model shows for a thread. +@MainActor +private final class AcceptedSendDetailHistory { + private let model: FeatureRootModel + private let threadID: String + private(set) var seen: [String] = [] + + init(model: FeatureRootModel, threadID: String) { + self.model = model + self.threadID = threadID + } + + func start() { + let text = withObservationTracking { + model.details[threadID]?.messages.last?.text + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.start() } + } + if let text, seen.last != text { seen.append(text) } + } +} + +@MainActor +private final class AcceptedSendDetailReceipt { + private let model: FeatureRootModel + private let threadID: String + private let expectedMessage: String + private var continuation: CheckedContinuation? + + init(model: FeatureRootModel, threadID: String, expectedMessage: String) { + self.model = model + self.threadID = threadID + self.expectedMessage = expectedMessage + } + + func wait() async throws { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + self.continuation = continuation + observe() + } + } onCancel: { + Task { @MainActor in self.finish(CancellationError()) } + } + } + + private func observe() { + guard continuation != nil else { return } + let ready = withObservationTracking { + model.details[threadID]?.messages.last?.text == expectedMessage + } onChange: { [weak self] in + Task { @MainActor [weak self] in self?.observe() } + } + if ready { finish(nil) } + } + + private func finish(_ error: Error?) { + guard let pending = continuation else { return } + continuation = nil + if let error { pending.resume(throwing: error) } else { pending.resume() } + } +} + +private actor AcceptedSendHTTPTransport: HTTPTransport { + struct HeldRead: Sendable { + let id: UUID + let path: String + } + nonisolated let heldReads: AsyncStream + nonisolated let cancellations: AsyncStream + private let reads: AsyncStream.Continuation + private let cancelled: AsyncStream.Continuation + private(set) var snapshotReadCount = 0 + private var holdsShell = true + private var holds = false + private var fails = false + private var pending: [UUID: CheckedContinuation] = [:] + private var detailMessages: [UUID: String] = [:] + + init() { + (heldReads, reads) = AsyncStream.makeStream() + (cancellations, cancelled) = AsyncStream.makeStream() + } + + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let path = request.url?.path ?? "" + if path == "/api/auth/websocket-ticket" { + return (Data(#"{"ticket":"fixture-ticket","expiresAt":"2026-09-07T12:05:00Z"}"#.utf8), retryHTTPResponse(request)) + } + guard path == "/api/orchestration/shell" || path.hasPrefix("/api/orchestration/threads/") else { + throw URLError(.unsupportedURL) + } + snapshotReadCount += 1 + if fails { throw URLError(.networkConnectionLost) } + var heldID: UUID? + if holds && (holdsShell || path != "/api/orchestration/shell") { + let id = UUID() + heldID = id + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()); return } + pending[id] = continuation + reads.yield(HeldRead(id: id, path: path)) + } + } onCancel: { + Task { await self.cancel(id) } + } + } + let data: Data + if path == "/api/orchestration/shell" { + data = try JSONEncoder.t3.encode(retryShellSnapshot()) + } else { + let snapshot = retryEmptyThreadDetail(id: request.url!.lastPathComponent) + if let heldID, let text = detailMessages.removeValue(forKey: heldID) { + var thread = try JSONValue.encode(snapshot.thread).decode([String: JSONValue].self) + thread["messages"] = .array([.object([ + "id": .string("replacement-generation-message"), "role": .string("assistant"), + "text": .string(text), "turnId": .null, "streaming": .bool(false), + "attachments": .array([]), "createdAt": .string("2026-07-30T12:00:00.000Z"), + "updatedAt": .string("2026-07-30T12:00:00.000Z"), + ])]) + data = try JSONEncoder.t3.encode(OrchestrationThreadDetailSnapshot( + snapshotSequence: 99, + thread: try JSONValue.object(thread).decode(OrchestrationThread.self) + )) + } else { data = try JSONEncoder.t3.encode(snapshot) } + } + return (data, retryHTTPResponse(request)) + } + + var heldCount: Int { pending.count } + + func isPending(_ id: UUID) -> Bool { pending[id] != nil } + + func holdFollowups(includeShell: Bool = true) { holds = true; holdsShell = includeShell } + + func release(_ id: UUID, detailMessage: String? = nil) { + guard let continuation = pending.removeValue(forKey: id) else { return } + if let detailMessage { detailMessages[id] = detailMessage } + continuation.resume() + } + + func failFollowups() { + fails = true + holds = false + let waiting = pending.values + pending.removeAll() + waiting.forEach { $0.resume(throwing: URLError(.networkConnectionLost)) } + } + + private func cancel(_ id: UUID) { + guard let continuation = pending.removeValue(forKey: id) else { return } + continuation.resume(throwing: CancellationError()) + cancelled.yield(id) + } +} + +private struct AcceptedSendConnector: WebSocketConnecting { + let socket: AcceptedSendSocket + let peer: AcceptedSendSocket + func connect(to url: URL) -> any WebSocketConnection { + url.host == "accepted-peer.example" ? peer : socket + } +} + +private actor AcceptedSendSocket: WebSocketConnection { + nonisolated let heldInterrupts: AsyncStream + private let interruptReceipts: AsyncStream.Continuation + private var holdsInterrupts = false + private var rejectsInterrupts = false + private var pendingInterruptResponses: [Data] = [] + + init() { (heldInterrupts, interruptReceipts) = AsyncStream.makeStream() } + func holdInterruptAcknowledgements() { holdsInterrupts = true } + func rejectInterruptAcknowledgements() { rejectsInterrupts = true } + func releaseInterruptAcknowledgements() { + holdsInterrupts = false + let responses = pendingInterruptResponses + pendingInterruptResponses.removeAll() + responses.forEach { enqueue($0) } + } + private(set) var commands: [JSONValue] = [] + private var queued: [Data] = [] + private var receiver: CheckedContinuation? + + func send(_ data: Data) throws { + let request = try JSONDecoder.t3.decode(JSONValue.self, from: data) + let tag = request["tag"]?.stringValue + if tag == RPCMethod.serverGetConfig.rawValue || tag == RPCMethod.subscribeServerConfig.rawValue, + let response = try retryConfigResponse(for: request) { + enqueue(response) + } else if tag == RPCMethod.dispatchCommand.rawValue, let payload = request["payload"] { + commands.append(payload) + let isInterrupt = payload["type"]?.stringValue == "thread.turn.interrupt" + let exit: JSONValue = isInterrupt && rejectsInterrupts + ? .object(["_tag": .string("Failure"), "cause": .array([ + .object(["_tag": .string("Fail"), "error": .object(["message": .string("Stop rejected by fixture")])]), + ])]) + : .object(["_tag": .string("Success"), "value": .object(["sequence": .number(2)])]) + let response = try JSONEncoder.t3.encode(JSONValue.object([ + "_tag": .string("Exit"), "requestId": request["id"]!, "exit": exit, + ])) + if isInterrupt && holdsInterrupts { + pendingInterruptResponses.append(response) + interruptReceipts.yield(()) + } else { enqueue(response) } + } + } + + func receive() async throws -> Data { + if !queued.isEmpty { return queued.removeFirst() } + return try await withCheckedThrowingContinuation { receiver = $0 } + } + + func close() { + receiver?.resume(throwing: CancellationError()) + receiver = nil + } + + private func enqueue(_ data: Data) { + if let receiver { self.receiver = nil; receiver.resume(returning: data) } + else { queued.append(data) } + } +} + +@MainActor +private final class AcceptedCommandSnapshotRecorder { + private(set) var history: [FeatureSnapshot] + private var task: Task? + private var waiters: [UUID: (@MainActor (FeatureSnapshot) -> Bool, CheckedContinuation)] = [:] + private var finished = false + + init(seed: FeatureSnapshot, events: AsyncStream) { + history = [seed] + task = Task { [weak self] in + for await event in events { + guard !Task.isCancelled else { break } + guard let self else { break } + switch event { + case let .snapshot(snapshot): self.record(snapshot) + case let .thread(thread): + guard var snapshot = self.history.last else { continue } + if let index = snapshot.threads.firstIndex(where: { $0.id == thread.id }) { + snapshot.threads[index] = thread + } else { snapshot.threads.append(thread) } + self.record(snapshot) + case let .threadRemoved(id): + guard var snapshot = self.history.last else { continue } + snapshot.threads.removeAll { $0.id == id } + self.record(snapshot) + case let .connection(connection, environmentID): + guard var snapshot = self.history.last else { continue } + snapshot.connection = connection + if let index = snapshot.environments.firstIndex(where: { $0.id == environmentID }) { + snapshot.environments[index].connectionState = connection.state + } + self.record(snapshot) + default: break + } + } + self?.stop() + } + } + + func record(_ snapshot: FeatureSnapshot) { + history.append(snapshot) + let ready = waiters.filter { $0.value.0(snapshot) } + for (id, waiter) in ready { + waiters[id] = nil + waiter.1.resume(returning: snapshot) + } + } + + func wait(_ predicate: @escaping @MainActor (FeatureSnapshot) -> Bool) async throws -> FeatureSnapshot { + try Task.checkCancellation() + if let snapshot = history.last, predicate(snapshot) { return snapshot } + guard !finished else { throw CancellationError() } + let id = UUID() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + waiters[id] = (predicate, continuation) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.1.resume(throwing: CancellationError()) + } + } + } + + func stop() { + finished = true + task?.cancel() + task = nil + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.1.resume(throwing: CancellationError()) } + } + + deinit { task?.cancel() } +} + +private extension NativeFeatureClient { + /// Sends plain text through the full submission API with fixture defaults. + func sendMessage(threadID: String, text: String, selection: FeatureSelection?) async throws { + try await sendMessage( + threadID: threadID, text: text, selection: selection, runtimeMode: .fullAccess, + attachments: [], identity: FeatureSubmissionIdentity(threadID: threadID) + ) + } +} + +private extension FeatureRootModel { + func sendMessage(threadID: String, text: String, selection: FeatureSelection?) async -> Bool { + await sendMessage(FeatureMessageSubmission(threadID: threadID, text: text, selection: selection)) + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift index e97c5d3f5718..678d7a5c6761 100644 --- a/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/NativeThreadCatchUpTests.swift @@ -5,6 +5,960 @@ import XCTest @MainActor @available(iOS 18.0, *) final class NativeThreadCatchUpTests: XCTestCase { + func testFaultInjectedOmittedMessageRepairsAtEqualWatermark() async throws { + // Deliberate application-delivery omission, not simulated TCP packet loss. + // HTTP is authoritative for both messages; only event 4 reaches the client. + for paginated in [false, true] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, + reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + // The missing event would have sequence 3; later event 4 is valid. + try await detail.sendMessage(text: "Later delivered message", sequence: 4) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let before = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(before.messages.map(\.id), ["message-4"]) + await fixture.http.setFaultInjectionMessages(paginated: paginated) + for attempt in 1...2 { + let pendingTick = await ticks.next() + let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)) + tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + let raw = try rawThreadForHistoryTest(fixture.client) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, attempt + 1) + print("FAULT_INJECTED_EQUAL_WATERMARK paginated=\(paginated) attempt=\(attempt) reads=\(reads) messageIDs=\(raw.messages.map(\.id))") + XCTAssertEqual(Set(raw.messages.map(\.id)), Set(["message-3", "message-4"]), + "An authoritative equal-watermark read must repair omitted message 3; attempt \(attempt).") + } + await fixture.client.disconnect() + } + } + + + func testSelectedReconciliationPreservesExplicitOlderRawFanoutBelowTenUsers() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300) + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.messages.count, 150) + XCTAssertEqual(initial.messages.filter { $0.role == .user }.count, 1) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let loaded = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(loaded?.messages.count, 300) + XCTAssertEqual(loaded?.messages.filter { $0.role == .user }.count, 2) + XCTAssertEqual(loaded?.messages.first?.text, "Raw 0") + XCTAssertEqual(loaded?.page?.hasMore, false) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.last?.text, "Changed raw tail") + XCTAssertEqual(value?.messages.count, 300, "Quiet repair must preserve explicitly loaded raw-turn history even below ten user rows.") + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertNotEqual(value?.page?.hasMore, true) + print("RAW_CAP_PROOF initial=\(initial.messages.count) loaded=\(loaded?.messages.count ?? -1) loadedUsers=\(loaded?.messages.filter { $0.role == .user }.count ?? -1) repaired=\(value?.messages.count ?? -1)") + print("RAW_CAP_REQUESTS \(await fixture.http.threadRequests.map { $0.url!.absoluteString })") + } + + func testSelectedReconciliationPreservesExplicitOlderRawFanoutWithZeroUsers() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.messages.count, 150) + XCTAssertEqual(initial.messages.filter { $0.role == .user }.count, 0) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let loaded = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(loaded?.messages.count, 300) + XCTAssertEqual(loaded?.messages.filter { $0.role == .user }.count, 0) + XCTAssertEqual(loaded?.messages.first?.text, "Raw 0") + XCTAssertEqual(loaded?.page?.hasMore, false) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.last?.text, "Changed raw tail") + XCTAssertEqual(value?.messages.count, 300, "Quiet repair must preserve explicitly loaded raw-turn history even below ten user rows.") + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertNotEqual(value?.page?.hasMore, true) + print("RAW_CAP_PROOF initial=\(initial.messages.count) loaded=\(loaded?.messages.count ?? -1) loadedUsers=\(loaded?.messages.filter { $0.role == .user }.count ?? -1) repaired=\(value?.messages.count ?? -1)") + print("RAW_CAP_REQUESTS \(await fixture.http.threadRequests.map { $0.url!.absoluteString })") + } + + func testSelectedReconciliationRetainsRawCollectionsAndAcceptsAuthoritativeDeletion() async throws { + for collection in ["activities", "checkpoints"] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: [], collection: collection) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + var raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(raw.messages.count, 0) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 300) + await fixture.http.editLatestUserText("Changed raw tail", sequence: 301) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release(); _ = await completed.next() + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 300) + var urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, collection == "activities" ? 4 : 3) + XCTAssertEqual(urls.last!.contains("turnLimit="), collection == "checkpoints", + "Only windowed activities need full fallback for an ordinary changed tail.") + await fixture.http.removePaginatedUser(id: "raw-0", sequence: 302) + let pendingDelete = await ticks.next(); let deleteTick = try XCTUnwrap(pendingDelete) + clock.advance(by: .seconds(30)); deleteTick.release(); _ = await completed.next() + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) + XCTAssertFalse(raw.activities.contains { $0.id == "activity-raw-0" }) + XCTAssertFalse(raw.checkpoints.contains { $0.turnId == "turn-raw-0" }) + urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, collection == "activities" ? 6 : 5) + XCTAssertFalse(urls.last!.contains("turnLimit=")) + let pendingUnchanged = await ticks.next(); let unchangedTick = try XCTUnwrap(pendingUnchanged) + clock.advance(by: .seconds(30)); unchangedTick.release(); _ = await completed.next() + let unchangedURLs = await fixture.http.threadRequests + XCTAssertEqual(unchangedURLs.count, collection == "activities" ? 8 : 6, "A matching cursor cannot certify omitted rows outside a capped recent window.") + raw = try rawThreadForHistoryTest(fixture.client) + XCTAssertEqual(collection == "activities" ? raw.activities.count : raw.checkpoints.count, 299) + } + } + + private func rawThreadForHistoryTest(_ client: NativeFeatureClient) throws -> OrchestrationThread { + // Read-only test inspection: checkpoints are retained raw state and have + // no standalone published collection in FeatureThreadDetail. + try XCTUnwrap(Mirror(reflecting: client).children.first { $0.label == "activeRawThread" }?.value as? OrchestrationThread) + } + + func testSelectedReconciliationRepairsSilentDetailWithoutSyncFlicker() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Quiet HTTP repair", sequence: 20) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + var messages: [String] = [] + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): messages = value.messages.map(\.text) + case .threadSync(fixture.firstID, .live): break + case .threadSync(fixture.firstID, .catchingUp): XCTFail("Quiet repair must not flash catch-up.") + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(messages, ["Quiet HTTP repair"]) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } + + func testSelectedReconciliationUnchangedSnapshotPublishesNothing() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + var publications = 0 + while let event = await events.next(isolation: #isolation) { + if case .detail = event { publications += 1 } + if case .detailDelta = event { publications += 1 } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(publications, 0) + await fixture.client.disconnect() + } + + func testSelectedReconciliationActualProgressPostponesButMarkerDoesNot() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pendingFirst = await ticks.next(); let first = try XCTUnwrap(pendingFirst) + clock.advance(by: .seconds(20)) + try await detail.sendMessage(text: "Live progress", sequence: 3) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + clock.advance(by: .seconds(10)); first.release() + let deferred = await completed.next() + XCTAssertEqual(deferred, .deferred(threadID: fixture.firstID)) + let pendingNext = await ticks.next(); let next = try XCTUnwrap(pendingNext) + XCTAssertEqual(next.duration, .seconds(20)) + clock.advance(by: .seconds(10)) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + clock.advance(by: .seconds(10)); next.release() + let finished = await completed.next() + XCTAssertEqual(finished, .finished(threadID: fixture.firstID)) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } + + func testSelectedReconciliationRejectsOlderHeldHTTPAfterNewerDetail() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Old HTTP", sequence: 3) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + try await detail.sendMessage(text: "Newer live response", sequence: 4) + try await detail.synchronize() + let live = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(live, ["Newer live response"]) + held.succeed() + _ = await completed.next() + try await detail.synchronize() + let replaced = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertTrue(replaced.isEmpty, "Older HTTP must not republish over newer detail.") + await fixture.client.disconnect() + } + + func testSelectedReconciliationBackgroundCancelsItsHeldRead() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + fixture.client.suspendForBackground() + held.succeed() + let cancelled = await held.finished.first { _ in true } + XCTAssertEqual(cancelled, true) + await fixture.client.disconnect() + } + + func testSelectedReconciliationRequiredReadKeepsItsFailureAfterQuietRead() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pendingTick = await ticks.next(); let tick = try XCTUnwrap(pendingTick) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + try await detail.invalidate(sequence: 3) + await nextCatchUp(&events, threadID: fixture.firstID) + quiet.succeed() + let required = try await nextHeldRead(&reads) + required.fail() + var failure: String? + while let event = await events.next(isolation: #isolation) { + if case let .threadSync(id, .failed(message)) = event, id == fixture.firstID { + failure = message; break + } + } + XCTAssertEqual(failure, URLError(.notConnectedToInternet).localizedDescription) + await fixture.client.disconnect() + } + + private func retainForSelectedReconciliationTest(_ fixture: CatchUpFixture, clock: SelectedReconciliationClock? = nil) { + addTeardownBlock { + await MainActor.run { clock?.cancelAll() } + await fixture.http.cancelHeldReads() + await fixture.client.disconnect() + await MainActor.run { fixture.cleanUp() } + } + } + + func testSelectedReconciliationPreservesLoadedHistoryWhenUnchanged() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let older = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(older?.messages.count, 30) + let retainedPage = older?.page + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let unchanged = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertTrue(unchanged.isEmpty, "An unchanged enlarged window must not publish extra history.") + let earlier = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(retainedPage?.beforeCursor, "user-20") + XCTAssertEqual(earlier?.messages.count, 50) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("beforeCursor=user-20")) + } + + func testSelectedReconciliationExpandedGapAndDeletedBoundaryUseAuthoritativeFullRead() async throws { + for deletedBoundary in [false, true] { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + let lower = deletedBoundary ? 21 : 0 + await fixture.http.setPaginatedUserMessages(lower..<110, sequence: 200) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages.first, "User \(lower)") + XCTAssertEqual(messages.last, "User 109") + XCTAssertEqual(messages.count, 110 - lower) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertFalse(urls[3].contains("turnLimit=")) + } + } + + func testSelectedReconciliationUnchangedReadDoesNotInvalidatePendingOlderPage() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed() + _ = await completed.next() + olderRead.succeed() + let older = try await olderTask.value + XCTAssertEqual(older?.messages.count, 30) + XCTAssertEqual(older?.page?.beforeCursor, "user-20") + XCTAssertEqual(older?.page?.isLoading, false) + } + + func testSelectedReconciliationNavigationRejectsHeldResponse() async throws { + let clock = SelectedReconciliationClock() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Must not arrive after navigation", sequence: 20) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let held = try await nextHeldRead(&reads) + fixture.client.releaseThread(id: fixture.firstID) + await fixture.http.holdThreadReads(false) + _ = try await fixture.client.loadThread(id: fixture.secondID) + _ = try await nextThreadRequest(&requests) + held.succeed() + let cancelled = await held.finished.first { _ in true } + XCTAssertEqual(cancelled, true) + fixture.client.releaseThread(id: fixture.secondID) + let restored = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertFalse(restored.messages.contains { $0.text == "Must not arrive after navigation" }) + } + + func testSelectedReconciliationUsesSelectedPassivePeerRoute() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make( + includePeer: true, reconciliationClock: clock, + reconciliationReceipt: { receipts.continuation.yield($0) } + ) + retainForSelectedReconciliationTest(fixture, clock: clock) + let peerID = FeatureScopedID.thread(environmentID: "two", wireID: "first") + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: peerID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: peerID) + await fixture.http.setResponse(text: "Peer HTTP repair", sequence: 20) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: peerID)) + try await detail.synchronize() + let messages = await messagesBeforeLive(&events, threadID: peerID) + XCTAssertEqual(messages, ["Peer HTTP repair"]) + let reads = await fixture.http.threadRequests + XCTAssertEqual(reads.count, 2) + XCTAssertTrue(reads.allSatisfy { $0.url?.host == "two.example" }) + } + + func testSelectedReconciliationDefaultDeadlineRepairsMessageAndWorkingState() async throws { + let fixture = try await CatchUpFixture.make() + retainForSelectedReconciliationTest(fixture) + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + let initial = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(initial.thread.state, .idle) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setRunningResponse(text: "Default deadline repair", sequence: 20) + let start = ContinuousClock.now + var repaired: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + if value.thread.id == fixture.firstID, + value.messages.contains(where: { $0.text == "Default deadline repair" }) { + repaired = value + } + case .threadSync(fixture.firstID, .catchingUp): XCTFail("Quiet recovery must not flash catch-up.") + default: break + } + if repaired != nil { break } + } + let elapsed = start.duration(to: .now) + XCTAssertEqual(repaired?.thread.state, .working) + XCTAssertGreaterThanOrEqual(elapsed, .seconds(29)) + XCTAssertLessThan(elapsed, .seconds(40)) + let reads = await fixture.http.threadRequests + XCTAssertEqual(reads.count, 2) + print("DEFAULT_RECONCILIATION elapsed=\(elapsed) state=\(String(describing: repaired?.thread.state)) reads=\(reads.count) bytes=\(await fixture.http.threadResponseBytes)") + } + + func testSelectedReconciliationRepeatedTailEditsKeepExplicitHistoryExtent() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + for edit in 1...3 { + await fixture.http.editLatestUserText("Tail edit \(edit)", sequence: 100 + edit) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 70") + XCTAssertEqual(value?.messages.last?.text, "Tail edit \(edit)") + XCTAssertEqual(value?.page?.beforeCursor, "user-70") + } + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 5) + XCTAssertTrue(urls.dropFirst(2).allSatisfy { $0.contains("turnLimit=30") }) + } + + func testSelectedReconciliationNewUserAddsOnlyRequiredWindowExtent() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 31) + XCTAssertEqual(value?.messages.first?.text, "User 70") + XCTAssertEqual(value?.messages.last?.text, "User 100") + XCTAssertEqual(value?.page?.beforeCursor, "user-70") + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + if urls.count > 3 { XCTAssertTrue(urls[3].contains("turnLimit=31")) } + } + + func testSelectedReconciliationUserlessPendingAheadPageRecoversWithoutStreamProgress() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("New head", sequence: 301) + let pendingPage = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(pendingPage?.page?.isLoading, true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 300) + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertEqual(value?.messages.last?.text, "New head") + XCTAssertNotEqual(value?.page?.isLoading, true) + } + + func testSelectedReconciliationPendingAheadPageRecoversWithoutStreamProgress() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("New head", sequence: 51) + let pendingPage = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + XCTAssertEqual(pendingPage?.page?.isLoading, true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let receipt = await completed.next() + XCTAssertEqual(receipt, .finished(threadID: fixture.firstID)) + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 20") + XCTAssertEqual(value?.messages.last?.text, "New head") + XCTAssertEqual(value?.page?.isLoading, false) + } + + func testSelectedReconciliationUserlessChangedQuietReadYieldsToOlderPageRequest() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setRawFanoutMessages(sequence: 300, userSlots: []) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed head", sequence: 301) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed(); _ = await completed.next() + olderRead.succeed() + let waitingOlder = try await olderTask.value + XCTAssertEqual(waitingOlder?.page?.isLoading, true, "Optional quiet read must not invalidate the user's older-page epoch.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let nextTick = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); nextTick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 300) + XCTAssertEqual(value?.messages.first?.text, "Raw 0") + XCTAssertEqual(value?.messages.last?.text, "Changed head") + XCTAssertNotEqual(value?.page?.isLoading, true) + } + + func testSelectedReconciliationChangedQuietReadYieldsToOlderPageRequest() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<50, sequence: 50) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.editLatestUserText("Changed head", sequence: 51) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let quiet = try await nextHeldRead(&reads) + let olderTask = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let olderRead = try await nextHeldRead(&reads) + quiet.succeed(); _ = await completed.next() + olderRead.succeed() + let waitingOlder = try await olderTask.value + XCTAssertEqual(waitingOlder?.page?.isLoading, true, "Optional quiet read must not invalidate the user's older-page epoch.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let nextTick = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); nextTick.release() + _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 30) + XCTAssertEqual(value?.messages.first?.text, "User 20") + XCTAssertEqual(value?.messages.last?.text, "Changed head") + XCTAssertEqual(value?.page?.isLoading, false) + } + + private func selectedDetailBeforeLive(_ events: inout AsyncStream.Iterator, threadID: String) async -> FeatureThreadDetail? { + var value: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(detail), let .detailDelta(detail, _): + if detail.thread.id == threadID { value = detail } + case .threadSync(threadID, .live): return value + default: break + } + } + return value + } + + func testSelectedReconciliationMovingAdaptiveHeadDefersWithoutFullRead() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + var reads = fixture.http.heldRequests.makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + await fixture.http.holdThreadReads(true) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release() + let first = try await nextHeldRead(&reads) + await fixture.http.setPaginatedUserMessages(0..<102, sequence: 102) + first.succeed() + let adaptive = try await nextHeldRead(&reads) + adaptive.succeed(); _ = await completed.next() + var urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 4) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("turnLimit=31")) + try await detail.synchronize() + let unchanged = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertNil(unchanged, "A moving second read must not widen or replace the retained range.") + await fixture.http.holdThreadReads(false) + let nextPending = await ticks.next(); let next = try XCTUnwrap(nextPending) + clock.advance(by: .seconds(30)); next.release(); _ = await completed.next() + try await detail.synchronize() + let repaired = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired?.messages.count, 32) + XCTAssertEqual(repaired?.messages.first?.text, "User 70") + XCTAssertEqual(repaired?.messages.last?.text, "User 101") + urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 6) + XCTAssertTrue(urls[5].contains("turnLimit=32")) + } + + func testSelectedReconciliationDeletedAnchorAfterAdaptiveReadUsesOneFullFallback() async throws { + let clock = SelectedReconciliationClock() + let receipts = AsyncStream.makeStream() + let fixture = try await CatchUpFixture.make(reconciliationClock: clock, reconciliationReceipt: { receipts.continuation.yield($0) }) + retainForSelectedReconciliationTest(fixture, clock: clock) + var ticks = clock.requests.stream.makeAsyncIterator() + var completed = receipts.stream.makeAsyncIterator() + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + await fixture.http.setPaginatedUserMessages(0..<100, sequence: 100) + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + try await detail.synchronize(); _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setPaginatedUserMessages(0..<101, sequence: 101) + await fixture.http.removePaginatedUser(id: "user-70", sequence: 102) + let pending = await ticks.next(); let tick = try XCTUnwrap(pending) + clock.advance(by: .seconds(30)); tick.release(); _ = await completed.next() + try await detail.synchronize() + let value = await selectedDetailBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(value?.messages.count, 100) + XCTAssertFalse(value?.messages.contains(where: { $0.text == "User 70" }) ?? true) + let urls = await fixture.http.threadRequests.map { $0.url!.absoluteString } + XCTAssertEqual(urls.count, 5) + XCTAssertTrue(urls[2].contains("turnLimit=30")) + XCTAssertTrue(urls[3].contains("turnLimit=31")) + XCTAssertFalse(urls[4].contains("turnLimit=")) + } + + func testStaleDetailReplaySkipsReductionOnlyAfterEnvelopeValidation() throws { + let thread = multiEnvironmentDetail( + projectID: "project", threadID: "first", snapshotSequence: 2, messages: [] + ).thread + let event = replayMessage(sequence: 2, text: "Duplicate") + let ordinary = NativeThreadDetailReducer.apply(event, to: thread) + guard case .updated = ordinary.result else { + return XCTFail("The control must exercise a real message reduction.") + } + let skipped = NativeThreadDetailReducer.apply(event, to: thread, afterSequence: 2) + XCTAssertEqual(skipped.sequence, 2) + guard case .unchanged = skipped.result, case .none = skipped.renderMutation else { + return XCTFail("A validated stale event must bypass message reduction.") + } + let newer = NativeThreadDetailReducer.apply(event, to: thread, afterSequence: 1) + guard case .updated = newer.result else { return XCTFail("New events must still reduce.") } + } + + func testStaleDetailReplayPreservesNewerMessageAndExplicitMarker() async throws { + let fixture = try await CatchUpFixture.make(completionMarker: false) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + var values = (0..<100).map { index in + JSONValue.object(["kind": .string("event"), "event": replayMessage( + sequence: Double(index % 3), text: "Must not appear" + )]) + } + // These valid envelopes were ignored by the existing post-reduction cursor guard. + for type in ["future.event", "thread.message-sent"] { + values.append(.object(["kind": .string("event"), "event": .object([ + "type": .string(type), "sequence": .number(2), + "occurredAt": .string("2026-09-02T12:00:00Z"), + "payload": .object(["threadId": .string("first")]), + ])])) + } + values.append(.object(["kind": .string("event"), "event": replayMessage( + sequence: 3, text: "New content" + )])) + values.append(.object(["kind": .string("synchronized")])) + try await stream.socket.chunk(id: stream.id, values: values) + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages, ["New content"]) + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 1) + await fixture.client.disconnect() + } catch { + await fixture.client.disconnect() + throw error + } + } + + func testMalformedStaleLookingDetailEnvelopesStillRepair() async throws { + var malformed: [JSONValue] = [.null] + for field in ["type", "occurredAt"] { + var object = try replayMessage(sequence: 2, text: "Invalid").decode([String: JSONValue].self) + object.removeValue(forKey: field) + malformed.append(.object(object)) + } + var wrongThread = try replayMessage(sequence: 2, text: "Invalid").decode([String: JSONValue].self) + wrongThread["payload"] = .object(["threadId": .string("another-thread")]) + malformed.append(.object(wrongThread)) + malformed.append(replayMessage(sequence: 1.5, text: "Invalid")) + malformed.append(replayMessage(sequence: -1, text: "Invalid")) + var items = malformed.map { JSONValue.object(["kind": .string("event"), "event": $0]) } + items.append(.object(["kind": .string("unknown")])) + for (index, item) in items.enumerated() { + let fixture = try await CatchUpFixture.make(completionMarker: false) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setResponse(text: "Recovered \(index)", sequence: 10) + try await stream.socket.chunk(id: stream.id, values: [ + item, .object(["kind": .string("synchronized")]), + ]) + let messages = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(messages, ["Recovered \(index)"], "Malformed case \(index) must repair.") + let reads = await fixture.http.threadRequests.count + XCTAssertEqual(reads, 2) + await fixture.client.disconnect() + } catch { + await fixture.client.disconnect() + throw error + } + } + } + + private func replayMessage(sequence: Double, text: String) -> JSONValue { + .object([ + "type": .string("thread.message-sent"), "sequence": .number(sequence), + "occurredAt": .string("2026-09-02T12:00:00Z"), "payload": .object([ + "threadId": .string("first"), "messageId": .string("replay-message"), + "role": .string("assistant"), "text": .string(text), "streaming": .bool(false), + "createdAt": .string("2026-09-02T12:00:00Z"), + "updatedAt": .string("2026-09-02T12:00:00Z"), + ]), + ]) + } + func testLegacyReplayPublishesOncePerReceivedBatchAndResumesAfterAppliedEvents() async throws { for batchSize in [1, 16, 500] { let fixture = try await CatchUpFixture.make(completionMarker: false) @@ -187,6 +1141,286 @@ final class NativeThreadCatchUpTests: XCTestCase { await fixture.client.disconnect() } + func testLegacyBufferedFinalPrecedesShellDone() async throws { + for shellSequence in [3, 10] { + let clock = CatchUpPublicationClock() + let fixture = try await CatchUpFixture.make( + completionMarker: nil, detailPublicationSleep: { try await clock.wait() } + ) + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + var entries = clock.entries.makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.sendBurst(count: 1, includeMarker: false) + guard await entries.next(isolation: #isolation) != nil else { throw CancellationError() } + try await shell.completeShell(sequence: shellSequence, assistantMessageID: "burst-message") + var reachedDone = false + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + guard value.thread.id == fixture.firstID else { continue } + if value.thread.state == .completed { + XCTAssertEqual(value.messages.last?.text, "x") + reachedDone = true + } + default: continue + } + if reachedDone { break } + } + try Task.checkCancellation() + XCTAssertTrue(reachedDone) + } catch { + await clock.release() + await fixture.client.disconnect() + throw error + } + await clock.release() + await fixture.client.disconnect() + } + } + + func testDelayedCompletedShellPreservesNewerRunningDetailAndBackgroundLiveness() async throws { + for background: String? in [nil, "working"] { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + do { + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + try await shell.completeShell(sequence: 10, assistantMessageID: nil, title: "Shell 10") + while let event = await events.next(isolation: #isolation) { + if case let .snapshot(snapshot) = event, + snapshot.threads.contains(where: { $0.id == fixture.firstID && $0.title == "Shell 10" }) { break } + } + try Task.checkCancellation() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + try await detail.runningTurnSnapshot(sequence: 12) + var latest: FeatureThreadDetail? + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): latest = value + case .threadSync(fixture.firstID, .live): break + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + XCTAssertEqual(latest?.thread.state, .working) + let startedAt = try XCTUnwrap(latest?.thread.workingStartedAt) + try await shell.completeShell( + sequence: 11, assistantMessageID: nil, backgroundLiveness: background, title: "Shell 11" + ) + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + latest = value + XCTAssertEqual(value.thread.state, .working) + XCTAssertEqual(value.thread.workingStartedAt, startedAt) + default: break + } + if case let .snapshot(snapshot) = event, + snapshot.threads.contains(where: { $0.id == fixture.firstID && $0.title == "Shell 11" }) { break } + } + try Task.checkCancellation() + // The published shell title proves that the delayed shell was applied + // before this detail marker. + try await detail.synchronize() + while let event = await events.next(isolation: #isolation) { + switch event { + case let .detail(value), let .detailDelta(value, _): + latest = value + XCTAssertEqual(value.thread.state, .working) + XCTAssertEqual(value.thread.workingStartedAt, startedAt) + case .threadSync(fixture.firstID, .live): break + default: continue + } + if case .threadSync(fixture.firstID, .live) = event { break } + } + try Task.checkCancellation() + XCTAssertEqual(latest?.thread.state, .working) + XCTAssertEqual(latest?.backgroundWorkIsActive, background == "working") + } catch { + await fixture.client.disconnect() + throw error + } + await fixture.client.disconnect() + } + } + + func testShellCompletionRepairsMissingOrStreamingFinalWithoutClosingDetailStream() async throws { + for mode in ["missing", "streaming", "no-message-id", "no-message-id-streaming", "no-message-id-completed"] { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + if mode == "streaming" || mode == "no-message-id-streaming" { + await fixture.http.setCompletionResponse(text: "Partial", sequence: 2, streaming: true) + } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + if mode == "no-message-id-streaming" || mode == "no-message-id-completed" { + try await detail.completeTurnWithoutMessageID(sequence: 3) + } + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + + await fixture.http.setCompletionResponse(text: "Final assistant response", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Completion starts a required snapshot: \(mode)") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + try await shell.completeShell(sequence: 10, assistantMessageID: mode.hasPrefix("no-message-id") ? nil : "answer-0") + // Failure watchdog only: successful progress is signaled by the held HTTP read. + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let read = await waiting.value else { + await fixture.client.disconnect() + continue + } + read.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final assistant response"], mode) + + // The same subscription remains usable after HTTP repair. + try await detail.sendMessage(text: "Next response", sequence: 11) + try await detail.synchronize() + let next = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(next, ["Final assistant response", "Next response"], mode) + try await shell.completeShell( + sequence: 30, assistantMessageID: mode.hasPrefix("no-message-id") ? nil : "answer-0", + activeOrderKey: "already-complete" + ) + while let event = await events.next(isolation: #isolation) { + if case .threadSync(fixture.firstID, .catchingUp) = event { + XCTFail("Already-complete detail must not start another repair") + } + if case let .detail(value) = event, value.thread.activeOrderKey == "already-complete" { break } + if case let .detailDelta(value, _) = event, value.thread.activeOrderKey == "already-complete" { break } + } + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 2, mode) + await fixture.client.disconnect() + } + } + + func testShellCompletionCoalescesWhileRequiredSnapshotTracksNewerDetailEvents() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let detail = try await nextThreadRequest(&requests) + try await detail.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.setCompletionResponse(text: "Final", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Completion starts required read") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + try await shell.completeShell(sequence: 10, assistantMessageID: "answer-0") + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let first = await waiting.value else { + await fixture.client.disconnect() + return + } + await nextCatchUp(&events, threadID: fixture.firstID) + + // An unrelated shell cursor advance must not invalidate the same completion twice. + try await shell.completeShell(sequence: 100, assistantMessageID: "answer-0", activeOrderKey: "completion-seen-again") + while let event = await events.next(isolation: #isolation) { + if case let .detail(value) = event, value.thread.activeOrderKey == "completion-seen-again" { break } + if case let .detailDelta(value, _) = event, value.thread.activeOrderKey == "completion-seen-again" { break } + } + await fixture.http.setCompletionResponse(text: "Final with newer detail", sequence: 20) + try await detail.sendMessage(text: "Final with newer detail", sequence: 20) + await nextCatchUp(&events, threadID: fixture.firstID) + var reads = fixture.http.heldRequests.makeAsyncIterator() + first.succeed() + let replacement = try await nextHeldRead(&reads) + replacement.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final with newer detail"]) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 3, "Initial read plus one stale repair and one cursor replacement") + await fixture.client.disconnect() + } + + func testWarmCachedOpenRepairsAlreadyCompletedShellWithoutAnotherShellEvent() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + var requests = fixture.requests.makeAsyncIterator() + let shell = try await nextShellRequest(&requests) + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let first = try await nextThreadRequest(&requests) + try await first.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + fixture.client.releaseThread(id: fixture.firstID) + try await shell.completeShell(sequence: 10, assistantMessageID: "answer-0", activeOrderKey: "cached-completion") + while let event = await events.next(isolation: #isolation) { + if case let .thread(value) = event, value.activeOrderKey == "cached-completion" { break } + if case let .snapshot(value) = event, + value.threads.contains(where: { $0.activeOrderKey == "cached-completion" }) { break } + } + await fixture.http.setCompletionResponse(text: "Final while closed", sequence: 10) + await fixture.http.holdThreadReads(true) + let began = expectation(description: "Cached completion starts required read on reopen") + let waiting = Task { @MainActor in + var reads = fixture.http.heldRequests.makeAsyncIterator() + let read = await reads.next(isolation: #isolation) + if read != nil { began.fulfill() } + return read + } + _ = try await fixture.client.loadThread(id: fixture.firstID) + let resumed = try await nextThreadRequest(&requests) + try await resumed.synchronize() + await fulfillment(of: [began], timeout: 2) + waiting.cancel() + guard let read = await waiting.value else { + await fixture.client.disconnect() + return + } + // Ignore the warm-cache immediate live receipt preceding the repair. + while let event = await events.next(isolation: #isolation) { + if case .threadSync(fixture.firstID, .catchingUp) = event { break } + } + read.succeed() + let repaired = await messagesBeforeLive(&events, threadID: fixture.firstID) + XCTAssertEqual(repaired, ["Final while closed"]) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 2) + await fixture.client.disconnect() + } + + private func nextShellRequest( + _ iterator: inout AsyncStream.Iterator + ) async throws -> CatchUpRequest { + while let request = await iterator.next(isolation: #isolation) { + if request.tag == RPCMethod.subscribeShell.rawValue { return request } + } + throw CancellationError() + } + func testRequestSnapshotsKeepTerminalRequestsClosedAndOtherFailuresRetryable() async throws { var activities: [OrchestrationActivity] = [] for kind in ["approval", "user-input"] { @@ -444,6 +1678,34 @@ final class NativeThreadCatchUpTests: XCTestCase { await fixture.client.disconnect() } + func testWarmReopenClearsAnInterruptedOlderPageLoadingState() async throws { + let fixture = try await CatchUpFixture.make() + defer { fixture.cleanUp() } + await fixture.http.setPage(.init(beforeCursor: "older", hasMore: true, snapshotSequence: 2)) + var requests = fixture.requests.makeAsyncIterator() + var events = fixture.client.events().makeAsyncIterator() + _ = try await fixture.client.loadThread(id: fixture.firstID) + let stream = try await nextThreadRequest(&requests) + try await stream.synchronize() + _ = await messagesBeforeLive(&events, threadID: fixture.firstID) + await fixture.http.holdThreadReads(true) + var reads = fixture.http.heldRequests.makeAsyncIterator() + let older = Task { try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) } + let held = try await nextHeldRead(&reads) + fixture.client.releaseThread(id: fixture.firstID) + let restored = try await fixture.client.loadThread(id: fixture.firstID) + XCTAssertEqual(restored.page?.isLoading, false) + XCTAssertEqual(restored.page?.hasMore, true) + held.succeed() + _ = try await older.value + await fixture.http.holdThreadReads(false) + await fixture.http.setPage(.init(beforeCursor: nil, hasMore: false, snapshotSequence: 2)) + _ = try await fixture.client.loadEarlierThreadTurns(id: fixture.firstID) + let count = await fixture.http.threadRequests.count + XCTAssertEqual(count, 3, "Reopened history must allow another older-page request") + await fixture.client.disconnect() + } + func testWarmNavigationResumesAfterAppliedMessagesWithoutAnotherHTTPRead() async throws { let fixture = try await CatchUpFixture.make() defer { fixture.cleanUp() } @@ -1116,8 +2378,15 @@ private struct CatchUpFixture { var secondID: String { FeatureScopedID.thread(environmentID: "one", wireID: "second") } static func make( - completionMarker: Bool = true, + completionMarker: Bool? = true, + includePeer: Bool = false, + reconciliationClock: SelectedReconciliationClock? = nil, + reconciliationReceipt: @escaping @MainActor @Sendable (NativeSelectedThreadReconciliationReceipt) -> Void = { _ in }, + aggregateRefreshReceipt: @escaping @MainActor @Sendable (NativePassiveShellReceipt) -> Void = { _ in }, activities: [OrchestrationActivity] = [], + detailPublicationSleep: @escaping @Sendable () async throws -> Void = { + try await Task.sleep(for: .milliseconds(80)) + }, threadRetryDelay: @escaping @Sendable (Int) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(250)) } @@ -1127,7 +2396,10 @@ private struct CatchUpFixture { try await store.save([Environment( id: "one", label: "Computer", httpBaseURL: URL(string: "https://one.example")!, webSocketBaseURL: URL(string: "wss://one.example/ws")! - )]) + )] + (includePeer ? [Environment( + id: "two", label: "Peer", httpBaseURL: URL(string: "https://two.example")!, + webSocketBaseURL: URL(string: "wss://two.example/ws")! + )] : [])) try await store.setActiveEnvironment(id: "one") let http = CatchUpHTTPTransport() await http.setActivities(activities) @@ -1135,20 +2407,40 @@ private struct CatchUpFixture { let delay = CatchUpDelay() let runtime = EnvironmentRuntime( environmentStore: store, - credentialStore: InMemoryCredentialStore(credentials: ["one": .init(accessToken: "test")]), + credentialStore: InMemoryCredentialStore(credentials: ["one": .init(accessToken: "test"), "two": .init(accessToken: "test")]), httpTransport: http, webSocketConnector: CatchUpConnector( requests: requests.continuation, completionMarker: completionMarker ) ) + let readiness = CatchUpBootstrapReadiness() + let peerReadiness = CatchUpBootstrapReadiness() let client = NativeFeatureClient( runtime: runtime, settingsStore: UserDefaults(suiteName: UUID().uuidString)!, fallbackPollingInitialDelay: .seconds(3_600), aggregateRefreshInterval: .seconds(3_600), + aggregateRefreshReceipt: { + readiness.record($0) + switch $0 { + case let .shellApplied("two", sequence): peerReadiness.record(.shellApplied(environmentID: "one", sequence: sequence)) + case .configurationApplied("two"): peerReadiness.record(.configurationApplied(environmentID: "one")) + default: break + } + aggregateRefreshReceipt($0) + }, + detailPublicationSleep: detailPublicationSleep, catchUpDelay: { try await delay.wait() }, + selectedThreadReconciliationSleep: { duration in + if let reconciliationClock { try await reconciliationClock.sleep(for: duration) } + else { try await Task.sleep(for: duration) } + }, + selectedThreadReconciliationNow: { reconciliationClock?.now ?? .now }, + selectedThreadReconciliationReceipt: reconciliationReceipt, threadRetryDelay: threadRetryDelay ) _ = try await client.initialSnapshot() + try await readiness.wait() + if includePeer { try await peerReadiness.wait() } return Self(client: client, http: http, requests: requests.stream, delay: delay, directory: directory) } @@ -1156,10 +2448,30 @@ private struct CatchUpFixture { } private actor CatchUpHTTPTransport: HTTPTransport { + func setFaultInjectionMessages(paginated: Bool) { + let authoritative = [3, 4].map { index in + OrchestrationMessage(id: "message-\(index)", role: "assistant", + text: index == 3 ? "Omitted message" : "Later delivered message", + attachments: [], turnId: nil, streaming: false, + createdAt: "2026-09-02T12:00:00Z", updatedAt: "2026-09-02T12:00:00Z") + } + messages = authoritative + paginatedMessages = paginated ? authoritative : nil + sequence = 4 + } + + private(set) var threadRequests: [URLRequest] = [] + private(set) var threadResponseBytes: [Int] = [] private var messages: [OrchestrationMessage] = [] + private var paginatedMessages: [OrchestrationMessage]? + private var rawTurnCap: Int? + private var rawCollection: String? + private var pendingHeldReads: [CatchUpHTTPRead] = [] private var activities: [OrchestrationActivity] = [] - private var page: OrchestrationThreadDetailPage? + private var completionResponse = false + private var responsePage: OrchestrationThreadDetailPage? + private var runningResponse = false private var sequence = 2 private var holdsThreadReads = false private let heldReadContinuation: AsyncStream.Continuation @@ -1171,6 +2483,47 @@ private actor CatchUpHTTPTransport: HTTPTransport { heldReadContinuation = reads.continuation } + func setRawFanoutMessages(sequence: Int, userSlots: Set = [0, 200], collection: String? = nil) { + rawTurnCap = 150 + rawCollection = collection + paginatedMessages = (0..<300).map { index in + let stamp = String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60) + return OrchestrationMessage(id: "raw-\(index)", role: userSlots.contains(index) ? "user" : "assistant", + text: "Raw \(index)", attachments: [], turnId: userSlots.contains(index) ? nil : "turn-\(index)", + streaming: false, createdAt: stamp, updatedAt: stamp) + } + self.sequence = sequence + } + + func setPaginatedUserMessages(_ range: Range, sequence: Int) { + paginatedMessages = range.map { index in + OrchestrationMessage( + id: "user-\(index)", role: "user", text: "User \(index)", attachments: [], + turnId: nil, streaming: false, + createdAt: String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60), + updatedAt: String(format: "2026-09-02T12:%02d:%02dZ", index / 60, index % 60) + ) + } + self.sequence = sequence + } + func removePaginatedUser(id: String, sequence: Int) { + paginatedMessages?.removeAll { $0.id == id } + self.sequence = sequence + } + func editLatestUserText(_ text: String, sequence: Int) { + guard let latest = paginatedMessages?.popLast() else { return } + paginatedMessages?.append(OrchestrationMessage( + id: latest.id, role: latest.role, text: text, attachments: latest.attachments, + turnId: latest.turnId, streaming: latest.streaming, createdAt: latest.createdAt, + updatedAt: latest.updatedAt + )) + self.sequence = sequence + } + func cancelHeldReads() { + pendingHeldReads.forEach { $0.fail() } + pendingHeldReads.removeAll() + } + func setResponse(text: String, sequence: Int, attachment: ChatAttachment? = nil) { messages = [catchUpMessage(text, index: 0, attachment: attachment)] self.sequence = sequence @@ -1183,11 +2536,26 @@ private actor CatchUpHTTPTransport: HTTPTransport { self.sequence = sequence } + func setRunningResponse(text: String, sequence: Int) { + setResponse(text: text, sequence: sequence) + runningResponse = true + } + + func setCompletionResponse(text: String, sequence: Int, streaming: Bool = false) { + messages = [OrchestrationMessage( + id: "answer-0", role: "assistant", text: text, attachments: [], turnId: "turn-1", + streaming: streaming, createdAt: "2026-09-02T12:00:00Z", updatedAt: "2026-09-02T12:00:00Z" + )] + self.sequence = sequence + completionResponse = !streaming + } + + func setPage(_ page: OrchestrationThreadDetailPage) { responsePage = page } + func holdThreadReads(_ hold: Bool) { holdsThreadReads = hold } func setActivities(_ activities: [OrchestrationActivity]) { self.activities = activities } - func setPage(_ page: OrchestrationThreadDetailPage) { self.page = page } func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { let value: JSONValue @@ -1206,19 +2574,77 @@ private actor CatchUpHTTPTransport: HTTPTransport { throw URLError(.unsupportedURL) } threadRequests.append(request) + var responseMessages = messages + var page: OrchestrationThreadDetailPage? + if let paginatedMessages { + let query = URLComponents(url: request.url!, resolvingAgainstBaseURL: false)?.queryItems ?? [] + let limit = query.first { $0.name == "turnLimit" }?.value.flatMap(Int.init) + let cursor = query.first { $0.name == "beforeCursor" }?.value + let end = cursor.flatMap { value in paginatedMessages.firstIndex { $0.id == value } } ?? paginatedMessages.count + var start = max(0, end - (limit ?? end)) + if let cap = rawTurnCap, let limit { + // One fixture message per raw turn. Match server candidates LIMIT150, + // then walk descending until the user-anchored turn limit is reached. + start = max(0, end - cap) + var usersSeen = 0 + for index in stride(from: end - 1, through: start, by: -1) { + if paginatedMessages[index].role == "user" { usersSeen += 1 } + if usersSeen == limit { start = index; break } + } + } + responseMessages = Array(paginatedMessages[start.. 0, + snapshotSequence: sequence, threadSequence: sequence + ) + } + } let snapshot = multiEnvironmentDetail( projectID: "project", threadID: request.url!.lastPathComponent, - snapshotSequence: sequence, messages: messages + snapshotSequence: sequence, messages: responseMessages ) var thread = snapshot.thread thread.activities = activities + if let rawCollection { + thread.messages = [] + if rawCollection == "activities" { + thread.activities = responseMessages.map { row in + OrchestrationActivity(id: "activity-\(row.id)", tone: "info", kind: "fixture.raw-turn", + summary: row.text, payload: .null, turnId: "turn-\(row.id)", sequence: nil, createdAt: row.createdAt) + } + } else { + // Server checkpoints are unwindowed even when messages/activities + // hit the raw-turn cap. Keep this fixture faithful to that contract. + thread.checkpoints = (paginatedMessages ?? responseMessages).enumerated().map { offset, row in + CheckpointSummary(turnId: "turn-\(row.id)", checkpointTurnCount: offset, + checkpointRef: row.text, status: "completed", files: [], assistantMessageId: nil, completedAt: row.createdAt) + } + } + } + if runningResponse { + var object = try JSONValue.encode(thread).decode([String: JSONValue].self) + object["latestTurn"] = .object([ + "turnId": .string("synthetic-turn"), "state": .string("running"), + "requestedAt": .string("2026-09-02T12:00:00Z"), "startedAt": .string("2026-09-02T12:00:00Z"), + "completedAt": .null, "assistantMessageId": .null + ]) + object["session"] = catchUpRunningSession(threadID: thread.id, turnID: "synthetic-turn") + thread = try JSONValue.object(object).decode(OrchestrationThread.self) + } + if completionResponse { + var object = try JSONValue.encode(thread).decode([String: JSONValue].self) + object["latestTurn"] = catchUpCompletedTurn(assistantMessageID: "answer-0") + thread = try JSONValue.object(object).decode(OrchestrationThread.self) + } value = try .encode(OrchestrationThreadDetailSnapshot( - snapshotSequence: snapshot.snapshotSequence, thread: thread, page: page ?? snapshot.page + snapshotSequence: snapshot.snapshotSequence, thread: thread, page: responsePage ?? page ?? snapshot.page )) } let response = (try JSONEncoder.t3.encode(value), HTTPURLResponse( url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: nil )!) + if request.url!.path.hasPrefix("/api/orchestration/threads/") { threadResponseBytes.append(response.0.count) } if holdsThreadReads, request.url!.path.hasPrefix("/api/orchestration/threads/") { let finished = AsyncStream.makeStream() defer { @@ -1226,21 +2652,30 @@ private actor CatchUpHTTPTransport: HTTPTransport { finished.continuation.finish() } return try await withCheckedThrowingContinuation { continuation in - heldReadContinuation.yield(CatchUpHTTPRead( - response: response, continuation: continuation, finished: finished.stream - )) + let read = CatchUpHTTPRead(response: response, continuation: continuation, finished: finished.stream) + pendingHeldReads.append(read) + heldReadContinuation.yield(read) } } return response } } -private struct CatchUpHTTPRead: Sendable { +private final class CatchUpHTTPRead: @unchecked Sendable { let response: (Data, HTTPURLResponse) - let continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error> + private var continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error>? + private let lock = NSLock() let finished: AsyncStream - func succeed() { continuation.resume(returning: response) } - func fail() { continuation.resume(throwing: URLError(.notConnectedToInternet)) } + init(response: (Data, HTTPURLResponse), continuation: CheckedContinuation<(Data, HTTPURLResponse), any Error>, finished: AsyncStream) { + self.response = response; self.continuation = continuation; self.finished = finished + } + private func take() -> CheckedContinuation<(Data, HTTPURLResponse), any Error>? { + lock.lock(); defer { lock.unlock() } + defer { continuation = nil } + return continuation + } + func succeed() { take()?.resume(returning: response) } + func fail() { take()?.resume(throwing: URLError(.notConnectedToInternet)) } } private func catchUpMessage( @@ -1258,9 +2693,9 @@ private func catchUpMessage( private struct CatchUpConnector: WebSocketConnecting { let requests: AsyncStream.Continuation - let completionMarker: Bool + let completionMarker: Bool? func connect(to url: URL) async throws -> any WebSocketConnection { - CatchUpSocket(requests: requests, completionMarker: completionMarker) + CatchUpSocket(requests: requests, completionMarker: completionMarker, publishInitialShell: url.host == "two.example") } } @@ -1288,6 +2723,35 @@ private struct CatchUpRequest: Sendable { ]) } + func completeShell(sequence: Int, assistantMessageID: String?, activeOrderKey: String? = nil, backgroundLiveness: String? = nil, title: String = "First") async throws { + let shell = multiEnvironmentShell(projectID: "project", threadID: "first", title: title) + var thread = try JSONValue.encode(shell.threads[0]).decode([String: JSONValue].self) + thread["latestTurn"] = catchUpCompletedTurn(assistantMessageID: assistantMessageID) + thread["activeOrderKey"] = activeOrderKey.map(JSONValue.string) + thread["backgroundLiveness"] = backgroundLiveness.map(JSONValue.string) + let snapshot = OrchestrationShellSnapshot( + snapshotSequence: sequence, projects: shell.projects, + threads: [try JSONValue.object(thread).decode(OrchestrationThreadShell.self)], updatedAt: shell.updatedAt + ) + try await socket.chunk(id: id, values: [.object([ + "kind": .string("snapshot"), "snapshot": try .encode(snapshot), + ])]) + } + + func completeTurnWithoutMessageID(sequence: Int) async throws { + try await socket.chunk(id: id, values: [.object([ + "kind": .string("event"), "event": .object([ + "type": .string("thread.turn-diff-completed"), "sequence": .number(Double(sequence)), + "occurredAt": .string("2026-09-02T12:01:00Z"), "payload": .object([ + "threadId": payload["threadId"]!, "turnId": .string("turn-1"), + "checkpointTurnCount": .number(1), "checkpointRef": .string("refs/t3/checkpoints/turn-1"), + "status": .string("ready"), "files": .array([]), + "completedAt": .string("2026-09-02T12:01:00Z"), "assistantMessageId": .null, + ]), + ]), + ])]) + } + func terminate(_ failure: CatchUpStreamFailure) async throws { switch failure { case .malformed: @@ -1342,6 +2806,45 @@ private struct CatchUpRequest: Sendable { ])]) } + func runningTurnSnapshot(sequence: Int) async throws { + let snapshot = multiEnvironmentDetail( + projectID: "project", threadID: payload["threadId"]!.stringValue!, snapshotSequence: sequence + ) + var thread = try JSONValue.encode(snapshot.thread).decode([String: JSONValue].self) + thread["latestTurn"] = .object([ + "turnId": .string("turn-2"), "state": .string("running"), + "requestedAt": .string("2026-09-02T12:02:00Z"), + "startedAt": .string("2026-09-02T12:02:00Z"), + "completedAt": .null, "assistantMessageId": .null, + ]) + thread["session"] = catchUpRunningSession(threadID: payload["threadId"]!.stringValue!, turnID: "turn-2") + let updated = OrchestrationThreadDetailSnapshot( + snapshotSequence: sequence, thread: try JSONValue.object(thread).decode(OrchestrationThread.self) + ) + try await socket.chunk(id: id, values: [ + .object(["kind": .string("snapshot"), "snapshot": try .encode(updated)]), + ]) + } + + func sendBurst(count: Int, includeMarker: Bool) async throws { + var values: [JSONValue] = (1...count).map { index in + .object([ + "kind": .string("event"), "event": .object([ + "type": .string("thread.message-sent"), "sequence": .number(Double(index + 2)), + "occurredAt": .string("2026-09-02T12:00:00Z"), "payload": .object([ + "threadId": payload["threadId"]!, "messageId": .string("burst-message"), + "role": .string("assistant"), "text": .string(index < count ? "x" : String(repeating: "x", count: count)), + "streaming": .bool(index < count), + "createdAt": .string("2026-09-02T12:00:00Z"), + "updatedAt": .string("2026-09-02T12:00:00Z"), + ]), + ]), + ]) + } + if includeMarker { values.append(.object(["kind": .string("synchronized")])) } + try await socket.chunk(id: id, values: values) + } + func sendMessage(text: String, sequence: Int) async throws { try await socket.chunk(id: id, values: [.object([ "kind": .string("event"), "event": .object([ @@ -1359,15 +2862,18 @@ private struct CatchUpRequest: Sendable { private actor CatchUpSocket: WebSocketConnection { let requests: AsyncStream.Continuation - let completionMarker: Bool + let completionMarker: Bool? private(set) var assetRequestCount = 0 private var pending: [Data] = [] private var receiver: CheckedContinuation? private var closed = false - init(requests: AsyncStream.Continuation, completionMarker: Bool) { + private let publishInitialShell: Bool + + init(requests: AsyncStream.Continuation, completionMarker: Bool?, publishInitialShell: Bool = false) { self.requests = requests self.completionMarker = completionMarker + self.publishInitialShell = publishInitialShell } func send(_ data: Data) throws { @@ -1377,13 +2883,20 @@ private actor CatchUpSocket: WebSocketConnection { try enqueue(.object(["_tag": .string("Pong")])) } guard let tag = request["tag"]?.stringValue, case let .number(id) = request["id"] else { return } + if publishInitialShell, tag == RPCMethod.subscribeShell.rawValue { + let snapshot = multiEnvironmentShell(projectID: "project", threadID: "first", title: "Peer") + try chunk(id: Int(id), values: [.object([ + "kind": .string("snapshot"), "snapshot": try .encode(snapshot) + ])]) + } if tag == RPCMethod.assetsCreateURL.rawValue { assetRequestCount += 1 } if tag == RPCMethod.subscribeServerConfig.rawValue { + var config: [String: JSONValue] = [ + "providers": .array([]), "threadSnapshotPagination": .bool(true), + ] + if let completionMarker { config["threadResumeCompletionMarker"] = .bool(completionMarker) } try chunk(id: Int(id), values: [.object([ - "type": .string("snapshot"), "config": .object([ - "providers": .array([]), "threadSnapshotPagination": .bool(true), - "threadResumeCompletionMarker": .bool(completionMarker), - ]), + "type": .string("snapshot"), "config": .object(config), ])]) } requests.yield(.init(tag: tag, id: Int(id), payload: request["payload"]!, socket: self)) @@ -1506,3 +3019,109 @@ private actor CatchUpDelay { waiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) } } + +/// The session, not `latestTurn`, marks a thread as working. +private func catchUpRunningSession(threadID: String, turnID: String) -> JSONValue { + .object([ + "threadId": .string(threadID), "status": .string("running"), + "providerName": .string("codex"), "providerInstanceId": .null, + "runtimeMode": .string("full-access"), "activeTurnId": .string(turnID), + "lastError": .null, "updatedAt": .string("2026-09-02T12:02:00Z"), + ]) +} + +private func catchUpCompletedTurn(assistantMessageID: String?) -> JSONValue { + .object([ + "turnId": .string("turn-1"), "state": .string("completed"), + "requestedAt": .string("2026-09-02T12:00:00Z"), + "startedAt": .string("2026-09-02T12:00:00Z"), + "completedAt": .string("2026-09-02T12:01:00Z"), + "assistantMessageId": assistantMessageID.map(JSONValue.string) ?? .null, + ]) +} + +private final class CatchUpPublicationClock: Sendable { + private let gate = CatchUpDelay() + private let receipt: AsyncStream.Continuation + let entries: AsyncStream + + init() { + let pair = AsyncStream.makeStream() + entries = pair.stream + receipt = pair.continuation + } + + func wait() async throws { + receipt.yield(()) + try await gate.wait() + } + + func release() async { await gate.release() } +} + +@MainActor +private final class CatchUpBootstrapReadiness { + private var hasShell = false + private var hasConfig = false + private var waiters: [UUID: CheckedContinuation] = [:] + + func record(_ receipt: NativePassiveShellReceipt) { + switch receipt { + case .shellApplied(environmentID: "one", sequence: _): hasShell = true + case .configurationApplied(environmentID: "one"): hasConfig = true + default: break + } + if hasShell && hasConfig { + let pending = waiters.values + waiters.removeAll() + pending.forEach { $0.resume() } + } + } + + func wait() async throws { + try Task.checkCancellation() + guard !hasShell || !hasConfig else { return } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { waiters[id] = $0 } + } onCancel: { + Task { @MainActor [weak self] in + self?.waiters.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } + } + } +} +@MainActor +private final class SelectedReconciliationClock { + var now = ContinuousClock.now + let requests = AsyncStream.makeStream() + private var pending: [UUID: CheckedContinuation] = [:] + struct Wait: Sendable { + let duration: Duration + let release: @MainActor @Sendable () -> Void + } + func advance(by duration: Duration) { now = now.advanced(by: duration) } + func cancelAll() { + let waits = pending.values + pending.removeAll() + waits.forEach { $0.resume(throwing: CancellationError()) } + requests.continuation.finish() + } + func sleep(for duration: Duration) async throws { + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + if Task.isCancelled { continuation.resume(throwing: CancellationError()) } + else { + pending[id] = continuation + requests.continuation.yield(Wait(duration: duration, release: { [weak self] in + self?.pending.removeValue(forKey: id)?.resume() + })) + } + } + pending.removeValue(forKey: id) + } onCancel: { Task { @MainActor [weak self] in + self?.pending.removeValue(forKey: id)?.resume(throwing: CancellationError()) + } } + } +} diff --git a/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift b/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift index 259fdca781fa..71782c698f3c 100644 --- a/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift +++ b/apps/swift-ios/Tests/FeatureTests/T3ConnectNativeCapabilityTests.swift @@ -39,7 +39,9 @@ final class T3ConnectNativeCapabilityTests: XCTestCase { guard case let .snapshot(snapshot)? = event else { return XCTFail("Managed connect did not publish its initial Home snapshot") } - XCTAssertEqual(snapshot.connection.state, .connected) + // HTTP hydration publishes content before the deliberately blocked live stream. + XCTAssertEqual(snapshot.connection.state, .reconnecting) + XCTAssertEqual(snapshot.environments.first(where: { $0.id == "managed-1" })?.connectionState, .connected) XCTAssertEqual(snapshot.connection.environmentName, "Managed Studio") let saved = try await store.load() diff --git a/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift b/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift new file mode 100644 index 000000000000..3ae6755bf29b --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/WorkspaceThreadPresentationTests.swift @@ -0,0 +1,33 @@ +import Foundation +import Testing +@testable import T3Code + +struct WorkspaceThreadPresentationTests { + @Test(arguments: [false, true], [false, true]) + func presentationTracksCompactColumnWithoutHidingRegularDetail(isCompact: Bool, showsDetail: Bool) { + var selection = WorkspaceThreadSelection() + selection.open("a") + let isHidden = isCompact && !showsDetail + #expect(selection.presentedID(isCompact: isCompact, showsDetailColumn: showsDetail) == (isHidden ? nil : "a")) + selection.reconcilePresentation(isCompact: isCompact, showsDetailColumn: showsDetail) + #expect(selection.selectedID == (isHidden ? nil : "a")) + #expect(selection.highlightedID == "a") + } + + @Test + func compactBackAndRegularToCompactHideCannotResurrectClosedThread() { + var selection = WorkspaceThreadSelection() + selection.open("a") + selection.reconcilePresentation(isCompact: false, showsDetailColumn: false) + #expect(selection.selectedID == "a") + selection.reconcilePresentation(isCompact: true, showsDetailColumn: false) + #expect(selection.selectedID == nil) + selection.reconcilePresentation(isCompact: false, showsDetailColumn: false) + #expect(selection.presentedID(isCompact: false, showsDetailColumn: false) == nil) + #expect(selection.highlightedID == "a") + selection.open("a") + #expect(selection.presentedID(isCompact: true, showsDetailColumn: true) == "a") + selection.close() + #expect(selection.presentedID(isCompact: true, showsDetailColumn: true) == nil) + } +}