diff --git a/scripts/core-boundaries/rules/source/required-rules.mjs b/scripts/core-boundaries/rules/source/required-rules.mjs index a59739a94e..f2dcae355e 100644 --- a/scripts/core-boundaries/rules/source/required-rules.mjs +++ b/scripts/core-boundaries/rules/source/required-rules.mjs @@ -5937,7 +5937,8 @@ export const requiredContentRules = [ message: 'missing remote chat history assembly delegation', }, { - regex: /\bproject_remote_chat_user\(\s*turn\.user_message\.metadata\.as_ref\(\),\s*&prompt_visible_content\s*\)/, + regex: + /\bproject_remote_chat_user\(\s*turn\.user_message\.metadata\.as_ref\(\),\s*&prompt_visible_content,\s*read_image_pixels,?\s*\)/, message: 'missing remote chat user projection owner delegation', }, { diff --git a/src/apps/mobile/android/app/src/androidTest/kotlin/com/openbitfun/mobile/app/ConversationViewTest.kt b/src/apps/mobile/android/app/src/androidTest/kotlin/com/openbitfun/mobile/app/ConversationViewTest.kt index 1dc74f7ae8..ce2e6c5878 100644 --- a/src/apps/mobile/android/app/src/androidTest/kotlin/com/openbitfun/mobile/app/ConversationViewTest.kt +++ b/src/apps/mobile/android/app/src/androidTest/kotlin/com/openbitfun/mobile/app/ConversationViewTest.kt @@ -25,6 +25,7 @@ import androidx.compose.ui.test.performClick import androidx.compose.ui.test.performTextReplacement import androidx.compose.ui.test.performTouchInput import androidx.compose.ui.test.swipeDown +import androidx.compose.ui.unit.Dp import androidx.compose.ui.unit.dp import androidx.test.platform.app.InstrumentationRegistry import com.openbitfun.mobile.app.ui.chat.CONVERSATION_LIST_TEST_TAG @@ -288,6 +289,21 @@ class ConversationViewTest { composeRule.onNodeWithTag(CHAT_STATUS_BAR_TEST_TAG).assertDoesNotExist() } + /** + * A placeholder that outlives its subject reads as a hang, so the skeleton + * gives up on a transcript that never lands and lets the pane speak for + * itself. Matches HarmonyOS's `DeferredLoadingGate` cap. + */ + @Test + fun loadingSkeletonStopsStandingForATranscriptThatNeverArrives() { + composeRule.mainClock.autoAdvance = false + setConversationContent(state = { readyState(sessionId = "pending") }) + composeRule.mainClock.advanceTimeBy(200) + composeRule.onNodeWithTag(CONVERSATION_LOADING_TEST_TAG).assertIsDisplayed() + composeRule.mainClock.advanceTimeBy(20_000) + composeRule.onNodeWithTag(CONVERSATION_LOADING_TEST_TAG).assertDoesNotExist() + } + @Test fun composerShowsTheStoreDraftAndTypingDispatchesUpdateDraft() { val intents = mutableListOf() @@ -375,6 +391,44 @@ class ConversationViewTest { composeRule.onNodeWithTag(COMPOSER_SEND_TEST_TAG).assertIsNotEnabled() } + /** + * The composer and the header float over the transcript rather than sitting + * above and below it in a column, so the list runs the full height of the + * pane and these insets are the only thing keeping the first and last + * message out from under them. The insets are measured from the live + * overlays, so growing one — a composer going multiline, an image strip + * appearing — has to push the transcript rather than cover it. + */ + @Test + fun insetsKeepTheEndsOfTheTranscriptClearOfTheFloatingOverlays() { + val bottomInset = mutableStateOf(96.dp) + composeRule.setContent { + OpenBitFunTheme(dark = false) { + TimelineForTest( + rows = listOf(assistantRow("inset-marker")), + topInset = 72.dp, + bottomInset = bottomInset.value, + ) + } + } + composeRule.waitForIdle() + + val listBounds = composeRule.onNodeWithTag(CONVERSATION_LIST_TEST_TAG) + .getUnclippedBoundsInRoot() + val resting = composeRule.onNodeWithText("inset-marker", substring = true) + .getUnclippedBoundsInRoot() + assertTrue(resting.bottom <= listBounds.bottom - 96.dp + 1.dp) + assertTrue(resting.top >= listBounds.top) + + composeRule.runOnIdle { bottomInset.value = 180.dp } + composeRule.waitForIdle() + + val lifted = composeRule.onNodeWithText("inset-marker", substring = true) + .getUnclippedBoundsInRoot() + assertTrue(lifted.bottom <= listBounds.bottom - 180.dp + 1.dp) + assertTrue(lifted.bottom < resting.bottom) + } + private fun setConversationContent( state: () -> RemoteSessionUiState.Ready, phase: ConnectionPhase = ConnectionPhase.CONNECTED, @@ -428,10 +482,17 @@ class ConversationViewTest { ) @Composable - private fun TimelineForTest(rows: List, hasMoreMessages: Boolean = false) { + private fun TimelineForTest( + rows: List, + hasMoreMessages: Boolean = false, + topInset: Dp = 0.dp, + bottomInset: Dp = 0.dp, + ) { ConversationTimelineView( rows = rows, hasMoreMessages = hasMoreMessages, + topInset = topInset, + bottomInset = bottomInset, onLoadOlder = {}, enabled = true, onApproveTool = { _, _ -> }, diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ComposerBar.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ComposerBar.kt index 4e0077c34f..a21118a81b 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ComposerBar.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ComposerBar.kt @@ -220,8 +220,8 @@ internal fun ComposerBar( modifier = modifier .fillMaxWidth() .padding( - start = MobileDesignGeometry.ContentGutter, - end = MobileDesignGeometry.ContentGutter, + start = MobileDesignGeometry.ConversationOverlaySideInset, + end = MobileDesignGeometry.ConversationOverlaySideInset, top = 8.dp, bottom = 14.dp, ) diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt index e1f5a3a639..3b7c9b7725 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt @@ -34,6 +34,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.platform.testTag import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.Dp import androidx.compose.ui.unit.dp import com.openbitfun.mobile.app.R import com.openbitfun.mobile.core.feature.session.ConversationRow @@ -85,6 +86,13 @@ internal fun ConversationTimelineView( downloadEnabled: Boolean, modifier: Modifier, historyLoadState: HistoryLoadState = HistoryLoadState.IDLE, + /** + * How much of the pane the floating header and composer cover. The list + * runs the full height behind them, so without these the first and last + * messages would sit under a capsule and never come out from under it. + */ + topInset: Dp = 0.dp, + bottomInset: Dp = 0.dp, ) { val listState = rememberLazyListState() var stickToBottom by rememberSaveable { mutableStateOf(true) } @@ -124,7 +132,12 @@ internal fun ConversationTimelineView( LazyColumn( state = listState, modifier = Modifier.fillMaxSize().testTag(CONVERSATION_LIST_TEST_TAG), - contentPadding = PaddingValues(start = 20.dp, end = 20.dp, bottom = 12.dp), + contentPadding = PaddingValues( + start = 20.dp, + end = 20.dp, + top = topInset, + bottom = if (bottomInset > 0.dp) bottomInset else 12.dp, + ), verticalArrangement = Arrangement.spacedBy(12.dp, Alignment.Bottom), ) { if (hasMoreMessages) { @@ -172,7 +185,10 @@ internal fun ConversationTimelineView( color = MaterialTheme.colorScheme.surface, shadowElevation = 5.dp, tonalElevation = 1.dp, - modifier = Modifier.align(Alignment.BottomCenter).offset(y = (-4).dp).size(42.dp), + // Sits above the floating composer rather than behind it. + modifier = Modifier.align(Alignment.BottomCenter) + .offset(y = -(bottomInset + 4.dp)) + .size(42.dp), ) { Box(contentAlignment = Alignment.Center) { Icon( diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt index f6963132bf..81b28e6168 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt @@ -29,6 +29,9 @@ import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.background +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.layout.onSizeChanged import androidx.compose.foundation.layout.height import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material3.MaterialTheme @@ -56,8 +59,10 @@ import androidx.compose.ui.platform.LocalFocusManager import androidx.compose.ui.platform.LocalSoftwareKeyboardController import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.testTag +import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.Dp import com.openbitfun.mobile.app.R import com.openbitfun.mobile.app.ui.settings.RemoteSettingsSheet import com.openbitfun.mobile.app.ui.common.AdaptiveModalSurface @@ -65,6 +70,7 @@ import com.openbitfun.mobile.core.feature.connection.ConnectionPhase import com.openbitfun.mobile.core.feature.layout.SettingsPlacement import com.openbitfun.mobile.core.feature.session.ChatComposerCapabilities import com.openbitfun.mobile.core.feature.session.ComposerImage +import com.openbitfun.mobile.core.feature.session.ConversationRow import com.openbitfun.mobile.core.feature.session.ConversationRowKind import com.openbitfun.mobile.core.feature.session.RemoteSessionIntent.AnswerStructuredQuestion import com.openbitfun.mobile.core.feature.session.QuestionAnswer @@ -79,6 +85,18 @@ internal const val CONVERSATION_TEST_TAG: String = "conversation" internal const val CONVERSATION_BACK_TEST_TAG: String = "conversation-back" internal const val CONVERSATION_LOADING_TEST_TAG: String = "conversation-loading" +/** + * Grace before a conversation open is announced as a wait, and how long that + * announcement may stand. A cached transcript normally arrives inside the + * grace period, and swapping it for a skeleton and back reads as a stall + * rather than as speed. The cap is the other end: a transcript that never + * lands would otherwise leave the skeleton standing for the rest of the + * session, and a placeholder that outlives its subject reads as a hang. + * Matched to HarmonyOS's `DeferredLoadingGate` and iOS's open gate. + */ +private const val CONVERSATION_LOADING_DELAY_MS: Long = 140 +private const val CONVERSATION_LOADING_MAX_VISIBLE_MS: Long = 20_000 + /** * The transcript itself, tagged so a test can scroll it to a row. * @@ -142,8 +160,10 @@ internal fun ConversationView( LaunchedEffect(state.selectedSessionId, timeline == null) { loadingVisible = false if (timeline == null) { - kotlinx.coroutines.delay(140) + kotlinx.coroutines.delay(CONVERSATION_LOADING_DELAY_MS) loadingVisible = true + kotlinx.coroutines.delay(CONVERSATION_LOADING_MAX_VISIBLE_MS) + loadingVisible = false } } val activeTurn = timeline?.activeTurn @@ -239,7 +259,48 @@ internal fun ConversationView( ) { androidx.compose.foundation.layout.BoxWithConstraints(modifier.fillMaxSize()) { val mailboxMaxHeight = maxHeight * 0.4f - Column(modifier = Modifier.fillMaxSize().testTag(CONVERSATION_TEST_TAG)) { + // How much of the pane the two floating layers cover, measured rather + // than assumed: the composer grows with multiline text and attachments, + // and the mailbox appears without warning. The transcript runs the full + // height behind them and borrows these as its content insets, so a + // stale number would leave the first or last message under a capsule. + var topInset by remember { mutableStateOf(0.dp) } + var bottomInset by remember { mutableStateOf(0.dp) } + val density = LocalDensity.current + Box(modifier = Modifier.fillMaxSize().testTag(CONVERSATION_TEST_TAG)) { + if (timeline == null) { + Box(Modifier.fillMaxSize()) { + if (loadingVisible) ConversationLoadingState(Modifier.fillMaxSize()) + } + } else if (visibleRows.isEmpty() && !state.hasMoreMessages) { + ConversationEmptyState(modifier = Modifier.fillMaxSize()) + } else { + key(attachmentOwner, sessionId) { + ConversationTimelineViewHost( + state = state, + sessionId = sessionId, + visibleRows = visibleRows, + images = images, + phase = phase, + previewingRemotePath = previewingRemotePath, + previewLoading = previewLoading, + download = download, + onDownloadFile = onDownloadFile, + onOpenFile = onOpenFile, + onIntent = onIntent, + topInset = topInset, + bottomInset = bottomInset, + ) + } + } + + Column( + modifier = Modifier + .align(Alignment.TopCenter) + .fillMaxWidth() + .onSizeChanged { topInset = with(density) { it.height.toDp() } }, + ) { + Column(modifier = Modifier.fillMaxWidth().background(overlayFill())) { ConversationHeader( title = state.sessions.firstOrNull { it.id == sessionId }?.title.orEmpty(), contextTitle = contextTitle, @@ -260,116 +321,84 @@ internal fun ConversationView( ) key(sessionId) { PermissionMailboxView(state.permissionMailbox, sessionId, mailboxMaxHeight, onIntent) } - if (timeline == null) { - Box(Modifier.weight(1f).fillMaxWidth()) { - if (loadingVisible) ConversationLoadingState(Modifier.fillMaxSize()) - } - } else if (visibleRows.isEmpty() && !state.hasMoreMessages) { - ConversationEmptyState(modifier = Modifier.weight(1f).fillMaxWidth()) - } else { - key(attachmentOwner, sessionId) { - ConversationTimelineView( - rows = visibleRows, - hasMoreMessages = state.hasMoreMessages, - historyLoadState = state.historyLoadState, - onLoadOlder = { onIntent(RemoteSessionIntent.LoadOlderMessages) }, - enabled = !state.busy, - onApproveTool = { toolId, updatedInput -> - onIntent(RemoteSessionIntent.ApproveTool(sessionId, toolId, updatedInput)) - }, - onRejectTool = { toolId, reason -> - onIntent(RemoteSessionIntent.RejectTool(sessionId, toolId, reason)) - }, - onCancelTool = { toolId, reason -> - onIntent(RemoteSessionIntent.CancelTool(sessionId, toolId, reason)) - }, - onAnswerTool = { toolId, answer -> - onIntent(RemoteSessionIntent.AnswerQuestion(sessionId, toolId, answer)) - }, - onAnswerToolStructured = { toolId, answers -> - onIntent(AnswerStructuredQuestion(sessionId, toolId, answers)) - }, - onRetry = { row -> - val retryImages = row.images.map { image -> - images.firstOrNull { it.dataUrl == image.dataUrl } ?: ComposerImage( - id = image.name, - dataUrl = image.dataUrl, - mimeType = image.dataUrl.substringAfter("data:").substringBefore(';'), - ) - } - onIntent(RemoteSessionIntent.SendMessage(sessionId, row.text, retryImages)) - }, - onOpenFile = onOpenFile, - previewingRemotePath = previewingRemotePath, - previewLoading = previewLoading, - download = download, - onDownloadFile = onDownloadFile, - downloadEnabled = !state.busy && phase == ConnectionPhase.CONNECTED, - modifier = Modifier.weight(1f).fillMaxWidth(), - ) } + ConversationTopEdgeFade() } - if (attachmentDraft.failed.value) { - TextButton(onClick = { attachmentDraft.retry() }) { - Text(stringResource(R.string.chat_attachment_recovery_failed)) + + Column( + modifier = Modifier + .align(Alignment.BottomCenter) + .fillMaxWidth() + .onSizeChanged { bottomInset = with(density) { it.height.toDp() } } + // The fade covers the whole layer, not just a strip above + // it: the transcript runs behind the composer, so anything + // short of that leaves a line of text sitting crisp and + // legible beside the pill after it has already faded out + // higher up. + .background(bottomOverlayFade()), + ) { + if (attachmentDraft.failed.value) { + TextButton(onClick = { attachmentDraft.retry() }) { + Text(stringResource(R.string.chat_attachment_recovery_failed)) + } } + ComposerBar( + draft = draft, + images = images, + // An empty session id would send nowhere, so it reads as busy. + busy = state.busy || preparingImage || attachmentsBlocked || sessionId.isEmpty(), + streaming = activeTurn != null, + phase = phase, + model = timeline?.selectedModelOption(stringResource(R.string.models_unnamed)), + modelOptions = timeline?.modelOptions(stringResource(R.string.models_unnamed)) ?: emptyList(), + modelCatalogFailed = state.modelCatalogFailure != null, + capabilities = ChatComposerCapabilities.RemoteChat, + placeholder = stringResource(R.string.message_input_label), + onDraftChange = { + submittedDraft = null + onIntent(RemoteSessionIntent.UpdateDraft(it)) + }, + onRemoveImage = { id -> images = images.filterNot { it.id == id } }, + onOpenModels = { showSettings = true }, + onSelectModel = { modelId -> + onIntent(RemoteSessionIntent.SelectModel(sessionId, modelId)) + }, + modifier = Modifier, + onAttach = { + pickerOwner = attachmentKey + val remaining = MAX_COMPOSER_IMAGES - images.size + if (remaining == 1) singlePhotoPicker.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) + else if (remaining > 1) photoPicker.launch( + PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly, maxItems = remaining), + ) + }, + onVoice = { + voiceInput.launch( + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra( + RecognizerIntent.EXTRA_LANGUAGE_MODEL, + RecognizerIntent.LANGUAGE_MODEL_FREE_FORM, + ) + }, + ) + }, + onSend = { + submittedDraft = draft + keyboard?.hide() + focusManager.clearFocus(force = true) + onIntent( + RemoteSessionIntent.SendMessage( + sessionId, + draft, + images.takeIf { it.isNotEmpty() }, + ), + ) + }, + onStop = { + onIntent(RemoteSessionIntent.CancelTurn(sessionId, activeTurn?.turnId)) + }, + ) } - ComposerBar( - draft = draft, - images = images, - // An empty session id would send nowhere, so it reads as busy. - busy = state.busy || preparingImage || attachmentsBlocked || sessionId.isEmpty(), - streaming = activeTurn != null, - phase = phase, - model = timeline?.selectedModelOption(stringResource(R.string.models_unnamed)), - modelOptions = timeline?.modelOptions(stringResource(R.string.models_unnamed)) ?: emptyList(), - modelCatalogFailed = state.modelCatalogFailure != null, - capabilities = ChatComposerCapabilities.RemoteChat, - placeholder = stringResource(R.string.message_input_label), - onDraftChange = { - submittedDraft = null - onIntent(RemoteSessionIntent.UpdateDraft(it)) - }, - onRemoveImage = { id -> images = images.filterNot { it.id == id } }, - onOpenModels = { showSettings = true }, - onSelectModel = { modelId -> - onIntent(RemoteSessionIntent.SelectModel(sessionId, modelId)) - }, - modifier = Modifier, - onAttach = { - pickerOwner = attachmentKey - val remaining = MAX_COMPOSER_IMAGES - images.size - if (remaining == 1) singlePhotoPicker.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) - else if (remaining > 1) photoPicker.launch( - PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly, maxItems = remaining), - ) - }, - onVoice = { - voiceInput.launch( - Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { - putExtra( - RecognizerIntent.EXTRA_LANGUAGE_MODEL, - RecognizerIntent.LANGUAGE_MODEL_FREE_FORM, - ) - }, - ) - }, - onSend = { - submittedDraft = draft - keyboard?.hide() - focusManager.clearFocus(force = true) - onIntent( - RemoteSessionIntent.SendMessage( - sessionId, - draft, - images.takeIf { it.isNotEmpty() }, - ), - ) - }, - onStop = { - onIntent(RemoteSessionIntent.CancelTurn(sessionId, activeTurn?.turnId)) - }, - ) } } @@ -391,6 +420,115 @@ internal fun ConversationView( } } +/** + * The timeline call is long enough that inlining it under the floating layout + * buried the layout itself. Nothing here is new behaviour; it is the same call + * with the measured insets handed through. + */ +@Composable +private fun ConversationTimelineViewHost( + state: RemoteSessionUiState.Ready, + sessionId: String, + visibleRows: List, + images: List, + phase: ConnectionPhase, + previewingRemotePath: String, + previewLoading: Boolean, + download: RemoteFileDownloadUiState, + onDownloadFile: (String, String) -> Unit, + onOpenFile: (String, String) -> Unit, + onIntent: (RemoteSessionIntent) -> Unit, + topInset: Dp, + bottomInset: Dp, +) { + ConversationTimelineView( + rows = visibleRows, + hasMoreMessages = state.hasMoreMessages, + historyLoadState = state.historyLoadState, + onLoadOlder = { onIntent(RemoteSessionIntent.LoadOlderMessages) }, + enabled = !state.busy, + onApproveTool = { toolId, updatedInput -> + onIntent(RemoteSessionIntent.ApproveTool(sessionId, toolId, updatedInput)) + }, + onRejectTool = { toolId, reason -> + onIntent(RemoteSessionIntent.RejectTool(sessionId, toolId, reason)) + }, + onCancelTool = { toolId, reason -> + onIntent(RemoteSessionIntent.CancelTool(sessionId, toolId, reason)) + }, + onAnswerTool = { toolId, answer -> + onIntent(RemoteSessionIntent.AnswerQuestion(sessionId, toolId, answer)) + }, + onAnswerToolStructured = { toolId, answers -> + onIntent(AnswerStructuredQuestion(sessionId, toolId, answers)) + }, + onRetry = { row -> + val retryImages = row.images.map { image -> + images.firstOrNull { it.dataUrl == image.dataUrl } ?: ComposerImage( + id = image.name, + dataUrl = image.dataUrl, + mimeType = image.dataUrl.substringAfter("data:").substringBefore(';'), + ) + } + onIntent(RemoteSessionIntent.SendMessage(sessionId, row.text, retryImages)) + }, + onOpenFile = onOpenFile, + previewingRemotePath = previewingRemotePath, + previewLoading = previewLoading, + download = download, + onDownloadFile = onDownloadFile, + downloadEnabled = !state.busy && phase == ConnectionPhase.CONNECTED, + modifier = Modifier.fillMaxSize(), + topInset = topInset, + bottomInset = bottomInset, + ) +} + +/** + * Softens where the transcript meets a floating layer. Without it a line of + * text is cut in half at the capsule's edge and reads as clipped rather than as + * continuing underneath. + */ +/** + * The floor under the bottom layer: transparent where the transcript is still + * fully readable, settling into the page colour by the time it reaches the + * composer. Matches the gradient HarmonyOS paints behind the same layer. + */ +@Composable +private fun bottomOverlayFade(): Brush = Brush.verticalGradient( + // The page colour at zero alpha rather than a bare transparent: a gradient + // run to plain transparent interpolates through grey and leaves a smudge + // over the transcript. + 0f to overlayFill().copy(alpha = 0f), + 0.45f to overlayFill(), + 1f to overlayFill(), +) + +/** + * Carries the header's opaque band down into the transcript, so a line of text + * is not cut in half at its edge. The bottom layer needs no strip of its own — + * its whole background is [bottomOverlayFade]. + */ +@Composable +private fun ConversationTopEdgeFade() { + val page = MaterialTheme.colorScheme.background + Box( + modifier = Modifier + .fillMaxWidth() + .height(MobileDesignGeometry.ConversationEdgeFadeHeight) + .background(Brush.verticalGradient(listOf(page, page.copy(alpha = 0f)))), + ) +} + +/** + * The floor under the top layer. Compose has no first-party backdrop blur, so + * where iOS and HarmonyOS put a material this puts a near-opaque page colour: + * the transcript still passes behind it, and the header stays readable over + * whatever is passing. + */ +@Composable +private fun overlayFill(): Color = MaterialTheme.colorScheme.background.copy(alpha = 0.94f) + @Composable private fun ConversationLoadingState(modifier: Modifier) { Column(modifier.testTag(CONVERSATION_LOADING_TEST_TAG).padding(horizontal = 22.dp, vertical = 28.dp), diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/miniapps/MiniApps.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/miniapps/MiniApps.kt index 747c45189b..63fe76828f 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/miniapps/MiniApps.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/miniapps/MiniApps.kt @@ -54,7 +54,7 @@ internal fun MiniAppsButton( ) { var open by rememberSaveable { mutableStateOf(false) } if (sidebar) { - Row(Modifier.fillMaxWidth().padding(vertical = 16.dp).heightIn(min = 48.dp) + Row(Modifier.fillMaxWidth().padding(top = 4.dp, bottom = 8.dp).heightIn(min = 48.dp) .clip(RoundedCornerShape(12.dp)).clickable { open = true } .padding(start = 4.dp, end = 8.dp).testTag("sidebar-miniapps"), horizontalArrangement = Arrangement.spacedBy(14.dp), verticalAlignment = Alignment.CenterVertically) { diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/MobileScreen.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/MobileScreen.kt index 082fd13347..631f0be612 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/MobileScreen.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/MobileScreen.kt @@ -70,6 +70,7 @@ import com.openbitfun.mobile.core.feature.layout.SettingsPlacementPolicy import com.openbitfun.mobile.core.feature.layout.SettingsSheetKind import com.openbitfun.mobile.core.feature.session.RemoteSessionUiState import com.openbitfun.mobile.core.feature.session.RemoteSessionIntent +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryUiState import com.openbitfun.mobile.core.feature.workspace.RemoteFilePreviewUiState import com.openbitfun.mobile.core.feature.workspace.RemoteFileDownloadUiState import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceIntent @@ -133,6 +134,7 @@ internal fun MobileScreen() { val accountState by accountViewModel.state.collectAsStateWithLifecycle() val accountRemoteState by accountViewModel.remoteState.collectAsStateWithLifecycle() val accountPhase by accountViewModel.connectionPhase.collectAsStateWithLifecycle() + val accountWorkspaceDirectory by accountViewModel.workspaceDirectory.collectAsStateWithLifecycle() val readyAccount = accountState as? AccountUiState.Ready val linkContext = androidx.compose.ui.platform.LocalContext.current var pendingDeviceLink by rememberSaveable { mutableStateOf(null) } @@ -180,6 +182,10 @@ internal fun MobileScreen() { RemoteControlSource.ACCOUNT_DEVICE -> accountRemoteState RemoteControlSource.NONE -> RemoteSessionUiState.Idle } + val activeWorkspaceDirectory = when (controlSummary.source) { + RemoteControlSource.ACCOUNT_DEVICE -> accountWorkspaceDirectory + RemoteControlSource.NONE -> WorkspaceSessionDirectoryUiState(emptyList()) + } fun dispatchActiveWorkspace(intent: RemoteWorkspaceIntent) { when (controlSummary.source) { @@ -316,6 +322,7 @@ internal fun MobileScreen() { remoteDeviceName = controlSummary.desktopName, remoteState = activeRemoteState, workspaceState = activeWorkspaceState, + workspaceDirectory = activeWorkspaceDirectory, remoteActive = shell.surface == MobileSurface.REMOTE, remoteSelectedSessionId = shell.remoteSessionId, query = shell.sidebarQuery, @@ -374,6 +381,16 @@ internal fun MobileScreen() { shell.closeRemoteSession() closeDrawer() }, + onExpandRemoteWorkspace = { path, connectionId, sshHost -> + dispatchActiveSession( + RemoteSessionIntent.LoadWorkspaceSessions(path, connectionId, sshHost), + ) + }, + onRetryRemoteWorkspaceSessions = { path, connectionId, sshHost -> + dispatchActiveSession( + RemoteSessionIntent.RetryWorkspaceSessions(path, connectionId, sshHost), + ) + }, onDeleteRemoteSession = { id -> dispatchActiveSession(RemoteSessionIntent.DeleteSession(id)) }, onOpenSettings = { // HarmonyOS' `onSidebar.settings` always opens root settings. diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/AppSidebar.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/AppSidebar.kt index 9c9e36d61c..89cc825f98 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/AppSidebar.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/AppSidebar.kt @@ -4,7 +4,6 @@ import androidx.compose.foundation.background import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll import androidx.compose.foundation.clickable -import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row @@ -43,6 +42,7 @@ import com.openbitfun.mobile.core.feature.connection.RemoteControlSource import com.openbitfun.mobile.core.feature.session.SessionActionPolicy import com.openbitfun.mobile.core.feature.session.SessionActionScope import com.openbitfun.mobile.core.feature.session.RemoteSessionUiState +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryUiState import com.openbitfun.mobile.core.feature.layout.SettingsPlacement import com.openbitfun.mobile.core.feature.shell.RemoteSidebarSessionRow import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceUiState @@ -76,6 +76,7 @@ internal fun AppSidebar( remoteDeviceName: String, remoteState: RemoteSessionUiState, workspaceState: RemoteWorkspaceUiState, + workspaceDirectory: WorkspaceSessionDirectoryUiState, remoteActive: Boolean, remoteSelectedSessionId: String?, query: String, @@ -91,6 +92,8 @@ internal fun AppSidebar( onOpenRemoteSession: (String) -> Unit, onCreateRemoteInWorkspace: (String, String?, String?, String) -> Unit, onOpenRemoteWorkspace: (String) -> Unit, + onExpandRemoteWorkspace: (String, String?, String?) -> Unit, + onRetryRemoteWorkspaceSessions: (String, String?, String?) -> Unit, onAddRemoteWorkspace: (() -> Unit)? = null, onWorkspaceTool: (String, String?, Boolean) -> Unit, onDeleteRemoteSession: (String) -> Unit, @@ -104,11 +107,12 @@ internal fun AppSidebar( var remoteDetailsSessionId by rememberSaveable { mutableStateOf(null) } Box(modifier = modifier.fillMaxSize().testTag(SIDEBAR_TEST_TAG)) { + // No spacedBy: the MiniApps row carries its own 4dp/8dp rhythm, and a + // column-level gap on top of it would push the section header away. Column( modifier = Modifier .fillMaxSize() .padding(start = 20.dp, end = 20.dp, top = 0.dp, bottom = 16.dp), - verticalArrangement = Arrangement.spacedBy(8.dp), ) { if (signedIn) { SidebarAuthenticatedHeader(searchOpen, query, onQueryChange, onToggleSearch) @@ -127,6 +131,7 @@ internal fun AppSidebar( deviceName = remoteDeviceName, remoteState = remoteState, workspaceState = workspaceState, + workspaceDirectory = workspaceDirectory, selectedSessionId = remoteSelectedSessionId.takeIf { remoteActive }, onConnect = onScanDesktop, onRetryActive = onRetryRemoteDevice, @@ -141,6 +146,8 @@ internal fun AppSidebar( }, onCreateInWorkspace = onCreateRemoteInWorkspace, onOpenWorkspace = onOpenRemoteWorkspace, + onExpandWorkspace = onExpandRemoteWorkspace, + onRetryWorkspaceSessions = onRetryRemoteWorkspaceSessions, onAddWorkspace = onAddRemoteWorkspace, onWorkspaceTool = onWorkspaceTool, ) diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarHeader.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarHeader.kt index 27e2c38027..78f218af56 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarHeader.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarHeader.kt @@ -75,6 +75,9 @@ private fun SidebarSearchField(query: String, onQueryChange: (String) -> Unit) { Row( modifier = Modifier .fillMaxWidth() + // The header column adds no spacing of its own, so the field keeps + // its own gap under the title, matching the source's 12dp. + .padding(top = 12.dp) .height(42.dp) .clip(RoundedCornerShape(8.dp)) .background(MaterialTheme.colorScheme.surfaceVariant) diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarRemoteWorkspaceSection.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarRemoteWorkspaceSection.kt index b6243cc1fd..5b40491ddd 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarRemoteWorkspaceSection.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/shell/sidebar/SidebarRemoteWorkspaceSection.kt @@ -39,6 +39,7 @@ import androidx.compose.ui.semantics.contentDescription import androidx.compose.ui.semantics.semantics import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.Dp import androidx.compose.ui.unit.IntRect import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp @@ -49,6 +50,8 @@ import com.openbitfun.mobile.core.feature.connection.ConnectionPhase import com.openbitfun.mobile.core.feature.connection.ConnectionStatusPresenter import com.openbitfun.mobile.core.feature.connection.RemoteControlSource import com.openbitfun.mobile.core.feature.shell.RemoteSidebarSessionRow +import com.openbitfun.mobile.core.feature.shell.RemoteSidebarWorkspaceLoad +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryUiState import com.openbitfun.mobile.core.feature.session.RemoteSessionUiState import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceUiState import com.openbitfun.mobile.app.ui.theme.openBitFunColors @@ -69,6 +72,7 @@ internal fun SidebarRemoteWorkspaceSection( deviceName: String, remoteState: RemoteSessionUiState, workspaceState: RemoteWorkspaceUiState, + workspaceDirectory: WorkspaceSessionDirectoryUiState, selectedSessionId: String?, onConnect: () -> Unit, onRetryActive: () -> Unit, @@ -80,6 +84,8 @@ internal fun SidebarRemoteWorkspaceSection( onOpenActions: (RemoteSidebarSessionRow, IntRect) -> Unit, onCreateInWorkspace: (String, String?, String?, String) -> Unit, onOpenWorkspace: (String) -> Unit, + onExpandWorkspace: (String, String?, String?) -> Unit, + onRetryWorkspaceSessions: (String, String?, String?) -> Unit, onAddWorkspace: (() -> Unit)? = null, onWorkspaceTool: (String, String?, Boolean) -> Unit, ) { @@ -90,7 +96,7 @@ internal fun SidebarRemoteWorkspaceSection( var visibleDeviceCount by rememberSaveable { mutableStateOf(DEVICES_PER_BATCH) } val workspacePanels = androidx.compose.runtime.saveable.rememberSaveableStateHolder() - Column(modifier = Modifier.fillMaxWidth().padding(top = 18.dp)) { + Column(modifier = Modifier.fillMaxWidth().padding(top = 10.dp)) { Row( modifier = Modifier.fillMaxWidth().height(38.dp), verticalAlignment = Alignment.CenterVertically, @@ -147,6 +153,7 @@ internal fun SidebarRemoteWorkspaceSection( deviceKey = deviceName, remoteState = remoteState as? RemoteSessionUiState.Ready, workspaceState = workspaceState as? RemoteWorkspaceUiState.Ready, + workspaceDirectory = workspaceDirectory, selectedSessionId = selectedSessionId, onConnect = onConnect, onRetry = onRetryActive, @@ -155,6 +162,8 @@ internal fun SidebarRemoteWorkspaceSection( canActOnSessions = true, onCreateInWorkspace = onCreateInWorkspace, onOpenWorkspace = onOpenWorkspace, + onExpandWorkspace = onExpandWorkspace, + onRetryWorkspaceSessions = onRetryWorkspaceSessions, onAddWorkspace = onAddWorkspace, onWorkspaceTool = onWorkspaceTool, ) @@ -195,6 +204,7 @@ internal fun SidebarRemoteWorkspaceSection( deviceKey = id, remoteState = remoteState as? RemoteSessionUiState.Ready, workspaceState = workspaceState as? RemoteWorkspaceUiState.Ready, + workspaceDirectory = workspaceDirectory, selectedSessionId = selectedSessionId, onConnect = onConnect, onRetry = onRetryActive, @@ -203,6 +213,8 @@ internal fun SidebarRemoteWorkspaceSection( canActOnSessions = true, onCreateInWorkspace = onCreateInWorkspace, onOpenWorkspace = onOpenWorkspace, + onExpandWorkspace = onExpandWorkspace, + onRetryWorkspaceSessions = onRetryWorkspaceSessions, onAddWorkspace = onAddWorkspace, onWorkspaceTool = onWorkspaceTool, ) @@ -221,6 +233,7 @@ private fun SidebarActiveDeviceBody( deviceKey: String, remoteState: RemoteSessionUiState.Ready?, workspaceState: RemoteWorkspaceUiState.Ready?, + workspaceDirectory: WorkspaceSessionDirectoryUiState, selectedSessionId: String?, onConnect: () -> Unit, onRetry: () -> Unit, @@ -229,15 +242,24 @@ private fun SidebarActiveDeviceBody( canActOnSessions: Boolean, onCreateInWorkspace: (String, String?, String?, String) -> Unit, onOpenWorkspace: (String) -> Unit, + onExpandWorkspace: (String, String?, String?) -> Unit, + onRetryWorkspaceSessions: (String, String?, String?) -> Unit, onAddWorkspace: (() -> Unit)? = null, onWorkspaceTool: (String, String?, Boolean) -> Unit, ) { val readyWorkspace = workspaceState val readySessions = remoteState val busy = remoteState?.busy == true - val entries = remember(readyWorkspace, readySessions) { - com.openbitfun.mobile.core.feature.shell.RemoteSidebarPresentation.workspaces(readyWorkspace, readySessions) + val entries = remember(readyWorkspace, readySessions, workspaceDirectory) { + com.openbitfun.mobile.core.feature.shell.RemoteSidebarPresentation.workspacesWithDirectory( + readyWorkspace, readySessions, workspaceDirectory, + ) } + // A branch is open when the reader opened it, or when it is the selected + // workspace and they have not closed it. Everything else starts closed: + // each branch costs its own `list_sessions`, so they are fetched on + // disclosure rather than all at once, the way HarmonyOS already does it. + var expandedPaths by rememberSaveable(deviceKey) { mutableStateOf(emptyList()) } var collapsedPaths by rememberSaveable(deviceKey) { mutableStateOf(emptyList()) } var expandedSessionPaths by rememberSaveable(deviceKey) { mutableStateOf(emptyList()) } var visibleWorkspaceCount by rememberSaveable(deviceKey) { @@ -289,7 +311,12 @@ private fun SidebarActiveDeviceBody( val path = entry.path val identityKey = entry.key val workspaceSessions = entry.sessions - val collapsed = identityKey in collapsedPaths + val expanded = if (entry.selected) { + identityKey !in collapsedPaths + } else { + identityKey in expandedPaths + } + val collapsed = !expanded Column(modifier = Modifier.fillMaxWidth().padding(bottom = 6.dp)) { Row( modifier = Modifier @@ -299,10 +326,22 @@ private fun SidebarActiveDeviceBody( .combinedClickable( role = Role.Button, onClick = { - collapsedPaths = if (collapsed) { - collapsedPaths - identityKey + if (expanded) { + if (entry.selected) { + collapsedPaths = collapsedPaths + identityKey + } else { + expandedPaths = expandedPaths - identityKey + } } else { - collapsedPaths + identityKey + if (entry.selected) { + collapsedPaths = collapsedPaths - identityKey + } else { + expandedPaths = expandedPaths + identityKey + } + // `list_sessions` answers per workspace, so + // this branch's rows only exist once asked + // for. The store ignores a repeat. + onExpandWorkspace(path, entry.remoteConnectionId, entry.remoteSshHost) } }, onLongClick = { onOpenWorkspace(path) }, @@ -359,7 +398,24 @@ private fun SidebarActiveDeviceBody( ) } - if (!collapsed) { + if (expanded) { + when (entry.load) { + RemoteSidebarWorkspaceLoad.IDLE, + RemoteSidebarWorkspaceLoad.LOADING, + -> if (workspaceSessions.isEmpty()) DeviceLoadingRow(startPadding = 26.dp) + RemoteSidebarWorkspaceLoad.FAILED -> WorkspaceFailedRow { + onRetryWorkspaceSessions(path, entry.remoteConnectionId, entry.remoteSshHost) + } + RemoteSidebarWorkspaceLoad.READY -> if (workspaceSessions.isEmpty()) { + Text( + stringResource(R.string.sidebar_workspace_empty_sessions), + fontSize = 13.sp, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.fillMaxWidth() + .padding(start = 26.dp, top = 6.dp, bottom = 6.dp), + ) + } + } val limit = if (identityKey in expandedSessionPaths) { workspaceSessions.size } else { @@ -398,9 +454,9 @@ private fun SidebarActiveDeviceBody( } @Composable -private fun DeviceLoadingRow() { +private fun DeviceLoadingRow(startPadding: Dp = 10.dp) { Row( - modifier = Modifier.fillMaxWidth().height(40.dp).padding(start = 10.dp, end = 10.dp), + modifier = Modifier.fillMaxWidth().height(40.dp).padding(start = startPadding, end = 10.dp), horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically, ) { @@ -417,6 +473,32 @@ private fun DeviceLoadingRow() { } } +/** One workspace whose own session list could not be read, indented under it. */ +@Composable +private fun WorkspaceFailedRow(onRetry: () -> Unit) { + val retryLabel = stringResource(R.string.sidebar_device_retry) + Row( + modifier = Modifier.fillMaxWidth().height(40.dp).padding(start = 26.dp, end = 10.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Text( + stringResource(R.string.sidebar_workspace_load_failed), + fontSize = 13.sp, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.weight(1f), + ) + Text( + retryLabel, + fontSize = 13.sp, + color = MaterialTheme.colorScheme.onSurface, + modifier = Modifier + .clickable(role = Role.Button, onClick = onRetry) + .semantics { contentDescription = retryLabel }, + ) + } +} + @Composable private fun DeviceFailedRow(onRetry: () -> Unit) { val retryLabel = stringResource(R.string.sidebar_device_retry) diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/theme/generated/MobileDesignTokens.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/theme/generated/MobileDesignTokens.kt index 571d1e80b3..aba82c2ece 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/theme/generated/MobileDesignTokens.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/theme/generated/MobileDesignTokens.kt @@ -13,11 +13,13 @@ internal object MobileDesignColors { val Transparent = Color(0x00000000) val PageBg = Color(0xFFFFFFFF) val PageBgFade = Color(0x00FFFFFF) + val PageBgOverlay = Color(0xB8FFFFFF) val Ink = Color(0xFF171717) val Muted = Color(0xFF706F6A) val Subtle = Color(0xFFA5A39B) val Line = Color(0xFFE9E7E2) val Card = Color(0xFFFFFFFF) + val CardOverlay = Color(0xCCFFFFFF) val Accent = Color(0xFF111111) val FileLink = Color(0xFF2563EB) val PrimaryAction = Color(0xFF111111) @@ -65,11 +67,13 @@ internal object MobileDesignColors { val Transparent = Color(0x00000000) val PageBg = Color(0xFF151514) val PageBgFade = Color(0x00151514) + val PageBgOverlay = Color(0xB8151514) val Ink = Color(0xFFF4F3EF) val Muted = Color(0xFFAAA8A0) val Subtle = Color(0xFF77756E) val Line = Color(0xFF363531) val Card = Color(0xFF252522) + val CardOverlay = Color(0xCC252522) val Accent = Color(0xFF5B5954) val FileLink = Color(0xFF60A5FA) val PrimaryAction = Color(0xFF454540) @@ -140,6 +144,8 @@ internal object MobileDesignGeometry { val ContentGutter = 16.dp val ConnectionStripHeight = 48.dp val TimelineTopPadding = 22.dp + val ConversationOverlaySideInset = 16.dp + val ConversationEdgeFadeHeight = 28.dp val MessageSpacing = 18.dp val MessageBubbleMaxWidth = 276.dp val MessageBubbleHorizontalPadding = 14.dp diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/viewmodel/AccountViewModel.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/viewmodel/AccountViewModel.kt index f3dab2e143..e75f31b8ec 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/viewmodel/AccountViewModel.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/viewmodel/AccountViewModel.kt @@ -16,6 +16,7 @@ import com.openbitfun.mobile.core.feature.connection.ConnectionPhase import com.openbitfun.mobile.core.feature.session.RemoteSessionIntent import com.openbitfun.mobile.core.feature.session.RemoteSessionStore import com.openbitfun.mobile.core.feature.session.RemoteSessionUiState +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryUiState import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceIntent import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceStore import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceUiState @@ -50,10 +51,15 @@ internal class AccountViewModel(application: Application) : AndroidViewModel(app val connectionPhase: StateFlow = _connectionPhase.asStateFlow() private val _workspaceState = MutableStateFlow(RemoteWorkspaceUiState.Idle) val workspaceState: StateFlow = _workspaceState.asStateFlow() + private val _workspaceDirectory = MutableStateFlow(WorkspaceSessionDirectoryUiState(emptyList())) + + /** Per-workspace session lists, filled in as the sidebar discloses each branch. */ + val workspaceDirectory: StateFlow = _workspaceDirectory.asStateFlow() private var remoteStore: RemoteSessionStore? = null private var workspaceStore: RemoteWorkspaceStore? = null private var remoteJob: Job? = null private var connectionJob: Job? = null + private var directoryJob: Job? = null private var workspaceJob: Job? = null private var activeTarget: String? = null @@ -104,6 +110,8 @@ internal class AccountViewModel(application: Application) : AndroidViewModel(app remoteJob?.cancel() workspaceJob?.cancel() connectionJob?.cancel() + directoryJob?.cancel() + _workspaceDirectory.value = WorkspaceSessionDirectoryUiState(emptyList()) _remoteState.value = RemoteSessionUiState.Loading _workspaceState.value = RemoteWorkspaceUiState.Loading _connectionPhase.value = ConnectionPhase.IDLE @@ -119,6 +127,7 @@ internal class AccountViewModel(application: Application) : AndroidViewModel(app remoteJob?.cancel() connectionJob?.cancel() workspaceJob?.cancel() + directoryJob?.cancel() remoteStore?.dispatch(RemoteSessionIntent.Stop) workspaceStore?.dispatch(RemoteWorkspaceIntent.Stop) remoteStore = null @@ -126,6 +135,7 @@ internal class AccountViewModel(application: Application) : AndroidViewModel(app _remoteState.value = RemoteSessionUiState.Idle _connectionPhase.value = ConnectionPhase.IDLE _workspaceState.value = RemoteWorkspaceUiState.Idle + _workspaceDirectory.value = WorkspaceSessionDirectoryUiState(emptyList()) completionNotifier.reset() activeTarget = target if (target == null) return @@ -138,6 +148,9 @@ internal class AccountViewModel(application: Application) : AndroidViewModel(app connectionJob = viewModelScope.launch { created.connectionPhase.collect { _connectionPhase.value = it } } + directoryJob = viewModelScope.launch { + created.workspaceDirectory.collect { _workspaceDirectory.value = it } + } created.dispatch(RemoteSessionIntent.Load) created.dispatch(RemoteSessionIntent.SetForeground(foreground)) } diff --git a/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml b/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml index 1cad722afc..002c36bf48 100644 --- a/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml +++ b/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml @@ -322,6 +322,8 @@ 连接电脑 添加连接 这台电脑还没有工作区 + 此工作区暂无会话 + 这个工作区暂时无法读取 还有 %1$d 个工作区 还有 %1$d 台设备 在此工作区新建 diff --git a/src/apps/mobile/android/app/src/main/res/values/strings.xml b/src/apps/mobile/android/app/src/main/res/values/strings.xml index e6817f7081..649558e8ab 100644 --- a/src/apps/mobile/android/app/src/main/res/values/strings.xml +++ b/src/apps/mobile/android/app/src/main/res/values/strings.xml @@ -335,6 +335,8 @@ Connect a computer Add connection This computer has no workspaces yet + No chats in this workspace + This workspace cannot be read right now %1$d more workspaces %1$d more devices New in this workspace diff --git a/src/apps/mobile/design-system/components/mobile-components.json b/src/apps/mobile/design-system/components/mobile-components.json index ed44fc1916..e5b953b5c4 100644 --- a/src/apps/mobile/design-system/components/mobile-components.json +++ b/src/apps/mobile/design-system/components/mobile-components.json @@ -64,6 +64,27 @@ "tokens": ["body_large", "message_bubble_max_width", "message_bubble_horizontal_padding", "message_bubble_vertical_padding", "message_bubble_radius", "message_spacing", "timeline_top_padding"], "platformNotes": "The maximum width is the 276-unit compact HarmonyOS baseline; wide layouts may use a separate responsive policy." }, + "conversation_overlay": { + "purpose": "Lets the transcript own the whole pane while the header and composer stay reachable above it.", + "anatomy": [ + "top_overlay", + "top_edge_fade", + "timeline", + "bottom_overlay_fade" + ], + "states": [ + "at_rest", + "scrolled", + "composer_expanded", + "mailbox_open", + "keyboard_open" + ], + "tokens": [ + "conversation_overlay_side_inset", + "conversation_edge_fade_height" + ], + "platformNotes": "The timeline fills the pane and scrolls beneath both overlays; its content insets are measured from the live overlay heights rather than fixed, and are applied with each platform's content-inset primitive \u2014 Compose contentPadding, SwiftUI safeAreaInset, ArkUI contentStartOffset/contentEndOffset \u2014 never with padding, which would shrink the viewport and stop the transcript at the overlay instead of letting it run underneath, because the composer grows with multiline text and attachments and the mailbox appears without warning. The top layer is an opaque band with a short fade carrying it into the transcript; the bottom layer has no strip of its own \u2014 its whole background is the fade, because the transcript runs behind the composer and a short strip would leave a line of text crisp beside the pill after it had already faded out higher up. Fades never take touches. Blur is the platform's own: iOS ultraThinMaterial, HarmonyOS backgroundBlurStyle. Compose has no first-party backdrop blur, so Android uses a high-opacity band instead and reads the same at a glance." + }, "composer_bar": { "purpose": "Collects the next instruction and exposes one unambiguous primary action.", "anatomy": ["attachment_action", "text_input", "model_control", "primary_action"], @@ -174,7 +195,7 @@ "anatomy": ["outlined_apps_glyph", "leading_aligned_label", "full_row_touch_target"], "states": ["idle", "pressed", "focused", "signed_out", "connected"], "tokens": ["ink", "body_large", "control_touch_size"], - "platformNotes": "Use a quiet 48vp icon-and-label row in compact and wide sidebars, with 16vp medium-weight navigation text, a 24vp icon slot and a 14vp gap. HarmonyOS uses the native outlined sys.symbol.square_grid_2x2 at 24vp, centered in its slot and tinted with the semantic ink color. Render the catalog inside the shell content pane so selecting it slides the right pane left as the drawer closes, using the existing 250ms ease-out shell motion. Sidebar session selection uses the same shell motion without a second route-push animation. Do not promote this navigation destination to a primary action on remote home." + "platformNotes": "Use a quiet 48vp icon-and-label row in compact and wide sidebars, with 16vp medium-weight navigation text, a 24vp icon slot and a 14vp gap. In the sidebar the row keeps 4vp above and 8vp below itself, and the device section header keeps 10vp above it, inside the shared 76vp header frame: the header control sits 20vp above the row and the Devices label 18vp below it. A platform whose header control is not 44vp inherits that difference instead of adding a second gap, and no column-level spacing is layered around the row. HarmonyOS uses the native outlined sys.symbol.square_grid_2x2 at 24vp, centered in its slot and tinted with the semantic ink color. Render the catalog inside the shell content pane so selecting it slides the right pane left as the drawer closes, using the existing 250ms ease-out shell motion. Sidebar session selection uses the same shell motion without a second route-push animation. Do not promote this navigation destination to a primary action on remote home." }, "miniapp_catalog": { "purpose": "Presents bundled apps with large visual previews that open each app directly.", diff --git a/src/apps/mobile/design-system/preview/generated/mobile-design-data.js b/src/apps/mobile/design-system/preview/generated/mobile-design-data.js index 09d1795ce8..fff122e026 100644 --- a/src/apps/mobile/design-system/preview/generated/mobile-design-data.js +++ b/src/apps/mobile/design-system/preview/generated/mobile-design-data.js @@ -22,6 +22,10 @@ export const mobileTokens = { "light": "#00FFFFFF", "dark": "#00151514" }, + "page_bg_overlay": { + "light": "#B8FFFFFF", + "dark": "#B8151514" + }, "ink": { "light": "#171717", "dark": "#F4F3EF" @@ -42,6 +46,10 @@ export const mobileTokens = { "light": "#FFFFFF", "dark": "#252522" }, + "card_overlay": { + "light": "#CCFFFFFF", + "dark": "#CC252522" + }, "accent": { "light": "#111111", "dark": "#5B5954" @@ -297,6 +305,8 @@ export const mobileTokens = { "content_gutter": 16, "connection_strip_height": 48, "timeline_top_padding": 22, + "conversation_overlay_side_inset": 16, + "conversation_edge_fade_height": 28, "message_spacing": 18, "message_bubble_max_width": 276, "message_bubble_horizontal_padding": 14, @@ -600,6 +610,27 @@ export const mobileComponents = { ], "platformNotes": "The maximum width is the 276-unit compact HarmonyOS baseline; wide layouts may use a separate responsive policy." }, + "conversation_overlay": { + "purpose": "Lets the transcript own the whole pane while the header and composer stay reachable above it.", + "anatomy": [ + "top_overlay", + "top_edge_fade", + "timeline", + "bottom_overlay_fade" + ], + "states": [ + "at_rest", + "scrolled", + "composer_expanded", + "mailbox_open", + "keyboard_open" + ], + "tokens": [ + "conversation_overlay_side_inset", + "conversation_edge_fade_height" + ], + "platformNotes": "The timeline fills the pane and scrolls beneath both overlays; its content insets are measured from the live overlay heights rather than fixed, and are applied with each platform's content-inset primitive — Compose contentPadding, SwiftUI safeAreaInset, ArkUI contentStartOffset/contentEndOffset — never with padding, which would shrink the viewport and stop the transcript at the overlay instead of letting it run underneath, because the composer grows with multiline text and attachments and the mailbox appears without warning. The top layer is an opaque band with a short fade carrying it into the transcript; the bottom layer has no strip of its own — its whole background is the fade, because the transcript runs behind the composer and a short strip would leave a line of text crisp beside the pill after it had already faded out higher up. Fades never take touches. Blur is the platform's own: iOS ultraThinMaterial, HarmonyOS backgroundBlurStyle. Compose has no first-party backdrop blur, so Android uses a high-opacity band instead and reads the same at a glance." + }, "composer_bar": { "purpose": "Collects the next instruction and exposes one unambiguous primary action.", "anatomy": [ @@ -1044,7 +1075,7 @@ export const mobileComponents = { "body_large", "control_touch_size" ], - "platformNotes": "Use a quiet 48vp icon-and-label row in compact and wide sidebars, with 16vp medium-weight navigation text, a 24vp icon slot and a 14vp gap. HarmonyOS uses the native outlined sys.symbol.square_grid_2x2 at 24vp, centered in its slot and tinted with the semantic ink color. Render the catalog inside the shell content pane so selecting it slides the right pane left as the drawer closes, using the existing 250ms ease-out shell motion. Sidebar session selection uses the same shell motion without a second route-push animation. Do not promote this navigation destination to a primary action on remote home." + "platformNotes": "Use a quiet 48vp icon-and-label row in compact and wide sidebars, with 16vp medium-weight navigation text, a 24vp icon slot and a 14vp gap. In the sidebar the row keeps 4vp above and 8vp below itself, and the device section header keeps 10vp above it, inside the shared 76vp header frame: the header control sits 20vp above the row and the Devices label 18vp below it. A platform whose header control is not 44vp inherits that difference instead of adding a second gap, and no column-level spacing is layered around the row. HarmonyOS uses the native outlined sys.symbol.square_grid_2x2 at 24vp, centered in its slot and tinted with the semantic ink color. Render the catalog inside the shell content pane so selecting it slides the right pane left as the drawer closes, using the existing 250ms ease-out shell motion. Sidebar session selection uses the same shell motion without a second route-push animation. Do not promote this navigation destination to a primary action on remote home." }, "miniapp_catalog": { "purpose": "Presents bundled apps with large visual previews that open each app directly.", diff --git a/src/apps/mobile/design-system/tokens/mobile-tokens.json b/src/apps/mobile/design-system/tokens/mobile-tokens.json index 9bdf8d1d2c..46d6916e04 100644 --- a/src/apps/mobile/design-system/tokens/mobile-tokens.json +++ b/src/apps/mobile/design-system/tokens/mobile-tokens.json @@ -21,6 +21,10 @@ "light": "#00FFFFFF", "dark": "#00151514" }, + "page_bg_overlay": { + "light": "#B8FFFFFF", + "dark": "#B8151514" + }, "ink": { "light": "#171717", "dark": "#F4F3EF" @@ -41,6 +45,10 @@ "light": "#FFFFFF", "dark": "#252522" }, + "card_overlay": { + "light": "#CCFFFFFF", + "dark": "#CC252522" + }, "accent": { "light": "#111111", "dark": "#5B5954" @@ -296,6 +304,8 @@ "content_gutter": 16, "connection_strip_height": 48, "timeline_top_padding": 22, + "conversation_overlay_side_inset": 16, + "conversation_edge_fade_height": 28, "message_spacing": 18, "message_bubble_max_width": 276, "message_bubble_horizontal_padding": 14, diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/generated/MobileDesignTokens.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/generated/MobileDesignTokens.ets index d006f83c3e..fd3a78070e 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/generated/MobileDesignTokens.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/generated/MobileDesignTokens.ets @@ -15,11 +15,13 @@ export class MobileDesignColors { static readonly transparent: MobileColorPair = new MobileColorPair("#00000000", "#00000000"); static readonly pageBg: MobileColorPair = new MobileColorPair("#FFFFFF", "#151514"); static readonly pageBgFade: MobileColorPair = new MobileColorPair("#00FFFFFF", "#00151514"); + static readonly pageBgOverlay: MobileColorPair = new MobileColorPair("#B8FFFFFF", "#B8151514"); static readonly ink: MobileColorPair = new MobileColorPair("#171717", "#F4F3EF"); static readonly muted: MobileColorPair = new MobileColorPair("#706F6A", "#AAA8A0"); static readonly subtle: MobileColorPair = new MobileColorPair("#A5A39B", "#77756E"); static readonly line: MobileColorPair = new MobileColorPair("#E9E7E2", "#363531"); static readonly card: MobileColorPair = new MobileColorPair("#FFFFFF", "#252522"); + static readonly cardOverlay: MobileColorPair = new MobileColorPair("#CCFFFFFF", "#CC252522"); static readonly accent: MobileColorPair = new MobileColorPair("#111111", "#5B5954"); static readonly fileLink: MobileColorPair = new MobileColorPair("#2563EB", "#60A5FA"); static readonly primaryAction: MobileColorPair = new MobileColorPair("#111111", "#454540"); @@ -101,6 +103,8 @@ export class MobileDesignGeometry { static readonly contentGutter: number = 16; static readonly connectionStripHeight: number = 48; static readonly timelineTopPadding: number = 22; + static readonly conversationOverlaySideInset: number = 16; + static readonly conversationEdgeFadeHeight: number = 28; static readonly messageSpacing: number = 18; static readonly messageBubbleMaxWidth: number = 276; static readonly messageBubbleHorizontalPadding: number = 14; diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/EnUsMessages.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/EnUsMessages.ets index 5674f18741..228c89b3e1 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/EnUsMessages.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/EnUsMessages.ets @@ -514,6 +514,7 @@ export const EN_US_MESSAGES: [string, string][] = [ ['chat.voiceListeningPlaceholder', 'Listening, please speak...'], ['chat.queuedAfterRunningTurn', 'Queued. It will run after the current task finishes.'], ['chat.image', 'Image'], + ['chat.imageUnavailable', 'Image not synced to this device'], ['chat.thinkingRunning', 'Running'], ['chat.thinkingDone', 'Done'], ['chat.thinkingInProgress', 'Thinking'], diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/ZhCnMessages.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/ZhCnMessages.ets index f32183e110..471908f8f5 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/ZhCnMessages.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/i18n/ZhCnMessages.ets @@ -514,6 +514,7 @@ export const ZH_CN_MESSAGES: [string, string][] = [ ['chat.voiceListeningPlaceholder', '正在听,请说话...'], ['chat.queuedAfterRunningTurn', '已排队,当前任务结束后执行'], ['chat.image', '图片'], + ['chat.imageUnavailable', '图片未同步到此设备'], ['chat.thinkingRunning', '运行中'], ['chat.thinkingDone', '已完成'], ['chat.thinkingInProgress', '正在思考'], diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/model/RemoteModels.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/model/RemoteModels.ets index a21ec0f92e..00f996d9e3 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/model/RemoteModels.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/model/RemoteModels.ets @@ -54,6 +54,9 @@ export interface RemoteSession { workspaceIdentity?: RemoteWorkspaceIdentity; pinned?: boolean; titleGenerated?: boolean; + /** Set when the session hangs off another one (btw/review/miniapp/subagent). */ + parentSessionId?: string; + relationshipKind?: string; /** Stamped by the phone after mapping; the wire payload has no device id. */ deviceId?: string; } @@ -255,6 +258,8 @@ export interface SessionItemResponse { message_count?: number; workspace_path?: string; workspace_name?: string; + parent_session_id?: string; + relationship_kind?: string; } export interface SessionListResponse extends CommandStatusResponse { diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/AppSidebar.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/AppSidebar.ets index 8a133e371f..2ff40fd2c4 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/AppSidebar.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/AppSidebar.ets @@ -75,7 +75,7 @@ export struct AppSidebar { } .width('100%').height(48).padding({ left: 4, right: 8 }) .backgroundColor(TRANSPARENT).borderRadius(12) - .margin({ top: 16, bottom: 16 }).stateEffect(true) + .margin({ top: 4, bottom: 8 }).stateEffect(true) .accessibilityText(RemoteI18n.t('miniapp.title')) .onClick(() => this.onOpenMiniApps()) if (this.showWorkspaceSection) { diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatMessageContent.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatMessageContent.ets index 5d40586747..e92db7926b 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatMessageContent.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatMessageContent.ets @@ -9,7 +9,7 @@ import { import { FileReferenceCard } from './FileReferenceCard'; import { MarkdownContent } from './MarkdownContent'; import { StreamingMarkdownContent } from './StreamingMarkdownContent'; -import { LINE } from './Theme'; +import { LINE, MEDIA_BACKGROUND, MUTED } from './Theme'; @ComponentV2 export struct MessageImageGallery { @@ -19,16 +19,45 @@ export struct MessageImageGallery { build() { Flex({ direction: FlexDirection.Row, wrap: FlexWrap.Wrap }) { ForEach(this.images, (image: ConversationUiImage, index: number) => { - Image(image.data_url) - .width(this.userStyle ? 112 : 92) - .height(this.userStyle ? 112 : 92) - .objectFit(ImageFit.Cover) - .borderRadius(this.userStyle ? 12 : 14) - .border({ width: 1, color: LINE }) - .margin({ - right: this.userStyle ? (index % 2 === 0 && this.images.length > 1 ? 8 : 0) : 8, - bottom: this.userStyle ? (index < this.images.length - 2 ? 8 : 0) : 8 - }) + Stack() { + if (ConversationMessageRenderPolicy.hasRenderableImagePixels(image)) { + Image(image.data_url) + .width('100%') + .height('100%') + .objectFit(ImageFit.Cover) + } else { + // Without pixels an Image() draws an unexplained blank box, so name + // the attachment instead and say why it is not here. + Column() { + Text(image.name || RemoteI18n.t('chat.image')) + .fontSize(11) + .fontColor(MUTED) + .maxLines(2) + .textOverflow({ overflow: TextOverflow.Ellipsis }) + .textAlign(TextAlign.Center) + Text(RemoteI18n.t('chat.imageUnavailable')) + .fontSize(10) + .fontColor(MUTED) + .maxLines(2) + .textOverflow({ overflow: TextOverflow.Ellipsis }) + .textAlign(TextAlign.Center) + .margin({ top: 4 }) + } + .width('100%') + .padding(8) + .justifyContent(FlexAlign.Center) + } + } + .width(this.userStyle ? 112 : 92) + .height(this.userStyle ? 112 : 92) + .backgroundColor(MEDIA_BACKGROUND) + .borderRadius(this.userStyle ? 12 : 14) + .border({ width: 1, color: LINE }) + .clip(true) + .margin({ + right: this.userStyle ? (index % 2 === 0 && this.images.length > 1 ? 8 : 0) : 8, + bottom: this.userStyle ? (index < this.images.length - 2 ? 8 : 0) : 8 + }) }, (image: ConversationUiImage, index: number) => `${image.name}-${index}`) } .width(this.userStyle ? (this.images.length > 1 ? 232 : 112) : '100%') diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatTimeline.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatTimeline.ets index 07e486bd07..741bd6b8c3 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatTimeline.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ChatTimeline.ets @@ -138,6 +138,13 @@ export struct ChatTimeline { @Param activeFilePreviewPath: string = ''; @Param activeFilePreviewLoading: boolean = false; @Param maxContentWidth: number = 0; + /** + * How much of the pane the floating header and composer cover. The list runs + * the full height behind them, so without these the first and last messages + * would sit under a capsule and never come out from under it. + */ + @Param topInset: number = 0; + @Param bottomInset: number = 0; @Event onLoadOlder: () => void = () => {}; @Event onApproveTool: (toolId: string, updatedInput?: Object) => void = (_toolId: string, _updatedInput?: Object) => {}; @@ -316,9 +323,15 @@ export struct ChatTimeline { .padding({ left: MobileDesignGeometry.contentGutter, right: MobileDesignGeometry.contentGutter, - top: 0, - bottom: 12 + bottom: this.bottomInset > 0 ? 0 : 12 }) + // Content offsets, not padding: padding shrinks the viewport, so the + // transcript would stop at the floating header and composer instead of + // running underneath them. These reserve the space at rest and still let + // the content scroll through — ArkUI's equivalent of Compose's + // contentPadding and SwiftUI's safeAreaInset. + .contentStartOffset(this.topInset) + .contentEndOffset(this.bottomInset) // A streaming reply grows in place rather than arriving as a new item, so // the bottom of the list has to be the anchor — otherwise the text the // agent is writing right now renders below the fold and never comes back. @@ -354,7 +367,8 @@ export struct ChatTimeline { .backgroundColor(CARD) .borderRadius(21) .shadow({ radius: 14, color: SHADOW_SUBTLE, offsetY: 5 }) - .margin({ bottom: 4 }) + // Rides above the floating composer rather than hiding behind it. + .margin({ bottom: this.bottomInset + 4 }) .onClick(() => { this.stickToBottom = true; this.userScrollActive = false; diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ComposerBar.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ComposerBar.ets index 7c0070b4f3..d6ae0b8a28 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ComposerBar.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ComposerBar.ets @@ -12,6 +12,7 @@ import { ConversationModelPresentationPolicy } from '../policy/ConversationModel import { TemplateIcon } from './TemplateIcon'; import { CARD, + CARD_OVERLAY, INK, LINE, MEDIA_SCRIM, @@ -116,8 +117,10 @@ export struct ComposerBar { .constraintSize({ maxWidth: this.presentation === ComposerPresentation.Floating ? 760 : 10000 }) .alignSelf(ItemAlign.Center) .padding({ - left: this.presentation === ComposerPresentation.Floating ? 24 : COMPOSER_COMPACT_HORIZONTAL_GUTTER, - right: this.presentation === ComposerPresentation.Floating ? 24 : COMPOSER_COMPACT_HORIZONTAL_GUTTER, + left: this.presentation === ComposerPresentation.Floating + ? 24 : MobileDesignGeometry.conversationOverlaySideInset, + right: this.presentation === ComposerPresentation.Floating + ? 24 : MobileDesignGeometry.conversationOverlaySideInset, top: 8, bottom: this.presentation === ComposerPresentation.Floating ? 16 : 12 }) @@ -190,7 +193,12 @@ export struct ComposerBar { top: this.isComposerExpanded() ? 4 : 0, bottom: this.isComposerExpanded() ? 2 : 0 }) - .backgroundColor(CARD) + // The transcript now runs underneath the pill, so the fill is translucent + // and the blur does the separating. The colour is the floor under it: on a + // device that does not land the blur the pill is still opaque enough to + // read against whatever text is passing behind it. + .backgroundColor(CARD_OVERLAY) + .backgroundBlurStyle(BlurStyle.Thin) .border({ width: this.presentation === ComposerPresentation.Compact ? 0.5 : 1, color: LINE }) .borderRadius(this.composerRadius()) .shadow({ diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationHeader.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationHeader.ets index e392009197..59c83ec861 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationHeader.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationHeader.ets @@ -28,6 +28,13 @@ export struct ConversationHeader { @Param showActions: boolean = false; @Param showSidebarButton: boolean = true; @Param showBackButton: boolean = false; + /** + * Set when something behind the header is meant to show through — a + * conversation whose transcript scrolls underneath it. The caller owns the + * fill and the blur in that case; painting an opaque page colour here would + * cover both. + */ + @Param translucent: boolean = false; @Param showSidebarRestoreButton: boolean = false; @Param showActionsMenu: boolean = false; @BuilderParam actionsMenu: () => void = this.EmptyBuilder; @@ -48,7 +55,7 @@ export struct ConversationHeader { } } .width('100%') - .backgroundColor(PAGE_BG) + .backgroundColor(this.translucent ? TRANSPARENT : PAGE_BG) } @Builder @@ -67,7 +74,7 @@ export struct ConversationHeader { top: 8, bottom: 8 }) - .backgroundColor(PAGE_BG) + .backgroundColor(this.translucent ? TRANSPARENT : PAGE_BG) } @Builder diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationView.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationView.ets index e519044e32..3307a1eebe 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationView.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/ConversationView.ets @@ -1,6 +1,6 @@ import { InteractionMailboxPanel } from './InteractionMailboxPanel'; import { InteractionMailboxAction } from '../../model/InteractionMailbox'; -import { MobileDesignTypography } from '../../generated/MobileDesignTokens'; +import { MobileDesignGeometry, MobileDesignTypography } from '../../generated/MobileDesignTokens'; import { KeyboardAvoidMode } from '@kit.ArkUI'; import { RemoteI18n } from '../../i18n/RemoteI18n'; import { ChatTimelineItem, ObservableChatTimelineItem } from '../../model/ChatTimelineModels'; @@ -31,6 +31,8 @@ import { LINE, MUTED, PAGE_BG, + PAGE_BG_FADE, + PAGE_BG_OVERLAY, SHADOW_SUBTLE, SOFT, STATUS_DANGER, @@ -45,6 +47,15 @@ export struct ConversationView { @Event onIntent: (intent: ConversationIntent) => void = (_intent: ConversationIntent) => {}; @Event onRestoreSidebar: () => void = () => {}; @Local showHeaderActions: boolean = false; + /** + * How much of the pane the two floating layers cover, measured rather than + * assumed: the composer grows with multiline text and attachments, and the + * mailbox appears without warning. The transcript runs the full height behind + * them and borrows these as its content insets, so a stale number would leave + * the first or last message stuck under a capsule. + */ + @Local topInset: number = 0; + @Local bottomInset: number = 0; private previousKeyboardAvoidMode: KeyboardAvoidMode = KeyboardAvoidMode.OFFSET; aboutToAppear(): void { @@ -59,52 +70,152 @@ export struct ConversationView { } build() { - Stack() { - Column() { - this.Header() - this.Mailbox() - if (this.hoverOperateMode) { - this.HoverBody() - } else { - this.StandardBody() - } + Stack({ alignContent: Alignment.Top }) { + if (this.hoverOperateMode) { + this.HalfFoldedLayout() + } else { + this.FloatingLayout() } - .width('100%').height('100%').backgroundColor(PAGE_BG) } .width('100%') .height('100%') .backgroundColor(PAGE_BG) } + /** + * The transcript owns the whole pane; the header and the composer float over + * it and the transcript scrolls underneath both, fading out as it reaches + * either edge. Stack paints in order, so the body goes down first and the two + * overlay layers land on top of it. + */ @Builder - private StandardBody() { + private FloatingLayout() { Column() { this.ConversationBody() - this.Composer() } - .layoutWeight(1) + .width('100%') + .height('100%') + + // A full-height layer that pins its content to the bottom. Stack aligns + // every child the same way, so the bottom layer carries its own alignment + // rather than fighting `alignContent`. It is see-through to touches above + // the composer, otherwise it would swallow every tap on the transcript. + Column() { + this.BottomOverlay() + } + .width('100%') + .height('100%') + .justifyContent(FlexAlign.End) + .hitTestBehavior(HitTestMode.Transparent) + + Column() { + this.TopOverlay() + } .width('100%') } + /** + * Half-folded on a foldable the screen is physically two panes, and the + * transcript and the composer already have one each. Floating one over the + * other has nothing left to win there, so this layout stays stacked. + */ @Builder - private HoverBody() { - FolderStack() { - Column() { - this.ConversationBody() + private HalfFoldedLayout() { + Column() { + this.Header() + this.Mailbox() + FolderStack() { + Column() { + this.ConversationBody() + } + .width('100%') + .height('100%') + .padding({ bottom: 12 }) + Column() { + this.InlineStatusLine() + this.Composer() + } + .width('100%') } + .alignContent(Alignment.Bottom) + .layoutWeight(1) .width('100%') - .height('100%') - .padding({ bottom: 12 }) + .enableAnimation(true) + .autoHalfFold(true) + } + .width('100%') + .height('100%') + .backgroundColor(PAGE_BG) + } + + @Builder + private TopOverlay() { + Column() { Column() { - this.Composer() + this.Header() + this.Mailbox() } .width('100%') + .backgroundColor(PAGE_BG_OVERLAY) + .backgroundBlurStyle(BlurStyle.Thin) + this.TopEdgeFade() } - .alignContent(Alignment.Bottom) - .layoutWeight(1) .width('100%') - .enableAnimation(true) - .autoHalfFold(true) + .onAreaChange((_oldArea: Area, newArea: Area) => { + this.topInset = newArea.height as number; + }) + } + + @Builder + private BottomOverlay() { + Column() { + this.InlineStatusLine() + this.Composer() + } + .width('100%') + // The fade covers the whole layer, not just a strip above it: the transcript + // runs behind the composer, so anything short of that leaves a line of text + // sitting crisp and legible beside the pill after it has already faded out + // higher up. It stops short of the page colour so the pill's blur still has + // something to work with. + .linearGradient({ + angle: 180, + colors: [[PAGE_BG_FADE, 0.0], [PAGE_BG_OVERLAY, 0.45], [PAGE_BG_OVERLAY, 1.0]] + }) + .onAreaChange((_oldArea: Area, newArea: Area) => { + this.bottomInset = newArea.height as number; + }) + } + + /** + * Carries the header's opaque band down into the transcript. Without it a + * line of text is cut in half at the band's edge and reads as clipped rather + * than as continuing underneath. Never takes a touch: the transcript below + * it is still the thing being pointed at. The bottom layer needs no strip of + * its own — its whole background is the gradient. + */ + @Builder + private TopEdgeFade() { + Column() + .width('100%') + .height(MobileDesignGeometry.conversationEdgeFadeHeight) + .hitTestBehavior(HitTestMode.None) + .linearGradient({ + angle: 180, + colors: [[PAGE_BG, 0.0], [PAGE_BG_FADE, 1.0]] + }) + } + + @Builder + private InlineStatusLine() { + if (this.inlineStatusText.length > 0 && !this.shouldCenterInlineStatus()) { + Text(this.inlineStatusText) + .fontSize(MobileDesignTypography.labelSmall.size) + .lineHeight(MobileDesignTypography.labelSmall.lineHeight) + .fontColor(MUTED) + .width('100%') + .padding({ left: 20, right: 20, bottom: 8 }) + } } @Builder @@ -130,7 +241,7 @@ export struct ConversationView { } .width('100%') .height('100%') - if (this.isLoadingConversation) { + if (this.isLoadingTranscript) { ConversationLoadingState({ maxContentWidth: this.composerPresentation === ComposerPresentation.Floating ? 760 : 0 }) @@ -139,19 +250,15 @@ export struct ConversationView { .backgroundColor(PAGE_BG) } } - .layoutWeight(1) .width('100%') - } - if (this.inlineStatusText.length > 0 && !this.shouldCenterInlineStatus()) { - Text(this.inlineStatusText) - .fontSize(MobileDesignTypography.labelSmall.size).lineHeight(MobileDesignTypography.labelSmall.lineHeight).fontColor(MUTED).width('100%') - .padding({ left: 20, right: 20, bottom: 8 }) + .height('100%') } } @Builder Header() { ConversationHeader({ + translucent: true, title: this.headerTitle(), subtitle: this.headerSubtitle(), titleFallback: this.headerFallback(), @@ -204,6 +311,8 @@ export struct ConversationView { activeFilePreviewPath: this.activeFilePreviewPath, activeFilePreviewLoading: this.activeFilePreviewLoading, maxContentWidth: this.composerPresentation === ComposerPresentation.Floating ? 760 : 0, + topInset: this.hoverOperateMode ? 0 : this.topInset, + bottomInset: this.hoverOperateMode ? 0 : this.bottomInset, onLoadOlder: () => { this.onLoadOlder(); }, @@ -247,7 +356,6 @@ export struct ConversationView { .textAlign(TextAlign.Center) .maxLines(2) } - .layoutWeight(1) .width('100%') .height('100%') .padding({ left: 32, right: 32, bottom: 48 }) @@ -435,6 +543,14 @@ export struct ConversationView { private get composerCapabilities(): ChatComposerCapabilities { return this.state.composerCapabilities; } private get isBusy(): boolean { return this.state.isBusy; } private get isLoadingConversation(): boolean { return this.state.isLoadingConversation; } + /** + * The skeleton stands for a body with nothing in it yet, which is true both + * while the session is being identified and while its transcript is still + * crossing the relay. Only the first of those blanks the header. + */ + private get isLoadingTranscript(): boolean { + return this.state.isLoadingConversation || this.state.isLoadingTranscript; + } private get canStop(): boolean { return this.state.canStop; } private get hasMoreMessages(): boolean { return this.state.hasMoreMessages; } private get timelineItems(): ObservableChatTimelineItem[] { return this.state.timelineItems; } diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/SidebarWorkspaceSection.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/SidebarWorkspaceSection.ets index 10ffdabff2..383bdbc134 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/SidebarWorkspaceSection.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/SidebarWorkspaceSection.ets @@ -106,7 +106,7 @@ export struct SidebarWorkspaceSection { } .width('100%') .alignItems(HorizontalAlign.Start) - .margin({ top: 18 }) + .margin({ top: 10 }) .bindSheet($$this.showWorkspacePicker, this.WorkspacePicker(), this.workspacePickerSheetOptions()) } diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/Theme.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/Theme.ets index 5d523840b3..5163a24c93 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/Theme.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/components/Theme.ets @@ -1,12 +1,21 @@ export const PAGE_BG: ResourceColor = $r('app.color.page_bg'); /** PAGE_BG at zero alpha, per theme — the transparent end of a fade-out. */ export const PAGE_BG_FADE: ResourceColor = $r('app.color.page_bg_fade'); +/** + * PAGE_BG at partial alpha, for a surface the transcript scrolls underneath. + * It is the floor under `backgroundBlurStyle`, not a replacement for it: the + * blur carries the effect, this keeps the text on top readable on a build or a + * device where the blur does not land. + */ +export const PAGE_BG_OVERLAY: ResourceColor = $r('app.color.page_bg_overlay'); export const TRANSPARENT: ResourceColor = $r('app.color.transparent'); export const INK: ResourceColor = $r('app.color.ink'); export const MUTED: ResourceColor = $r('app.color.muted'); export const SUBTLE: ResourceColor = $r('app.color.subtle'); export const LINE: ResourceColor = $r('app.color.line'); export const CARD: ResourceColor = $r('app.color.card'); +/** CARD at partial alpha, for a card that floats over scrolling content. */ +export const CARD_OVERLAY: ResourceColor = $r('app.color.card_overlay'); export const ACCENT: ResourceColor = $r('app.color.accent'); export const FILE_LINK: ResourceColor = $r('app.color.file_link'); export const PRIMARY_ACTION: ResourceColor = $r('app.color.primary_action'); diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/policy/ConversationMessageRenderPolicy.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/policy/ConversationMessageRenderPolicy.ets index e9c075c733..b2e802f38a 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/policy/ConversationMessageRenderPolicy.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/policy/ConversationMessageRenderPolicy.ets @@ -1,6 +1,14 @@ -import { ConversationUiMessage, ConversationUiMessageItem } from '../state/ConversationUiModels'; +import { ConversationUiImage, ConversationUiMessage, ConversationUiMessageItem } from '../state/ConversationUiModels'; export class ConversationMessageRenderPolicy { + /** + * Desktop attachments reach this device as pixels, never as a readable path, + * so an image the host could not inline has nothing to draw. + */ + static hasRenderableImagePixels(image: ConversationUiImage): boolean { + return (image.data_url || '').trim().length > 0; + } + static visibleText(text?: string): string { const value = (text || '').trim(); if (value === '(空消息)' || value === '(empty message)') { diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets index fcdd8a3bb6..9c1092b8f7 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets @@ -398,7 +398,7 @@ export abstract class AppRootRuntimeComposition { this.conversationController.updateKnownMessageCount(pollVersion, knownMessageCount); }, onTimelineReady: () => { - this.remotePageState.setConversationLoading(false); + this.remoteSessionViewModel.settleTranscriptLoading(); }, onComposerSubmitted: (sessionId: string, text: string, images: SelectedImageAttachment[]) => { if (this.remotePageState.activeSession.sessionId !== sessionId) { @@ -528,6 +528,10 @@ export abstract class AppRootRuntimeComposition { } }, onSnapshot: (snapshot: RemoteChatPollingSnapshot) => { + // A snapshot is the transcript on screen, which is what an open was + // waiting for. Snapshots are already fenced to the active session by + // the poller's canPoll guard, so this cannot end another tap's wait. + this.remoteSessionViewModel.settleTranscriptLoading(); this.remotePageState.setConversationLoading(false); if (snapshot.hasMoreMessages !== undefined) this.remotePageState.setHasMoreMessages(snapshot.hasMoreMessages); this.remotePageState.setHistoryLoadState(snapshot.historyLoading === true, snapshot.historyFailed === true); @@ -553,6 +557,7 @@ export abstract class AppRootRuntimeComposition { } }, onError: (error: Object) => { + this.remoteSessionViewModel.settleTranscriptLoading(); this.remotePageState.setConversationLoading(false); this.remotePageState.setStatusText(ConnectionErrorPolicy.errorText(error)); } diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/ConversationViewState.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/ConversationViewState.ets index 810250efca..7452089d09 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/ConversationViewState.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/ConversationViewState.ets @@ -36,6 +36,7 @@ export class ConversationViewState { composerCapabilities: ChatComposerCapabilities = GENERAL_CHAT_COMPOSER_CAPABILITIES; isBusy: boolean = false; isLoadingConversation: boolean = false; + isLoadingTranscript: boolean = false; canStop: boolean = false; hasMoreMessages: boolean = false; historyLoading: boolean = false; @@ -72,6 +73,10 @@ export class ConversationViewState { state.workspaceBranch = remote.workspaceBranch; state.connectionState = remote.connectionState; state.isLoadingConversation = remote.isLoadingConversation; + // Nothing is being waited for once the pane has no session: a skeleton left + // standing after a disconnect or a delete would describe a load that was + // abandoned rather than one still running. + state.isLoadingTranscript = remote.isLoadingTranscript && remote.activeSession.sessionId.length > 0; if (state.isLoadingConversation) { state.activeSession = ConversationViewState.emptySession(); } diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/RemotePageState.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/RemotePageState.ets index cbfb88b9c2..a2951e7363 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/RemotePageState.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/state/RemotePageState.ets @@ -62,6 +62,11 @@ export class RemotePageState { @Trace hasMoreSessions: boolean = false; @Trace isLoadingSessions: boolean = false; @Trace isLoadingConversation: boolean = false; + // Separate from isLoadingConversation, which means the session itself is not + // known yet and so blanks the header. Here the session is known and named; + // only its transcript is still on the way, and the header should keep saying + // which conversation the pane is about while the body says it is filling. + @Trace isLoadingTranscript: boolean = false; @Trace pendingSessionId: string = ''; @Trace isConversationDismissed: boolean = false; @Trace sessionErrorText: string = ''; @@ -239,6 +244,7 @@ export class RemotePageState { this.interactionMailbox = new InteractionMailboxState(); this.conversation.clearActiveSession(); this.isLoadingConversation = false; + this.isLoadingTranscript = false; this.pendingSessionId = ''; this.isConversationDismissed = false; this.setModelCatalog(RemoteUiState.emptyModelCatalog(), ''); @@ -325,6 +331,10 @@ export class RemotePageState { this.isLoadingConversation = loading; } + setTranscriptLoading(loading: boolean): void { + this.isLoadingTranscript = loading; + } + setPendingSessionId(sessionId: string): void { this.pendingSessionId = sessionId; } @@ -343,6 +353,7 @@ export class RemotePageState { this.hasMoreSessions = false; this.isLoadingSessions = false; this.isLoadingConversation = false; + this.isLoadingTranscript = false; this.pendingSessionId = ''; this.isConversationDismissed = false; this.isLoadingHome = false; @@ -357,6 +368,7 @@ export class RemotePageState { this.hasMoreSessions = false; this.isLoadingSessions = false; this.isLoadingConversation = false; + this.isLoadingTranscript = false; this.pendingSessionId = ''; this.isConversationDismissed = false; this.isLoadingHome = false; diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteSessionViewModel.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteSessionViewModel.ets index 1fce560ede..8390bac1cc 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteSessionViewModel.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteSessionViewModel.ets @@ -37,9 +37,10 @@ export class RemoteSessionViewModel { private readonly files: RemoteFileDownloadController; private readonly hooks: RemoteSessionViewModelHooks; // Long enough that a cached transcript never flashes a skeleton on its way in, - // short enough that a transcript coming over the relay still says so. + // short enough that a transcript coming over the relay still says so. The cap + // is the backstop for a session whose records never arrive at all. private readonly openLoadingGate: DeferredLoadingGate = - new DeferredLoadingGate(140, (visible: boolean): void => this.pageState.setConversationLoading(visible)); + new DeferredLoadingGate(140, (visible: boolean): void => this.pageState.setTranscriptLoading(visible), 20000); // How many opens this view model has started and not yet settled. Opening a // session raises the busy flag, so without this the flag an open sets would // read, to the next tap, as a run in progress. @@ -270,6 +271,7 @@ export class RemoteSessionViewModel { // straight from one conversation to the next. const loadingToken = this.openLoadingGate.arm(); const openGeneration = ++this.openGeneration; + let requested = false; onRouteChat(item.id); RemoteLogger.info(`session open routed session=${RemoteSessionViewModel.shortId(item.id)}`); this.openInFlight += 1; @@ -301,11 +303,20 @@ export class RemoteSessionViewModel { this.pageState.activeSession.sessionId === session.sessionId) { this.hooks.onStartPolling(); } + requested = true; } ); } finally { this.openInFlight -= 1; - this.openLoadingGate.release(loadingToken); + // The command returning is not the transcript arriving. `open` hands the + // durable record stream a request for this session's history and comes + // back in a few milliseconds; the records themselves land seconds later, + // and releasing here left that whole gap blank with nothing to read. + // `settleTranscriptLoading` ends the wait when the records do arrive. + // An open that never got as far as asking has nothing to wait for. + if (!requested) { + this.openLoadingGate.release(loadingToken); + } // Only the tap that still owns the gate owns the highlight: a second tap // during the first open has already moved both. if (loadingToken === this.openLoadingGate.currentToken()) { @@ -315,6 +326,11 @@ export class RemoteSessionViewModel { } } + /** The transcript reached the pane, so the wait an open started is over. */ + settleTranscriptLoading(): void { + this.openLoadingGate.settle(); + } + async deleteSession(item: RemoteSession, currentWorkspacePath: string): Promise { await this.sessions.delete( item, diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets index 6bcc56328a..afc8a004d0 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets @@ -19,6 +19,9 @@ import { shortSessionId } from './ConversationRuntime'; +/** Longest a streaming turn's transcript write is held back, in milliseconds. */ +const REMOTE_TRANSCRIPT_CACHE_INTERVAL_MS: number = 400; + export class RemoteTranscriptController { private readonly remote: RemotePageState; private readonly remoteRuntime?: RemoteConversationDependencies; @@ -28,6 +31,7 @@ export class RemoteTranscriptController { private knownRemoteMessageCount: number = 0; private isSyncingAfterTurn: boolean = false; private isRebuildingRemoteTranscript: boolean = false; + private pendingTranscriptCache: number = -1; constructor( remote: RemotePageState, @@ -276,6 +280,10 @@ export class RemoteTranscriptController { resetRemoteTimeline(sessionId: string): void { const runtime = requireRemoteRuntime(this.remoteRuntime); + if (this.pendingTranscriptCache !== -1) { + clearTimeout(this.pendingTranscriptCache); + this.pendingTranscriptCache = -1; + } runtime.timeline.reset(sessionId); this.knownPollVersionValue = 0; this.syncRemoteTimeline(); @@ -314,7 +322,12 @@ export class RemoteTranscriptController { runtime.timeline.applySnapshot(snapshot); this.syncRemoteTimeline(); if (snapshot.messageSnapshot !== undefined) { - this.cacheRemoteTranscript(snapshot.sessionId, true); + // A streaming turn restates the whole transcript on every chunk. Writing + // all of it back each time re-encrypts the session dozens of times a + // second for a cache nothing reads until the next open, so during a turn + // only the last write of a burst is kept. + if (this.hasRunningRemoteTurn()) this.scheduleTranscriptCache(snapshot.sessionId); + else this.cacheTranscriptNow(snapshot.sessionId, true); } else if (snapshot.newMessages.length > 0) { this.cacheRemoteTranscript(snapshot.sessionId, false); } @@ -348,6 +361,22 @@ export class RemoteTranscriptController { * that cannot be written only costs the next open a fetch it used to pay for * anyway. */ + private scheduleTranscriptCache(sessionId: string): void { + if (this.pendingTranscriptCache !== -1) return; + this.pendingTranscriptCache = setTimeout(() => { + this.pendingTranscriptCache = -1; + this.cacheRemoteTranscript(sessionId, true); + }, REMOTE_TRANSCRIPT_CACHE_INTERVAL_MS); + } + + private cacheTranscriptNow(sessionId: string, authoritativeSnapshot: boolean): void { + if (this.pendingTranscriptCache !== -1) { + clearTimeout(this.pendingTranscriptCache); + this.pendingTranscriptCache = -1; + } + this.cacheRemoteTranscript(sessionId, authoritativeSnapshot); + } + private cacheRemoteTranscript(sessionId: string, authoritativeSnapshot: boolean): void { const runtime = requireRemoteRuntime(this.remoteRuntime); const state: ChatTimelineState = runtime.timeline.snapshotState(); diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/services/CloudAccountClient.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/services/CloudAccountClient.ets index 01006eb76d..887300dd2b 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/services/CloudAccountClient.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/services/CloudAccountClient.ets @@ -46,6 +46,26 @@ export class CloudAccountRequestError extends Error { } } +/** + * Whether a failed sign-in poll should be tried again inside the window. + * + * Retry what the next tick could plausibly get past: a transport failure, and + * a relay that is rate limiting or briefly unavailable. Stop for anything the + * relay meant — a rejected transaction stays rejected however long the phone + * keeps asking. + * + * Kept in step with `shared/core-feature/.../AccountStore.kt`, which carries + * the same decision for the Android and iOS clients. + */ +export function retryableAuthorizationPollError(err: Object): boolean { + if (err instanceof CloudAccountTransportError) return true; + if (err instanceof CloudAccountRequestError) { + const status: number = err.statusCode; + return status === 429 || (status >= 500 && status < 600); + } + return false; +} + export interface CloudAccountSession { token: string; userId: string; @@ -178,11 +198,27 @@ export class CloudAccountClient { } if (!await openAuthorization(start.authorizationUrl)) throw new Error('Could not open account sign-in.'); const deadline = Math.min(start.expiresAt * 1000, Date.now() + 600000); + let lastTransient: Error | undefined = undefined; while (Date.now() < deadline && generation === this.authorizationGeneration) { await new Promise((resolve) => setTimeout(resolve, Math.min(30, Math.max(1, start.pollIntervalSeconds)) * 1000)); if (generation !== this.authorizationGeneration) break; const request: GitHubPollRequest = { transactionId: start.transactionId, transactionSecret: start.transactionSecret }; - const poll = await this.post(relayUrl, '/api/auth/github/poll', request); + let poll: GitHubPoll; + try { + poll = await this.post(relayUrl, '/api/auth/github/poll', request); + } catch (err) { + // The poll window spans the minutes the user spends in a browser and a + // mail app, which is exactly when a phone drops a connection, hops + // networks, or sleeps its radio. Ending the sign-in on the first hiccup + // would send them back to the start for something the next tick fixes + // by itself, so only a refusal the relay actually means stops the loop. + const failure = err as Error; + if (!retryableAuthorizationPollError(failure)) throw failure; + lastTransient = failure; + RemoteLogger.warn('account authorization poll retrying'); + continue; + } + lastTransient = undefined; if (generation !== this.authorizationGeneration) break; if (poll.status === 'authorized' && poll.tokens?.accessToken) { if (deviceSecret.length !== 32) throw new Error('Invalid device identity.'); @@ -204,6 +240,9 @@ export class CloudAccountClient { } if (poll.status === 'expired' || poll.status === 'denied') break; } + // A window that ran out while every poll was failing is a network problem, + // not a rejected sign-in: report the one the user can act on. + if (lastTransient !== undefined) throw lastTransient; throw new Error('GitHub sign-in expired or was cancelled.'); } diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/services/DeferredLoadingGate.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/services/DeferredLoadingGate.ets index 0b758bb1ab..0cd29dfe0e 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/services/DeferredLoadingGate.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/services/DeferredLoadingGate.ets @@ -12,21 +12,31 @@ * Each `arm` supersedes the one before it, so a second tap while the first open * is still settling cannot have its skeleton dismissed by the first one * finishing. The token is what enforces that. + * + * `maxVisibleMs` bounds how long the wait may be announced. What ends a wait is + * usually the content arriving, and content that never arrives would otherwise + * leave a skeleton standing for the rest of the session — a placeholder that + * outlives its subject reads as a hang. Past the cap the gate gives up saying + * "loading" and lets whatever the pane does have speak for itself. */ export class DeferredLoadingGate { private readonly delayMs: number; + private readonly maxVisibleMs: number; private readonly onVisible: (visible: boolean) => void; private timerId: number = 0; + private holdTimerId: number = 0; private token: number = 0; - constructor(delayMs: number, onVisible: (visible: boolean) => void) { + constructor(delayMs: number, onVisible: (visible: boolean) => void, maxVisibleMs: number = 0) { this.delayMs = delayMs; this.onVisible = onVisible; + this.maxVisibleMs = maxVisibleMs; } /** Starts a wait. The returned token is what `release` has to be given. */ arm(): number { this.cancelTimer(); + this.cancelHoldTimer(); this.token += 1; const armed = this.token; this.timerId = setTimeout(() => { @@ -35,6 +45,7 @@ export class DeferredLoadingGate { return; } this.onVisible(true); + this.startHoldTimer(armed); }, this.delayMs); return armed; } @@ -49,15 +60,40 @@ export class DeferredLoadingGate { if (token !== this.token) { return; } + this.settle(); + } + + /** + * Ends whatever wait is open because the thing waited for has arrived. + * + * Unlike `release` this takes no token: the arrival is reported by whatever + * carries the content — a transcript snapshot, an error — and that path has + * no memory of which tap asked for it. It is already fenced by the active + * session, so a signal that reaches here is about the wait that is open. + */ + settle(): void { this.cancelTimer(); + this.cancelHoldTimer(); this.onVisible(false); } /** Ends whatever wait is open, for teardown. */ reset(): void { - this.cancelTimer(); this.token += 1; - this.onVisible(false); + this.settle(); + } + + private startHoldTimer(armed: number): void { + if (this.maxVisibleMs <= 0) { + return; + } + this.holdTimerId = setTimeout(() => { + this.holdTimerId = 0; + if (armed !== this.token) { + return; + } + this.onVisible(false); + }, this.maxVisibleMs); } private cancelTimer(): void { @@ -66,4 +102,11 @@ export class DeferredLoadingGate { this.timerId = 0; } } + + private cancelHoldTimer(): void { + if (this.holdTimerId !== 0) { + clearTimeout(this.holdTimerId); + this.holdTimerId = 0; + } + } } diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/services/DurableSessionReducer.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/services/DurableSessionReducer.ets index 9ba5aa5fad..9e9c42520c 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/services/DurableSessionReducer.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/services/DurableSessionReducer.ets @@ -15,7 +15,7 @@ interface ItemData { } interface ItemRecord { type: string; data: ItemData; } export interface SessionRecord { sessionId: string; id: string; revision: number; deleted?: boolean; turn?: TurnRecord; round?: RoundRecord; item?: ItemRecord; } -interface StoredTurn { revision: number; data?: TurnRecord; } +interface StoredTurn { revision: number; data?: TurnRecord; authoritative?: boolean; } interface StoredRound { revision: number; data?: RoundRecord; } interface StoredItem { revision: number; turnId: string; roundId: string; data: ItemRecord; } @@ -56,7 +56,13 @@ export class DurableSessionReducer { if (record.item && !record.round) throw new Error('Session item is missing its round'); this.records.set(record.id, record); const turn = this.turns.get(data.turnId); - if (!turn || turn.revision < record.revision) this.turns.set(data.turnId, { revision: record.revision, data: data }); + // A round or item record repeats its parent turn so a partial page still + // reads, but that copy is a header, not the turn's own state: the turn + // record is what the user message and its attachment pixels come from, so a + // header fills a turn it has never seen and never overwrites one. + const header = !!record.round; + if (!turn || (turn.revision < record.revision && !(header && turn.authoritative))) + this.turns.set(data.turnId, { revision: record.revision, data: data, authoritative: !header }); if (record.round) { const round = this.rounds.get(record.round.id); if (!round || round.revision < record.revision) this.rounds.set(record.round.id, { revision: record.revision, data: record.round }); @@ -71,6 +77,15 @@ export class DurableSessionReducer { messages(): ChatMessage[] { const result: ChatMessage[] = []; const turns = Array.from(this.turns.values()).filter((turn: StoredTurn) => !!turn.data).sort((a: StoredTurn, b: StoredTurn) => (a.data?.turnIndex || 0) - (b.data?.turnIndex || 0)); + // Grouping the items under their turn once keeps a rebuild linear in the + // session's length. Scanning the whole item map again for every turn made a + // long session pay for its own length twice over, on the thread that draws + // it, which is where the scrolling went. + const byTurn: Map = new Map(); + this.items.forEach((item: StoredItem) => { + const bucket = byTurn.get(item.turnId); + if (bucket) bucket.push(item); else byTurn.set(item.turnId, [item]); + }); for (const stored of turns) { const turn = stored.data; if (!turn) continue; @@ -80,10 +95,10 @@ export class DurableSessionReducer { if (userText.length > 0 || (metadata?.images?.length || 0) > 0) result.push({ id: turn.userMessage.id, turnId: turn.turnId, role: 'user', text: userText, images: metadata?.images, status: 'sent', timestamp: new Date(turn.userMessage.timestamp).toISOString() }); const turnFence = this.tombstones.get(`turn/${turn.turnId}`) ?? 0; - const entries = Array.from(this.items.values()).filter((item: StoredItem) => { + const entries = (byTurn.get(turn.turnId) ?? []).filter((item: StoredItem) => { const round = this.rounds.get(item.roundId); const roundFence = Math.max(turnFence, this.tombstones.get(`round/${item.roundId}`) ?? 0); - return item.turnId === turn.turnId && !!round?.data && round.revision > roundFence && + return !!round?.data && round.revision > roundFence && item.revision > Math.max(roundFence, this.tombstones.get(`item/${item.data.data.id}`) ?? 0); }) .sort((a: StoredItem, b: StoredItem) => { diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteResponseMapper.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteResponseMapper.ets index 6a8d14c6aa..89bf10ffa5 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteResponseMapper.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteResponseMapper.ets @@ -102,19 +102,32 @@ export class RemoteResponseMapper { ]), messageCount: item.message_count || 0, workspacePath: item.workspace_path, - workspaceName: item.workspace_name + workspaceName: item.workspace_name, + parentSessionId: item.parent_session_id, + relationshipKind: item.relationship_kind }; return session; }); } - /** ACP sessions are created on Desktop and are not controllable on HarmonyOS. */ + /** + * ACP sessions are created on Desktop and are not controllable on HarmonyOS. + * Child sessions (Desktop's 侧问 / review / miniapp / subagent) belong under + * their parent, so they must not appear as standalone conversations. + */ static visibleSessions(sessions: RemoteSession[]): RemoteSession[] { return sessions.filter((session: RemoteSession) => RemoteResponseMapper.isVisibleSession(session)); } static isVisibleSession(session: RemoteSession): boolean { - return !(session.agentType || '').trim().toLowerCase().startsWith('acp:'); + if ((session.agentType || '').trim().toLowerCase().startsWith('acp:')) { + return false; + } + return !RemoteResponseMapper.isChildSession(session); + } + + static isChildSession(session: RemoteSession): boolean { + return (session.parentSessionId || '').trim().length > 0; } static chatMessage(item: ChatMessageResponse): ChatMessage { diff --git a/src/apps/mobile/harmonyos/entry/src/main/resources/base/element/color.json b/src/apps/mobile/harmonyos/entry/src/main/resources/base/element/color.json index 5020c4c5b6..d63161fa23 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/resources/base/element/color.json +++ b/src/apps/mobile/harmonyos/entry/src/main/resources/base/element/color.json @@ -16,6 +16,10 @@ "name": "page_bg_fade", "value": "#00FFFFFF" }, + { + "name": "page_bg_overlay", + "value": "#B8FFFFFF" + }, { "name": "ink", "value": "#171717" @@ -36,6 +40,10 @@ "name": "card", "value": "#FFFFFF" }, + { + "name": "card_overlay", + "value": "#CCFFFFFF" + }, { "name": "accent", "value": "#111111" diff --git a/src/apps/mobile/harmonyos/entry/src/main/resources/dark/element/color.json b/src/apps/mobile/harmonyos/entry/src/main/resources/dark/element/color.json index 8b971cbc62..d7597f1000 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/resources/dark/element/color.json +++ b/src/apps/mobile/harmonyos/entry/src/main/resources/dark/element/color.json @@ -16,6 +16,10 @@ "name": "page_bg_fade", "value": "#00151514" }, + { + "name": "page_bg_overlay", + "value": "#B8151514" + }, { "name": "ink", "value": "#F4F3EF" @@ -36,6 +40,10 @@ "name": "card", "value": "#252522" }, + { + "name": "card_overlay", + "value": "#CC252522" + }, { "name": "accent", "value": "#5B5954" diff --git a/src/apps/mobile/harmonyos/entry/src/test/ConversationPresentationUnit.test.ets b/src/apps/mobile/harmonyos/entry/src/test/ConversationPresentationUnit.test.ets index 206bb7a900..90c0c7cdc8 100644 --- a/src/apps/mobile/harmonyos/entry/src/test/ConversationPresentationUnit.test.ets +++ b/src/apps/mobile/harmonyos/entry/src/test/ConversationPresentationUnit.test.ets @@ -29,6 +29,7 @@ import { ChatMessageStructurePolicy } from '../main/ets/pages/policy/ChatMessage import { ToolStatusPresentationPolicy } from '../main/ets/pages/policy/ToolStatusPresentationPolicy'; import { PlanToolPolicy } from '../main/ets/pages/policy/PlanToolPolicy'; import { + ConversationUiImage, ConversationUiMessage, ConversationUiMessageItem, ConversationUiToolStatus @@ -215,6 +216,16 @@ export default function conversationPresentationUnitTest() { }); describe('ConversationMessageRenderPolicy', () => { + it('names an attachment the host could not inline instead of drawing a blank frame', 0, () => { + const inlined: ConversationUiImage = { name: 'chart.png', data_url: 'data:image/png;base64,YQ==' }; + const pathOnly: ConversationUiImage = { name: 'dropped-on-the-desktop.png', data_url: '' }; + + expect(ConversationMessageRenderPolicy.hasRenderableImagePixels(inlined)).assertTrue(); + expect(ConversationMessageRenderPolicy.hasRenderableImagePixels(pathOnly)).assertFalse(); + expect(ConversationMessageRenderPolicy.hasRenderableImagePixels({ name: 'blank.png', data_url: ' ' })) + .assertFalse(); + }); + it('does not throw when assistant text is missing and still shows structured content', 0, () => { expect(ConversationMessageRenderPolicy.visibleText(undefined)).assertEqual(''); expect(ConversationMessageRenderPolicy.visibleText('(空消息)')).assertEqual(''); diff --git a/src/apps/mobile/harmonyos/entry/src/test/ConversationStateUnit.test.ets b/src/apps/mobile/harmonyos/entry/src/test/ConversationStateUnit.test.ets index 17f64f02d4..30f5ab6309 100644 --- a/src/apps/mobile/harmonyos/entry/src/test/ConversationStateUnit.test.ets +++ b/src/apps/mobile/harmonyos/entry/src/test/ConversationStateUnit.test.ets @@ -1,4 +1,5 @@ import { describe, it, expect } from '@ohos/hypium'; +import { DurableSessionReducer, SessionRecord } from '../main/ets/services/DurableSessionReducer'; import { RemoteI18n } from '../main/ets/i18n/RemoteI18n'; import { ChatSessionController, ChatSessionSnapshot } from '../main/ets/services/ChatSessionController'; import { ChatComposerPolicy } from '../main/ets/services/ChatComposerPolicy'; @@ -121,6 +122,7 @@ import { } from '../main/ets/model/RemoteModels'; import { + durableRecord, ToolInputFixture, CryptoImageFixture, CryptoCommandFixture, @@ -167,6 +169,8 @@ import { FakeGeneralChatConfigStore } from './LocalTestFixtures'; +interface DurableUserMetadata { images: ImageAttachment[]; } + export default function conversationStateUnitTest() { describe('ChatTimelineProjector', () => { it('filters seed messages and returns empty state for empty real history', 0, () => { @@ -1522,6 +1526,26 @@ export default function conversationStateUnitTest() { expect(remote.activeSession.sessionId).assertEqual('remote-session'); expect(remoteProjection.isLoadingConversation).assertTrue(); + // A transcript still crossing the relay keeps the header: the pane knows + // which conversation it is showing, only not yet what is in it. The + // identity-unknown flag above is the one that blanks the title. + remote.setConversationLoading(false); + remote.setTranscriptLoading(true); + const loadingTranscript = ConversationViewState.project(AppRoute.RemoteChat, remote, general, ''); + expect(loadingTranscript.isLoadingConversation).assertFalse(); + expect(loadingTranscript.isLoadingTranscript).assertTrue(); + expect(loadingTranscript.activeSession.sessionId).assertEqual('remote-session'); + + // A pane with no session left is not waiting for one. + remote.clearActiveSession(); + remote.setTranscriptLoading(true); + expect(ConversationViewState.project(AppRoute.RemoteChat, remote, general, '').isLoadingTranscript).assertFalse(); + remote.setTranscriptLoading(false); + remote.setActiveSession({ + sessionId: 'remote-session', title: 'Remote', workspacePath: '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/workspace/OpenBitFun', agentType: 'code' + }); + remote.setConversationLoading(true); + const generalProjection = ConversationViewState.project(AppRoute.ChatHome, remote, general, 'Configure model'); expect(generalProjection.surface).assertEqual(ChatSurface.General); expect(generalProjection.chatInput).assertEqual('general draft'); @@ -1701,4 +1725,32 @@ export default function conversationStateUnitTest() { expect(remote.timelineItems.length).assertEqual(1); }); }); + + describe('DurableSessionRecordReduction', () => { + it('keeps the turn record user message when a child repeats it as a header', 0, () => { + const reducer = new DurableSessionReducer(); + const metadata: DurableUserMetadata = { + images: [{ name: 'shot.png', data_url: 'data:image/png;base64,AA' } as ImageAttachment] + }; + const turnRecord: SessionRecord = { + sessionId: 'session', id: 'turn/turn', revision: 1, + turn: { + sessionId: 'session', turnId: 'turn', turnIndex: 0, timestamp: 1, status: 'inprogress', + userMessage: { id: 'user', content: 'look at this', timestamp: 1, metadata: metadata } + } + }; + reducer.apply({ session_id: 'session', event: 'session-record', payload: turnRecord }); + expect((reducer.messages()[0].images || []).length).assertEqual(1); + + // The round and item records repeat the turn only as a parent header, and + // that copy carries the attachment as it was recorded rather than the + // pixels the turn record was given. It must not replace the turn. + reducer.apply(durableRecord('session', 5, 'on it')); + const reduced = reducer.messages(); + expect(reduced[0].text).assertEqual('look at this'); + expect((reduced[0].images || []).length).assertEqual(1); + expect((reduced[0].images || [])[0].data_url).assertEqual('data:image/png;base64,AA'); + expect(reduced[1].text).assertEqual('on it'); + }); + }); } diff --git a/src/apps/mobile/harmonyos/entry/src/test/LifecycleUnit.test.ets b/src/apps/mobile/harmonyos/entry/src/test/LifecycleUnit.test.ets index ee653b52ad..4b6f751ba0 100644 --- a/src/apps/mobile/harmonyos/entry/src/test/LifecycleUnit.test.ets +++ b/src/apps/mobile/harmonyos/entry/src/test/LifecycleUnit.test.ets @@ -8,6 +8,7 @@ import { ChatTimelineItem, ChatTimelineProjector } from '../main/ets/services/Ch import { ChatTimelineStore } from '../main/ets/services/ChatTimelineStore'; import { ConversationEvent } from '../main/ets/services/ConversationEvent'; import { AsyncLifecycleGate } from '../main/ets/services/AsyncLifecycleGate'; +import { DeferredLoadingGate } from '../main/ets/services/DeferredLoadingGate'; import { Encoding } from '../main/ets/services/Encoding'; import { GeneralChatDraftStore, @@ -389,6 +390,46 @@ export default function lifecycleUnitTest() { }); }); + const settleAfter = (delayMs: number): Promise => + new Promise((resolve: () => void) => setTimeout(resolve, delayMs)); + + describe('DeferredLoadingGate', () => { + it('announces a wait the content has not ended and caps one nothing ever ends', 0, async () => { + const seen: boolean[] = []; + const gate = new DeferredLoadingGate(10, (visible: boolean): void => { seen.push(visible); }, 60); + + // A wait the content ends: the command that started it returning is not + // what closes the gate, so the skeleton stands until `settle` says the + // transcript arrived. + gate.arm(); + await settleAfter(30); + expect(seen.length).assertEqual(1); + expect(seen[0]).assertTrue(); + gate.settle(); + expect(seen.length).assertEqual(2); + expect(seen[1]).assertFalse(); + + // A wait nothing ends: the cap takes the skeleton down rather than + // leaving a placeholder standing over content that never came. + gate.arm(); + await settleAfter(30); + expect(seen.length).assertEqual(3); + expect(seen[2]).assertTrue(); + await settleAfter(90); + expect(seen.length).assertEqual(4); + expect(seen[3]).assertFalse(); + + // A superseded wait never announces itself, so a second tap during the + // first open cannot be described by the first one's timers. + gate.arm(); + gate.arm(); + gate.settle(); + await settleAfter(40); + expect(seen.length).assertEqual(5); + expect(seen[4]).assertFalse(); + }); + }); + describe('RemoteHeartbeatController', () => { it('owns heartbeat timer restart, stop, and tick dispatch', 0, () => { const scheduler = new FakeRemoteHeartbeatScheduler(); diff --git a/src/apps/mobile/harmonyos/entry/src/test/TransportAndGeneralChatUnit.test.ets b/src/apps/mobile/harmonyos/entry/src/test/TransportAndGeneralChatUnit.test.ets index 7d926d37a5..3ff37f064b 100644 --- a/src/apps/mobile/harmonyos/entry/src/test/TransportAndGeneralChatUnit.test.ets +++ b/src/apps/mobile/harmonyos/entry/src/test/TransportAndGeneralChatUnit.test.ets @@ -97,7 +97,9 @@ import { CloudAccountDevice, CloudProvisionedDevice, CloudAccountRequestError, - CloudAccountSession + CloudAccountSession, + CloudAccountTransportError, + retryableAuthorizationPollError } from '../main/ets/services/CloudAccountClient'; import { CloudAccountSessionStore, @@ -587,6 +589,22 @@ class AccountDeviceSwitchHarness { export default function transportAndGeneralChatUnitTest() { describe('Account device links', () => { + /** + * The sign-in poll runs for the minutes the user spends in a browser and a + * mail app, which is where a phone most reliably drops a connection. A + * single failure there used to end the sign-in and send them back to the + * start, so a transport failure or a relay that is briefly unwilling has to + * cost one interval, while a refusal the relay meant still stops the loop. + */ + it('keeps polling for sign-in through transport failures but not refusals', 0, () => { + expect(retryableAuthorizationPollError(new CloudAccountTransportError(2300007))).assertTrue(); + expect(retryableAuthorizationPollError(new CloudAccountRequestError('too many polls', 429))).assertTrue(); + expect(retryableAuthorizationPollError(new CloudAccountRequestError('relay down', 503))).assertTrue(); + expect(retryableAuthorizationPollError(new CloudAccountRequestError('denied', 401))).assertFalse(); + expect(retryableAuthorizationPollError(new CloudAccountRequestError('bad request', 400))).assertFalse(); + expect(retryableAuthorizationPollError(new Error('unparseable relay response'))).assertFalse(); + }); + it('uses the same device selector for official and LAN endpoints', 0, () => { for (const endpoint of ['https://remote.openbitfun.com/v/1.0.1', 'http://192.168.1.10:9700']) { const parsed = accountDeviceLink(`${endpoint}/#/pair?did=desktop-1`); @@ -2011,6 +2029,27 @@ export default function transportAndGeneralChatUnitTest() { expect(raw.length).assertEqual(3); }); + it('hides child sessions so Desktop side questions are not standalone rows', 0, () => { + const items: SessionItemResponse[] = [{ + session_id: 'parent-session', + name: 'iOS sidebar spacing' + }, { + session_id: 'btw-session', + name: 'why did it grep with bash', + parent_session_id: 'parent-session', + relationship_kind: 'btw' + }]; + + const visible = RemoteResponseMapper.sessions(items); + const raw = RemoteResponseMapper.allSessions(items); + + expect(visible.length).assertEqual(1); + expect(visible[0].id).assertEqual('parent-session'); + expect(raw.length).assertEqual(2); + expect(raw[1].parentSessionId).assertEqual('parent-session'); + expect(raw[1].relationshipKind).assertEqual('btw'); + }); + it('maps alternate session time fields', 0, () => { const sessions = RemoteResponseMapper.sessions([{ session_id: 'session-time', diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift index f30a2c14ab..91cb500be4 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift @@ -7,6 +7,11 @@ import UIKit struct ChatTimelineView: View { @ObservedObject var model: MobileAppModel var onLoadOlderMessages: (() -> Void)? = nil + /// Height of the floating bottom layer, so the jump-to-bottom button rides + /// above the composer instead of hiding behind it. The transcript's own + /// padding comes from `safeAreaInset`; an overlay does not get that, which + /// is why this one number still has to be passed in. + var bottomOverlayInset: CGFloat = 0 @StateObject private var scrollController = TimelineScrollController() @State private var historyAnchor: (id: String, top: CGFloat, firstID: String)? @@ -130,6 +135,7 @@ struct ChatTimelineView: View { } .buttonStyle(.plain) .padding(18) + .padding(.bottom, bottomOverlayInset) .accessibilityIdentifier("timeline.scrollToBottom") .accessibilityLabel(Text(model.localized("滚动到底部"))) } diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift index b0b8dfc00c..8b6aa4b222 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift @@ -72,7 +72,10 @@ struct ComposerBar: View { .frame(minHeight: expanded ? MobileDesignGeometry.composerExpandedHeight : MobileDesignGeometry.composerCollapsedHeight) - .background(OpenBitFunTheme.card) + // The transcript now runs underneath the pill, so the fill is a material + // and the blur does the separating rather than an opaque card colour. + .background(MobileDesignColors.cardOverlay) + .background(.ultraThinMaterial) .overlay( RoundedRectangle( cornerRadius: expanded || !model.composerImages.isEmpty @@ -89,10 +92,9 @@ struct ComposerBar: View { ) ) .shadow(color: OpenBitFunTheme.shadowSubtle, radius: 10, y: 2) - .padding(.horizontal, MobileDesignGeometry.contentGutter) + .padding(.horizontal, MobileDesignGeometry.conversationOverlaySideInset) .padding(.top, 8) .padding(.bottom, 14) - .background(OpenBitFunTheme.page) .animation(.easeOut(duration: 0.22), value: expanded) .animation(.easeOut(duration: 0.18), value: model.composerImages.count) .onChange(of: model.composerSendGeneration) { _ in diff --git a/src/apps/mobile/ios/OpenBitFun/Features/DesignSystem/GeneratedMobileDesignTokens.swift b/src/apps/mobile/ios/OpenBitFun/Features/DesignSystem/GeneratedMobileDesignTokens.swift index 0a59abdbcb..12e20d1803 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/DesignSystem/GeneratedMobileDesignTokens.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/DesignSystem/GeneratedMobileDesignTokens.swift @@ -26,11 +26,13 @@ enum MobileDesignColors { static let transparent = dynamic(light: 0x00000000, dark: 0x00000000) static let pageBg = dynamic(light: 0xFFFFFFFF, dark: 0xFF151514) static let pageBgFade = dynamic(light: 0x00FFFFFF, dark: 0x00151514) + static let pageBgOverlay = dynamic(light: 0xB8FFFFFF, dark: 0xB8151514) static let ink = dynamic(light: 0xFF171717, dark: 0xFFF4F3EF) static let muted = dynamic(light: 0xFF706F6A, dark: 0xFFAAA8A0) static let subtle = dynamic(light: 0xFFA5A39B, dark: 0xFF77756E) static let line = dynamic(light: 0xFFE9E7E2, dark: 0xFF363531) static let card = dynamic(light: 0xFFFFFFFF, dark: 0xFF252522) + static let cardOverlay = dynamic(light: 0xCCFFFFFF, dark: 0xCC252522) static let accent = dynamic(light: 0xFF111111, dark: 0xFF5B5954) static let fileLink = dynamic(light: 0xFF2563EB, dark: 0xFF60A5FA) static let primaryAction = dynamic(light: 0xFF111111, dark: 0xFF454540) @@ -115,6 +117,8 @@ enum MobileDesignGeometry { static let contentGutter: CGFloat = 16 static let connectionStripHeight: CGFloat = 48 static let timelineTopPadding: CGFloat = 22 + static let conversationOverlaySideInset: CGFloat = 16 + static let conversationEdgeFadeHeight: CGFloat = 28 static let messageSpacing: CGFloat = 18 static let messageBubbleMaxWidth: CGFloat = 276 static let messageBubbleHorizontalPadding: CGFloat = 14 diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Shell/MiniAppsView.swift b/src/apps/mobile/ios/OpenBitFun/Features/Shell/MiniAppsView.swift index 3ad23e7836..0e73d9bae4 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Shell/MiniAppsView.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Shell/MiniAppsView.swift @@ -25,7 +25,8 @@ struct MiniAppsButton: View { .buttonStyle(.plain) .foregroundStyle(OpenBitFunTheme.ink) .accessibilityLabel(model.localized("小应用")) - .padding(.vertical, 16) + .padding(.top, 4) + .padding(.bottom, 8) } else { Button(model.localized("小应用")) { open = true } .frame(minHeight: 44) diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Shell/MobileShellView.swift b/src/apps/mobile/ios/OpenBitFun/Features/Shell/MobileShellView.swift index 0145d4214c..4bf74f7aea 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Shell/MobileShellView.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Shell/MobileShellView.swift @@ -2,11 +2,21 @@ import OpenBitFunMobileCore import SwiftUI import UniformTypeIdentifiers +/// The measured height of the conversation's floating bottom layer. Only the +/// jump-to-bottom button needs it: `safeAreaInset` already pads the transcript. +private struct BottomOverlayHeightKey: PreferenceKey { + static var defaultValue: CGFloat = 0 + static func reduce(value: inout CGFloat, nextValue: () -> CGFloat) { + value = max(value, nextValue()) + } +} + struct MobileShellView: View { @ObservedObject var model: MobileAppModel @State private var wideSidebarCollapsed = false @State private var sessionActionsOpen = false @State private var sidebarActionSession: ChatSession? + @State private var bottomOverlayHeight: CGFloat = 0 var body: some View { GeometryReader { proxy in @@ -316,34 +326,112 @@ struct MobileShellView: View { sidebarAction: (() -> Void)?, sidebarActionLabel: String ) -> some View { - VStack(spacing: 0) { - if !showsWelcomeHome { - ConversationHeader( - model: model, - actionsOpen: $sessionActionsOpen, - sidebarAction: sidebarAction, - sidebarActionLabel: sidebarActionLabel - ) - } + Group { if showsWelcomeHome { WelcomeHomeView(model: model) } else if model.surface == .remote && !model.remoteSessionSelected { - RemoteConnectedHomeView(model: model, onBrowse: sidebarAction) - } else { - PermissionMailboxPanel(model: model) - ZStack { - ChatTimelineView(model: model) - if model.surface == .remote && model.remoteConversationLoading { - ConversationLoadingState() - } + VStack(spacing: 0) { + ConversationHeader( + model: model, + actionsOpen: $sessionActionsOpen, + sidebarAction: sidebarAction, + sidebarActionLabel: sidebarActionLabel + ) + RemoteConnectedHomeView(model: model, onBrowse: sidebarAction) } - ComposerBar(model: model) + } else { + floatingConversation( + sidebarAction: sidebarAction, + sidebarActionLabel: sidebarActionLabel + ) } } .accessibilityElement(children: .contain) .accessibilityIdentifier(conversationAccessibilityIdentifier) } + /** + The transcript owns the whole pane; the header and the composer float over + it and the transcript scrolls underneath both, fading out as it reaches + either edge. `safeAreaInset` is exactly that arrangement: it reserves the + room at rest so nothing starts out hidden, and lets the scroll view pass + beneath once it moves — which is why this needs no height measurement, + unlike the same layout on Android and HarmonyOS. + */ + private func floatingConversation( + sidebarAction: (() -> Void)?, + sidebarActionLabel: String + ) -> some View { + ZStack { + ChatTimelineView(model: model, bottomOverlayInset: bottomOverlayHeight) + if model.surface == .remote && model.remoteConversationLoading { + ConversationLoadingState() + } + } + .onPreferenceChange(BottomOverlayHeightKey.self) { bottomOverlayHeight = $0 } + .safeAreaInset(edge: .top, spacing: 0) { + VStack(spacing: 0) { + VStack(spacing: 0) { + ConversationHeader( + model: model, + actionsOpen: $sessionActionsOpen, + sidebarAction: sidebarAction, + sidebarActionLabel: sidebarActionLabel + ) + PermissionMailboxPanel(model: model) + } + .background(MobileDesignColors.pageBgOverlay) + .background(.ultraThinMaterial) + conversationTopEdgeFade + } + } + .safeAreaInset(edge: .bottom, spacing: 0) { + VStack(spacing: 0) { + ComposerBar(model: model) + } + // The fade covers the whole layer, not just a strip above it: the + // transcript runs behind the composer, so anything short of that + // leaves a line of text sitting crisp and legible beside the pill + // after it has already faded out higher up. It stops short of the + // page colour so the pill's material still has something to blur. + .background( + LinearGradient( + stops: [ + .init(color: MobileDesignColors.pageBgFade, location: 0), + .init(color: MobileDesignColors.pageBgOverlay, location: 0.45), + .init(color: MobileDesignColors.pageBgOverlay, location: 1), + ], + startPoint: .top, + endPoint: .bottom + ) + ) + .background( + GeometryReader { proxy in + Color.clear.preference( + key: BottomOverlayHeightKey.self, + value: proxy.size.height + ) + } + ) + } + } + + /** + Carries the header's opaque band down into the transcript, so a line of + text is not cut in half at its edge. Never takes a touch: the transcript + below it is still the thing being pointed at. The bottom layer needs no + strip of its own — its whole background is the gradient. + */ + private var conversationTopEdgeFade: some View { + LinearGradient( + colors: [OpenBitFunTheme.page, OpenBitFunTheme.page.opacity(0)], + startPoint: .top, + endPoint: .bottom + ) + .frame(height: MobileDesignGeometry.conversationEdgeFadeHeight) + .allowsHitTesting(false) + } + private var conversationAccessibilityIdentifier: String { guard model.surface == .remote else { return "conversation.local" } let sessionID = model.selectedSessionID diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Shell/SidebarView.swift b/src/apps/mobile/ios/OpenBitFun/Features/Shell/SidebarView.swift index 2c1e7fbe23..1462601e7a 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Shell/SidebarView.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Shell/SidebarView.swift @@ -296,7 +296,7 @@ struct SidebarView: View { } .frame(height: 38) - .padding(.top, 18) + .padding(.top, 10) if let error = model.accountDirectoryError { Button { model.refreshRemoteDevices() } label: { diff --git a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+Account.swift b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+Account.swift index 247cae1b1a..e7c132af24 100644 --- a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+Account.swift +++ b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+Account.swift @@ -196,7 +196,9 @@ extension MobileAppModel { } func apply(accountState state: AccountUiState, generation: UInt64) { - guard !accountLoginPreview, !localActionPreview, !remoteCreatePreview, + // The directory fixture describes a signed-in account with a chosen + // device; a signed-out core would otherwise wipe it on launch. + guard !accountLoginPreview, !localActionPreview, !remoteCreatePreview, !directoryFixturePreview, generation == accountGeneration else { return } accountGeneration = generation if let ready = state as? AccountUiStateReady, let failure = ready.refreshFailure { diff --git a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift index 44f5b4acbf..d7de2640fc 100644 --- a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift +++ b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift @@ -369,6 +369,10 @@ extension MobileAppModel { coreAdapter?.openRemoteSession(sessionID: pending.sessionID) } + /// How long an open may be announced as a wait. Matched to HarmonyOS's + /// `DeferredLoadingGate` cap and Android's `CONVERSATION_LOADING_MAX_VISIBLE_MS`. + static let remoteConversationLoadingMaxVisibleNanoseconds: UInt64 = 20_000_000_000 + /// Mirrors HarmonyOS's deferred conversation-loading gate. Cached transcripts /// normally arrive inside the grace period; a relay fetch gets an explicit /// skeleton instead of leaving the previous session visible. @@ -400,6 +404,19 @@ extension MobileAppModel { self.remoteConversationLoadGeneration == generation, self.remoteConversationOpeningSessionID == sessionID else { return } self.remoteConversationLoading = true + // What ends this wait is the transcript arriving. One that never + // arrives would otherwise leave the skeleton standing for the rest + // of the session, and a placeholder that outlives its subject reads + // as a hang. Past the cap the pane stops saying "loading" and lets + // whatever it does have speak for itself. + do { + try await Task.sleep(nanoseconds: Self.remoteConversationLoadingMaxVisibleNanoseconds) + } catch { + return + } + guard self.remoteConversationLoadGeneration == generation, + self.remoteConversationOpeningSessionID == sessionID else { return } + self.remoteConversationLoading = false } } @@ -1322,7 +1339,11 @@ extension MobileAppModel { workspaceName: session.workspaceName, workspaceIdentity: session.workspaceScope.map { RemoteWorkspaceIdentity(path: $0.path, remoteConnectionId: $0.remoteConnectionId, remoteSshHost: $0.remoteSshHost) - } + }, + // These rows are already past the visibility filter, so they + // have no parent left to declare. + parentSessionId: nil, + relationshipKind: nil ) } } diff --git a/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/ConversationModels.kt b/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/ConversationModels.kt index 4842d36b33..8dc0c4715c 100644 --- a/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/ConversationModels.kt +++ b/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/ConversationModels.kt @@ -48,6 +48,9 @@ public data class RemoteSession public constructor( public val workspacePath: String?, public val workspaceName: String?, public val workspaceIdentity: RemoteWorkspaceIdentity?, + /** Set when the session hangs off another one (btw/review/miniapp/subagent). */ + public val parentSessionId: String? = null, + public val relationshipKind: String? = null, ) { public constructor(id: String, title: String, agentType: String, status: String, updatedAt: String, createdAt: String, messageCount: Int, workspacePath: String?, workspaceName: String?) : diff --git a/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicy.kt b/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicy.kt index 400d34a100..a4a29e074a 100644 --- a/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicy.kt +++ b/src/apps/mobile/shared/core-domain/src/commonMain/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicy.kt @@ -35,6 +35,21 @@ public object SessionAgentTypes { !agentType.trim().lowercase().startsWith("acp:") } +/** + * Which sessions belong in a flat mobile session list. + * + * Desktop renders child sessions (its 侧问 / review / mini app / subagent) nested + * under the conversation they were asked from. Mobile has no nesting yet, so a + * child would otherwise read as an unrelated standalone conversation. + */ +public object SessionListVisibility { + public fun isMobileVisible(session: RemoteSession): Boolean = + SessionAgentTypes.isMobileVisible(session.agentType) && !isChild(session) + + public fun isChild(session: RemoteSession): Boolean = + !session.parentSessionId.isNullOrBlank() +} + /** * Workspace paths as identity, from `ConversationSessionFilterPolicy`. * diff --git a/src/apps/mobile/shared/core-domain/src/commonTest/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicyTest.kt b/src/apps/mobile/shared/core-domain/src/commonTest/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicyTest.kt index 1e97d82066..8a3670b80a 100644 --- a/src/apps/mobile/shared/core-domain/src/commonTest/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicyTest.kt +++ b/src/apps/mobile/shared/core-domain/src/commonTest/kotlin/com/openbitfun/mobile/core/domain/SessionListPolicyTest.kt @@ -33,6 +33,40 @@ class SessionListPolicyTest { assertTrue(SessionAgentTypes.isMobileVisible("code")) } + @Test + fun childSessionsStayOffTheFlatMobileList() { + // Desktop's 侧问 keeps its own session, nested under the conversation it + // was asked from. A flat list would show it as an unrelated thread. + val parent = session(id = "parent") + val child = session(id = "btw", parentSessionId = "parent", relationshipKind = "btw") + + assertTrue(SessionListVisibility.isMobileVisible(parent)) + assertFalse(SessionListVisibility.isMobileVisible(child)) + assertTrue(SessionListVisibility.isChild(child)) + assertFalse(SessionListVisibility.isChild(session(id = "blank", parentSessionId = " "))) + assertFalse(SessionListVisibility.isMobileVisible(session(id = "acp", agentType = "acp:codex"))) + } + + private fun session( + id: String, + agentType: String = "code", + parentSessionId: String? = null, + relationshipKind: String? = null, + ) = RemoteSession( + id = id, + title = id, + agentType = agentType, + status = "idle", + updatedAt = "", + createdAt = "", + messageCount = 0, + workspacePath = null, + workspaceName = null, + workspaceIdentity = null, + parentSessionId = parentSessionId, + relationshipKind = relationshipKind, + ) + @Test fun untitledSessionsGetTheAgentSpecificWireName() { assertEquals("Remote Code Session", SessionNaming.wireSessionName("code", " ")) diff --git a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/account/AccountStore.kt b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/account/AccountStore.kt index dda6fbeeda..bfd3cfe650 100644 --- a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/account/AccountStore.kt +++ b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/account/AccountStore.kt @@ -604,6 +604,75 @@ public class AccountStore internal constructor( } } +/** + * The sign-in poll loop, kept apart from its transport so it can be tested. + * + * The loop is the whole of the fix it carries: a poll that fails has to be + * tried again rather than end the sign-in, and only this shape lets a test say + * so without a relay to fail against. + */ +internal object AuthorizationPoll { + /** + * Whether a failed poll should be tried again inside the sign-in window. + * + * Retry what the next tick could plausibly get past: a dropped connection, + * a timeout, a relay that is briefly unavailable, and a rate limit that + * asks for exactly the wait the loop already does between polls. Stop for + * anything the relay meant — a rejected or unparseable transaction stays + * rejected however long the phone keeps asking. + */ + fun retryable(failure: CloudAccountFailure): Boolean = when (failure) { + CloudAccountFailure.NETWORK, + CloudAccountFailure.TIMEOUT, + CloudAccountFailure.RATE_LIMITED, + CloudAccountFailure.RELAY_UNAVAILABLE -> true + CloudAccountFailure.INVALID_CREDENTIALS, + CloudAccountFailure.AUTHENTICATION, + CloudAccountFailure.MALFORMED_RESPONSE -> false + } + + /** + * Polls [poll] until the transaction is authorized, refused, or its window + * closes, and returns the access token it was granted. + * + * The window spans the minutes the user spends in a browser and a mail app, + * which is exactly when a phone drops a connection, hops networks, or + * sleeps its radio. Ending the sign-in on the first hiccup would send them + * back to the start for something the next tick fixes by itself, so a + * [retryable] failure only costs one interval. A window that ran out while + * every poll was failing surfaces that failure rather than + * [CloudAccountFailure.AUTHENTICATION]: the network is what the user can + * act on, and the transaction was never actually refused. + */ + suspend fun awaitAccessToken( + start: com.openbitfun.mobile.core.transport.GitHubAuthorization, + log: TransportLog, + nowSeconds: () -> Long = { kotlin.time.Clock.System.now().epochSeconds }, + poll: suspend () -> com.openbitfun.mobile.core.transport.GitHubAuthorizationPoll, + ): String { + var lastTransient: CloudAccountException? = null + while (nowSeconds() < start.expiresAt) { + kotlinx.coroutines.delay(start.pollIntervalSeconds.coerceIn(1, 30) * 1000L) + val result = try { + poll() + } catch (cause: CloudAccountException) { + if (!retryable(cause.failure)) throw cause + lastTransient = cause + log.warn("account authorization poll retrying reason=${cause.failure}") + continue + } + lastTransient = null + if (result.status == "authorized") { + val token = result.tokens?.accessToken + if (!token.isNullOrEmpty()) return token + break + } + if (result.status == "expired" || result.status == "denied") break + } + throw lastTransient ?: CloudAccountException(CloudAccountFailure.AUTHENTICATION) + } +} + private class CloudBackend( private val client: CloudAccountClient, private val log: TransportLog, @@ -617,17 +686,8 @@ private class CloudBackend( ): AccountSessionData { val start = client.startAuthorization(relayUrl) onAuthorization(start.authorizationUrl) - var accessToken: String? = null - while (kotlin.time.Clock.System.now().epochSeconds < start.expiresAt) { - kotlinx.coroutines.delay(start.pollIntervalSeconds.coerceIn(1, 30) * 1000L) - val poll = client.pollAuthorization(relayUrl, start) - if (poll.status == "authorized") { - accessToken = poll.tokens?.accessToken - break - } - if (poll.status == "expired" || poll.status == "denied") break - } - val session = client.login(relayUrl, accessToken ?: throw CloudAccountException(CloudAccountFailure.AUTHENTICATION), deviceId, deviceName, deviceSecret) + val token = AuthorizationPoll.awaitAccessToken(start, log) { client.pollAuthorization(relayUrl, start) } + val session = client.login(relayUrl, token, deviceId, deviceName, deviceSecret) return AccountSessionData( relayUrl = relayUrl, username = session.userId, diff --git a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt index d7b6ce07df..ac21906ac9 100644 --- a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt +++ b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt @@ -21,7 +21,7 @@ import com.openbitfun.mobile.core.persistence.PersistedRemoteMessage import com.openbitfun.mobile.core.persistence.PersistedRemoteSession import com.openbitfun.mobile.core.domain.RemoteSession import com.openbitfun.mobile.core.domain.SessionNaming -import com.openbitfun.mobile.core.domain.SessionAgentTypes +import com.openbitfun.mobile.core.domain.SessionListVisibility import com.openbitfun.mobile.core.domain.TranscriptIntegrityPolicy import com.openbitfun.mobile.core.feature.connection.ConnectionPhase import com.openbitfun.mobile.core.protocol.ActiveTurnSnapshotResponse @@ -57,6 +57,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -375,7 +376,7 @@ public class RemoteSessionStore internal constructor( ) val server = response.sessions .map(RemoteResponseMapper::session) - .filter { SessionAgentTypes.isMobileVisible(it.agentType) } + .filter { SessionListVisibility.isMobileVisible(it) } .map { session -> session.copy(workspacePath = session.workspacePath?.takeIf { it.isNotBlank() } ?: normalizedPath, workspaceIdentity = identity) } @@ -565,6 +566,7 @@ public class RemoteSessionStore internal constructor( }, ) sessionUpdates?.cancel() + transcriptWrite?.cancel() _connectionPhase.value = ConnectionPhase.DISCONNECTED } @@ -708,7 +710,7 @@ public class RemoteSessionStore internal constructor( val response = sendListSessions(pageSize, pageOffset, trimmedQuery, identity) val sessions = response.sessions.map(RemoteResponseMapper::session).map { it.copy(workspaceIdentity = identity) } sessions.filterTo(filtered) { - SessionAgentTypes.isMobileVisible(it.agentType) && filter.matches(it.agentType) + SessionListVisibility.isMobileVisible(it) && filter.matches(it.agentType) } hasMore = response.hasMore pageOffset += sessions.size @@ -734,7 +736,7 @@ public class RemoteSessionStore internal constructor( val known = sessions.mapTo(mutableSetOf()) { it.id } val local = locallyCreatedSessions.values.filter { session -> session.id !in known && - SessionAgentTypes.isMobileVisible(session.agentType) && + SessionListVisibility.isMobileVisible(session) && filter.matches(session.agentType) && (query.isEmpty() || session.title.contains(query, ignoreCase = true) || session.workspaceName.orEmpty().contains(query, ignoreCase = true) || @@ -894,6 +896,29 @@ public class RemoteSessionStore internal constructor( } private var sessionHistoryHasMore = false + private var transcriptWrite: Job? = null + + /** + * Holds the transcript write to one per [TRANSCRIPT_WRITE_DEBOUNCE_MS] while a turn streams. + * + * Every chunk of a streaming reply restates the whole session, and writing + * it re-encrypts and rewrites all of it — dozens of times a second, on the + * thread that draws the screen. Nothing reads the cache until the app is + * reopened, so only the last write of a burst ever mattered. + */ + private fun scheduleTranscriptWrite(sessionId: String) { + if (transcriptWrite?.isActive == true) return + transcriptWrite = scope.launch { + delay(TRANSCRIPT_WRITE_DEBOUNCE_MS) + persistTranscript(sessionId, preserveOlder = sessionHistoryHasMore) + } + } + + private fun writeTranscriptNow(sessionId: String) { + transcriptWrite?.cancel() + transcriptWrite = null + persistTranscript(sessionId, preserveOlder = sessionHistoryHasMore) + } private fun publishDurableTimeline() { val current = _state.value as? RemoteSessionUiState.Ready ?: return @@ -907,6 +932,7 @@ public class RemoteSessionStore internal constructor( val initialHistory = CompletableDeferred() permissionMailbox.select(sessionId) sessionUpdates?.cancel() + transcriptWrite?.cancel() sessionHistoryHasMore = false sessionUpdates = scope.launch { try { @@ -914,10 +940,35 @@ public class RemoteSessionStore internal constructor( val store = relayStreamStore ?: error("Durable session storage unavailable") val records = SessionRecordReplica(sessionId) var caughtUp = false + /** + * Renders everything received so far and reports the turn's phase. + * + * The cost is the whole session's length, so it runs only where + * something can read the result. The initial replay of a long + * session arrives one record at a time and used to render on each + * of them, for a screen fenced behind [caughtUp] that nobody could + * see yet — which is what made the first seconds after an open + * impossible to scroll. + */ + fun render(): ChatSyncPhase { + timelineStore.clearActiveTurn() + val messages = records.messages() + val active = messages.lastOrNull()?.takeIf { it.role == "assistant" && it.status == "streaming" } + timelineStore.setPersistedMessages(if (active == null) messages else messages.dropLast(1)) + timelineStore.setActiveTurn(active) + val phase = when (messages.lastOrNull()?.status) { + "streaming" -> ChatSyncPhase.STREAMING + "failed" -> ChatSyncPhase.ERROR + else -> ChatSyncPhase.IDLE + } + timelineStore.setSyncPhase(phase) + return phase + } source.subscribe(sessionId, PersistentSessionReplica(store, source.streamIdentity + ":session:" + sessionId), { handleFailure(it, _state.value as? RemoteSessionUiState.Ready) }, { caughtUp = true + if (!records.isEmpty) render() publishDurableTimeline() persistTranscript(sessionId, preserveOlder = sessionHistoryHasMore) initialHistory.complete(Unit) @@ -932,17 +983,15 @@ public class RemoteSessionStore internal constructor( val kind = (payload["toolEvent"] as? JsonObject)?.get("event_type")?.jsonPrimitive?.content if (kind in setOf("ConfirmationNeeded", "Confirmed", "Rejected", "Cancelled")) permissionMailbox.invalidate() } - timelineStore.clearActiveTurn() - val messages = records.messages() - val active = messages.lastOrNull()?.takeIf { it.role == "assistant" && it.status == "streaming" } - timelineStore.setPersistedMessages(if (active == null) messages else messages.dropLast(1)) - timelineStore.setActiveTurn(active) - timelineStore.setSyncPhase(when (messages.lastOrNull()?.status) { - "streaming" -> ChatSyncPhase.STREAMING - "failed" -> ChatSyncPhase.ERROR - else -> ChatSyncPhase.IDLE - }) - if (caughtUp) { publishDurableTimeline(); persistTranscript(sessionId, preserveOlder = sessionHistoryHasMore) } + if (caughtUp) { + val phase = render() + publishDurableTimeline() + // A streaming turn rewrites the same rows on every + // chunk; anything else is a settled transcript worth + // keeping now. + if (phase == ChatSyncPhase.STREAMING) scheduleTranscriptWrite(sessionId) + else writeTranscriptNow(sessionId) + } } "relay://session-resumed", "session-interaction-changed" -> permissionMailbox.invalidate() "relay://session-ready" -> { @@ -1281,6 +1330,7 @@ public class RemoteSessionStore internal constructor( val closingOpenSession = current.selectedSessionId == normalized if (closingOpenSession) { sessionUpdates?.cancel() + transcriptWrite?.cancel() timelineStore.reset("") } if (!isCurrentWork(operationToken)) return@launch @@ -1853,6 +1903,9 @@ public class RemoteSessionStore internal constructor( /** One screenful of sessions. Matches the desktop's own `list_sessions` default. */ private const val PAGE_SIZE: Int = 30 + /** Long enough to swallow a burst of stream chunks, short enough to lose nothing on a kill. */ + private const val TRANSCRIPT_WRITE_DEBOUNCE_MS = 400L + /** * Window used while narrowing by agent type. The desktop clamps `limit` * to 100, so asking for more only costs round trips. @@ -1928,6 +1981,10 @@ internal object RemoteResponseMapper { messageCount = item.messageCount ?: 0, workspacePath = item.workspacePath, workspaceName = item.workspaceName, + // Callers attach the identity of the workspace they listed under. + workspaceIdentity = null, + parentSessionId = item.parentSessionId, + relationshipKind = item.relationshipKind, ) } diff --git a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplica.kt b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplica.kt index 92ff7a42b8..e3be672b41 100644 --- a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplica.kt +++ b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplica.kt @@ -6,11 +6,19 @@ import kotlinx.serialization.json.* /** Revisioned canonical records retain the complete host DTO; rendering is a mobile adapter. */ internal class SessionRecordReplica(private val sessionId: String) { - private data class Versioned(val revision: Long, val value: JsonObject) + private data class Versioned(val revision: Long, val value: JsonObject, val authoritative: Boolean = false) private val turns = linkedMapOf() private val rounds = linkedMapOf() private val items = linkedMapOf() private val controls = mutableMapOf>() + + /** + * True until the stream has delivered a record of its own. + * + * An empty replica has nothing to say about the transcript, and rendering it + * would erase a timeline the cache or the history fetch already filled. + */ + val isEmpty: Boolean get() = turns.isEmpty() fun applyControl(payload: JsonObject) { val turn = payload.string("turnId") val event = payload["toolEvent"] as? JsonObject ?: return @@ -27,6 +35,15 @@ internal class SessionRecordReplica(private val sessionId: String) { private fun put(map: MutableMap, id: String, revision: Long, value: JsonObject) { if (revision > (map[id]?.revision ?: -1)) map[id] = Versioned(revision, value) } + /** A round or item record repeats its parent turn so a partial page still reads, + * but that copy is a header, not the turn's own state: the turn record is what the + * user message and its attachment pixels come from, so a header fills a turn it has + * never seen and never overwrites one. */ + private fun putTurn(id: String, revision: Long, value: JsonObject, authoritative: Boolean) { + val stored = turns[id] + if (stored != null && (revision <= stored.revision || (!authoritative && stored.authoritative))) return + turns[id] = Versioned(revision, value, authoritative) + } fun apply(payload: JsonObject) { check(payload.string("sessionId") == sessionId) { "Session record binding mismatch" } val revision = payload.getValue("revision").jsonPrimitive.long @@ -61,7 +78,7 @@ internal class SessionRecordReplica(private val sessionId: String) { // Validate the complete ancestry before changing either data or replay fences. // A rejected record must leave a corrected delivery at the same revision usable. recordVersions[recordId] = revision - put(turns, turnId, revision, turn) + putTurn(turnId, revision, turn, recordRound == null) if (turns[turnId]?.value?.string("status") != "inprogress") controls.entries.removeAll { it.value.first == turnId } if (recordRound != null && roundId != null) { put(rounds, roundId, revision, recordRound) @@ -72,42 +89,80 @@ internal class SessionRecordReplica(private val sessionId: String) { } } - fun messages(): List = turns.filter { (id, record) -> record.revision > (tombstones["turn/$id"] ?: -1) }.values.sortedBy { it.value.number("turnIndex") }.flatMap { record -> - val turn = record.value - val turnId = turn.string("turnId") - val user = turn.getValue("userMessage").jsonObject - val turnFence = tombstones["turn/$turnId"] ?: -1 - val children = rounds.filter { (id, record) -> record.revision > maxOf(turnFence, tombstones["round/$id"] ?: -1) }.values.map { it.value }.filter { it.string("turnId") == turnId } - .sortedBy { it.number("roundIndex") }.flatMap { round -> - items.filter { (id, record) -> itemRounds[id] == round.string("id") && record.revision > maxOf(turnFence, tombstones["round/" + round.string("id")] ?: -1, tombstones["item/$id"] ?: -1) }.values.map { it.value } - .filter { it.getValue("data").jsonObject.string("status") !in setOf("superseded", "retry_superseded") } - .sortedWith(compareBy({ it.getValue("data").jsonObject.number("orderIndex") }, { it.getValue("data").jsonObject.number("timestamp") })) + /** + * Groups every round and item under its parent once per rebuild. + * + * Reading the transcript used to filter the whole round map for each turn + * and the whole item map for each round, so a session paid for its own + * length twice over on every record that arrived. Long sessions spend that + * cost on the thread that draws them, which is where the scrolling went. + */ + private data class Children( + val roundsByTurn: Map>>, + val itemsByRound: Map>>, + ) + + private fun children(): Children { + val roundsByTurn = mutableMapOf>>() + rounds.forEach { (id, record) -> + roundsByTurn.getOrPut(record.value.string("turnId")) { mutableListOf() }.add(id to record) + } + val itemsByRound = mutableMapOf>>() + items.forEach { (id, record) -> + val roundId = itemRounds[id] ?: return@forEach + itemsByRound.getOrPut(roundId) { mutableListOf() }.add(id to record) + } + return Children(roundsByTurn, itemsByRound) + } + + fun messages(): List { + val index = children() + return turns.filter { (id, record) -> record.revision > (tombstones["turn/$id"] ?: -1) }.values.sortedBy { it.value.number("turnIndex") }.flatMap { record -> + val turn = record.value + val turnId = turn.string("turnId") + val user = turn.getValue("userMessage").jsonObject + val turnFence = tombstones["turn/$turnId"] ?: -1 + val children = index.roundsByTurn[turnId].orEmpty().filter { (id, record) -> record.revision > maxOf(turnFence, tombstones["round/$id"] ?: -1) } + .sortedBy { it.second.value.number("roundIndex") }.flatMap { (currentRoundId, _) -> + val roundFence = maxOf(turnFence, tombstones["round/$currentRoundId"] ?: -1) + index.itemsByRound[currentRoundId].orEmpty().filter { (id, record) -> record.revision > maxOf(roundFence, tombstones["item/$id"] ?: -1) }.map { it.second.value } + .filter { it.getValue("data").jsonObject.string("status") !in setOf("superseded", "retry_superseded") } + .sortedWith(compareBy({ it.getValue("data").jsonObject.number("orderIndex") }, { it.getValue("data").jsonObject.number("timestamp") })) + } + val rendered = children.map { item -> + val data = item.getValue("data").jsonObject + val type = item.string("type") + val result = data["toolResult"] as? JsonObject + ChatMessageItemResponse(type = type, content = data.string("content"), isSubagent = data["isSubagentItem"]?.jsonPrimitive?.booleanOrNull == true || data.string("subagentSessionId").isNotEmpty(), + tool = if (type != "tool") null else RemoteToolStatusResponse( + id = (data["toolCall"] as? JsonObject)?.string("id")?.takeIf { it.isNotEmpty() } ?: data.string("id"), name = data.string("toolName"), + status = data.string("status").takeIf { it.isNotEmpty() } + ?: if (result == null) "running" else if (result["success"]?.jsonPrimitive?.booleanOrNull == true) "completed" else "failed", + toolInput = (data["toolCall"] as? JsonObject)?.get("input"), toolOutput = result?.get("result"), + errorPreview = result?.string("error"), + startMs = data["startTime"]?.jsonPrimitive?.longOrNull, + durationMs = data["durationMs"]?.jsonPrimitive?.longOrNull + ?: result?.get("durationMs")?.jsonPrimitive?.longOrNull)) } - val rendered = children.map { item -> - val data = item.getValue("data").jsonObject - val type = item.string("type") - val result = data["toolResult"] as? JsonObject - ChatMessageItemResponse(type = type, content = data.string("content"), isSubagent = data["isSubagentItem"]?.jsonPrimitive?.booleanOrNull == true || data.string("subagentSessionId").isNotEmpty(), - tool = if (type != "tool") null else RemoteToolStatusResponse( - id = (data["toolCall"] as? JsonObject)?.string("id")?.takeIf { it.isNotEmpty() } ?: data.string("id"), name = data.string("toolName"), - status = data.string("status").takeIf { it.isNotEmpty() } - ?: if (result == null) "running" else if (result["success"]?.jsonPrimitive?.booleanOrNull == true) "completed" else "failed", - toolInput = (data["toolCall"] as? JsonObject)?.get("input"), toolOutput = result?.get("result"), - errorPreview = result?.string("error"), - startMs = data["startTime"]?.jsonPrimitive?.longOrNull, - durationMs = data["durationMs"]?.jsonPrimitive?.longOrNull - ?: result?.get("durationMs")?.jsonPrimitive?.longOrNull)) + rendered.mapNotNull { it.tool }.filter { it.status in setOf("completed", "failed", "cancelled", "rejected", "skipped") }.forEach { controls.remove(it.id) } + val controlTools = controls.values.filter { it.first == turnId }.map { it.second } + val shownItems = rendered.map { item -> item.tool?.id?.let { id -> controlTools.firstOrNull { it.id == id } }?.let { item.copy(tool = it) } ?: item } + + controlTools.filter { tool -> rendered.none { it.tool?.id == tool.id } }.map { ChatMessageItemResponse(type = "tool", tool = it) } + listOf(RemoteResponseMapper.chatMessage(ChatMessageResponse(id = user.string("id"), role = "user", content = user.string("content"), turnId = turnId, metadata = user["metadata"], images = userImages(user), timestamp = user.string("timestamp"))), + RemoteResponseMapper.chatMessage(ChatMessageResponse(id = "${turnId}_assistant", role = "assistant", turnId = turnId, + content = rendered.filter { it.type == "text" && it.isSubagent != true }.joinToString("") { it.content.orEmpty() }, + thinking = rendered.filter { it.type == "thinking" && it.isSubagent != true }.joinToString("") { it.content.orEmpty() }, + items = shownItems, status = when (turn.string("status")) { "inprogress" -> "streaming"; "error" -> "failed"; else -> turn.string("status") }, error = turn.string("error"), metadata = turn))) } - rendered.mapNotNull { it.tool }.filter { it.status in setOf("completed", "failed", "cancelled", "rejected", "skipped") }.forEach { controls.remove(it.id) } - val controlTools = controls.values.filter { it.first == turnId }.map { it.second } - val shownItems = rendered.map { item -> item.tool?.id?.let { id -> controlTools.firstOrNull { it.id == id } }?.let { item.copy(tool = it) } ?: item } + - controlTools.filter { tool -> rendered.none { it.tool?.id == tool.id } }.map { ChatMessageItemResponse(type = "tool", tool = it) } - listOf(RemoteResponseMapper.chatMessage(ChatMessageResponse(id = user.string("id"), role = "user", content = user.string("content"), turnId = turnId, metadata = user["metadata"], timestamp = user.string("timestamp"))), - RemoteResponseMapper.chatMessage(ChatMessageResponse(id = "${turnId}_assistant", role = "assistant", turnId = turnId, - content = rendered.filter { it.type == "text" && it.isSubagent != true }.joinToString("") { it.content.orEmpty() }, - thinking = rendered.filter { it.type == "thinking" && it.isSubagent != true }.joinToString("") { it.content.orEmpty() }, - items = shownItems, status = when (turn.string("status")) { "inprogress" -> "streaming"; "error" -> "failed"; else -> turn.string("status") }, error = turn.string("error"), metadata = turn))) } + + /** Attachments are recorded with the turn; one that kept only a host path has no + * pixels to hand a client that cannot reach that filesystem. */ + private fun userImages(user: JsonObject): List = + (((user["metadata"] as? JsonObject)?.get("images") as? JsonArray) ?: emptyList()).mapNotNull { entry -> + val image = entry as? JsonObject ?: return@mapNotNull null + image.string("data_url").takeIf { it.isNotEmpty() }?.let { ImageAttachment(name = image.string("name"), dataUrl = it) } + } private fun JsonObject.string(key: String): String = (get(key) as? JsonPrimitive)?.contentOrNull.orEmpty() private fun JsonObject.number(key: String): Long = (get(key) as? JsonPrimitive)?.longOrNull ?: 0 } diff --git a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentation.kt b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentation.kt index 8df074993b..00bda5ff8c 100644 --- a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentation.kt +++ b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentation.kt @@ -5,6 +5,8 @@ import com.openbitfun.mobile.core.domain.RemoteWorkspaceIdentity import com.openbitfun.mobile.core.domain.identity import com.openbitfun.mobile.core.domain.belongsTo import com.openbitfun.mobile.core.feature.session.RemoteSessionUiState +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryStatus +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryUiState import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceUiState import com.openbitfun.mobile.core.feature.workspace.projectWorkspaceCatalog @@ -15,6 +17,21 @@ public data class RemoteSidebarSessionRow public constructor( public val agentType: String, ) +/** + * How far the sidebar has got in learning a workspace's own sessions. + * + * `list_sessions` answers for one workspace at a time, so only the selected + * workspace's rows arrive with the session list. Every other branch stays + * [IDLE] until it is expanded, and an empty [IDLE] branch means "not asked + * yet", not "no sessions" — the difference the sidebar has to show. + */ +public enum class RemoteSidebarWorkspaceLoad { + IDLE, + LOADING, + READY, + FAILED, +} + /** One remote workspace and the sessions filed under it in the sidebar tree. */ public data class RemoteSidebarWorkspaceRow public constructor( public val path: String, @@ -23,6 +40,7 @@ public data class RemoteSidebarWorkspaceRow public constructor( public val sessions: List, public val remoteConnectionId: String?, public val remoteSshHost: String?, + public val load: RemoteSidebarWorkspaceLoad = RemoteSidebarWorkspaceLoad.READY, ) { public val key: String get() = RemoteWorkspaceIdentity(path, remoteConnectionId, remoteSshHost).key public constructor(path: String, name: String, selected: Boolean, sessions: List, remoteConnectionId: String?) : this(path, name, selected, sessions, remoteConnectionId, null) @@ -39,31 +57,79 @@ public object RemoteSidebarPresentation { public fun workspacesForSessions( workspaceState: RemoteWorkspaceUiState.Ready?, sessions: List, + ): List = project(workspaceState, sessions, null) + + /** + * The sidebar tree with each branch's own `list_sessions` result folded in. + * + * [sessions] only ever describes the selected workspace, so every other + * branch takes its rows and its status from [directory], which is what + * `RemoteSessionIntent.LoadWorkspaceSessions` fills in on expand. + */ + public fun workspacesWithDirectory( + workspaceState: RemoteWorkspaceUiState.Ready?, + sessionState: RemoteSessionUiState.Ready?, + directory: WorkspaceSessionDirectoryUiState?, + ): List = project(workspaceState, sessionState?.sessions.orEmpty(), directory) + + /** + * [workspacesWithDirectory] for a caller that already flattened its session + * list, which is the shape iOS keeps its projection in. + */ + public fun workspacesForSessionsWithDirectory( + workspaceState: RemoteWorkspaceUiState.Ready?, + sessions: List, + directory: WorkspaceSessionDirectoryUiState?, + ): List = project(workspaceState, sessions, directory) + + private fun project( + workspaceState: RemoteWorkspaceUiState.Ready?, + sessions: List, + directory: WorkspaceSessionDirectoryUiState?, ): List { if (workspaceState == null) return emptyList() val selected = workspaceState.selected val workspaceRows = (workspaceState.catalog ?: projectWorkspaceCatalog( workspaceState.workspaces, workspaceState.assistants, )).workspaces + val catalogIdentities = workspaceRows.map { row -> row.identity() } return workspaceRows.map { workspace -> - val path = workspace.path - val connectionId = workspace.remoteConnectionId + val identity = workspace.identity() + val isSelected = selected != null && identity.matches( + RemoteWorkspaceIdentity(selected.path, selected.remoteConnectionId, selected.remoteSshHost)) + val bucketed = sessions.filter { it.belongsTo(identity, catalogIdentities) } + val branch = directory?.workspace(workspace.path, workspace.remoteConnectionId, workspace.remoteSshHost) + // The selected workspace is the one `list_sessions` was asked about, + // so its bucket is authoritative even before any branch was loaded. + val load = when { + directory == null || isSelected -> RemoteSidebarWorkspaceLoad.READY + branch == null -> RemoteSidebarWorkspaceLoad.IDLE + else -> when (branch.status) { + WorkspaceSessionDirectoryStatus.IDLE -> RemoteSidebarWorkspaceLoad.IDLE + WorkspaceSessionDirectoryStatus.LOADING -> RemoteSidebarWorkspaceLoad.LOADING + WorkspaceSessionDirectoryStatus.READY -> RemoteSidebarWorkspaceLoad.READY + WorkspaceSessionDirectoryStatus.FAILED -> RemoteSidebarWorkspaceLoad.FAILED + } + } + val owned = if (!isSelected && branch != null && branch.sessions.isNotEmpty()) { + branch.sessions + } else { + bucketed + } RemoteSidebarWorkspaceRow( - path = path, + path = workspace.path, name = workspace.name, - selected = selected != null && workspace.identity().matches( - RemoteWorkspaceIdentity(selected.path, selected.remoteConnectionId, selected.remoteSshHost)), - remoteConnectionId = connectionId, + selected = isSelected, + remoteConnectionId = workspace.remoteConnectionId, remoteSshHost = workspace.remoteSshHost, - sessions = sessions - .filter { it.belongsTo(workspace.identity(), workspaceRows.map { row -> row.identity() }) } - .map { session -> - RemoteSidebarSessionRow( - id = session.id, - title = session.title, - agentType = session.agentType, - ) - }, + load = load, + sessions = owned.map { session -> + RemoteSidebarSessionRow( + id = session.id, + title = session.title, + agentType = session.agentType, + ) + }, ) } } diff --git a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/account/AccountStoreTest.kt b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/account/AccountStoreTest.kt index 67e01e0dab..12f85a4dd3 100644 --- a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/account/AccountStoreTest.kt +++ b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/account/AccountStoreTest.kt @@ -5,6 +5,10 @@ import com.openbitfun.mobile.core.protocol.CommandStatus import com.openbitfun.mobile.core.protocol.RemoteCommand import com.openbitfun.mobile.core.transport.CloudAccountException import com.openbitfun.mobile.core.transport.CloudAccountFailure +import com.openbitfun.mobile.core.transport.GitHubAuthorization +import com.openbitfun.mobile.core.transport.GitHubAuthorizationPoll +import com.openbitfun.mobile.core.transport.GitHubTokens +import com.openbitfun.mobile.core.transport.TransportLog import com.openbitfun.mobile.core.transport.RemoteCommandTransport import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.advanceUntilIdle @@ -12,6 +16,7 @@ import kotlinx.coroutines.test.runTest import kotlinx.serialization.DeserializationStrategy import kotlin.test.Test import kotlin.test.assertContentEquals +import kotlin.test.assertFailsWith import kotlinx.coroutines.test.advanceTimeBy import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -21,6 +26,52 @@ import kotlin.test.assertTrue @OptIn(ExperimentalCoroutinesApi::class) class AccountStoreTest { + /** + * The sign-in poll runs for the minutes the user spends in a browser and a + * mail app, which is where a phone most reliably drops a connection. A + * single failure there used to end the sign-in and send them back to the + * start, so the loop keeps asking until the window closes and stops early + * only for a refusal the relay actually meant. + */ + @Test fun signInPollOutlastsNetworkFailuresButNotARefusal() = runTest { + val start = GitHubAuthorization("txn", "secret", "https://auth.openbitfun.com/sign-in#ticket=t", 9_999_999_999L, 3) + + var attempts = 0 + val token = AuthorizationPoll.awaitAccessToken(start, TransportLog.None, nowSeconds = { 0 }) { + attempts++ + when (attempts) { + 1 -> throw CloudAccountException(CloudAccountFailure.NETWORK) + 2 -> throw CloudAccountException(CloudAccountFailure.TIMEOUT) + 3 -> throw CloudAccountException(CloudAccountFailure.RATE_LIMITED, 429) + 4 -> GitHubAuthorizationPoll("pending") + else -> GitHubAuthorizationPoll("authorized", GitHubTokens("granted")) + } + } + assertEquals("granted", token) + assertEquals(5, attempts) + + var refusals = 0 + assertFailsWith { + AuthorizationPoll.awaitAccessToken(start, TransportLog.None, nowSeconds = { 0 }) { + refusals++ + throw CloudAccountException(CloudAccountFailure.AUTHENTICATION, 401) + } + } + assertEquals(1, refusals) + + // A window that closed while every poll was failing is a network + // problem, and saying "authentication" would send the user looking in + // the wrong place. + var elapsed = 0L + val expired = assertFailsWith { + AuthorizationPoll.awaitAccessToken(start.copy(expiresAt = 9L), TransportLog.None, nowSeconds = { elapsed }) { + elapsed += 3 + throw CloudAccountException(CloudAccountFailure.NETWORK) + } + } + assertEquals(CloudAccountFailure.NETWORK, expired.failure) + } + @Test fun cancelledLoginDirectoryCannotReviveAccountOrPersistSelectedDevice() = runTest { for (failure in listOf(null, CloudAccountException(CloudAccountFailure.NETWORK), CloudAccountException(CloudAccountFailure.AUTHENTICATION), IllegalStateException("Late failure"))) { diff --git a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionPersistenceTest.kt b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionPersistenceTest.kt index 82dd467bb2..8d73519699 100644 --- a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionPersistenceTest.kt +++ b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionPersistenceTest.kt @@ -172,6 +172,46 @@ class RemoteSessionPersistenceTest { runCurrent(); reopened.stop() } + @Test + fun replayRebuildsTheTranscriptOnceInsteadOfPerRecord() = runTest { + // A long session replays record by record, and rebuilding the whole + // transcript on each of them is what made the first seconds after an + // open impossible to scroll. Nothing can read the result before + // catch-up, so one rebuild and one write is the whole job. + val stores = MemoryPersistence() + val transport = PersistenceTransport().apply { + initialRecords = (0 until 8).map { richRecord("server", "t-$it", it, 1, "completed", "msg $it") } + } + val store = RemoteSessionStore.create(this, transport, "device-a", stores.stores) + store.dispatch(RemoteSessionIntent.Open("server")); runCurrent() + assertEquals(1, stores.transcripts.replaces) + val timeline = assertIs(store.state.value).timeline + assertEquals("msg 7", timeline?.persistedMessages?.last()?.text) + store.stop() + } + + @Test + fun streamingChunksShareOneTranscriptWriteAndSettleImmediately() = runTest { + val stores = MemoryPersistence() + val transport = PersistenceTransport().apply { + initialRecords = listOf(richRecord("server", "t-1", 0, 1, "completed", "done")) + } + val store = RemoteSessionStore.create(this, transport, "device-a", stores.stores) + store.dispatch(RemoteSessionIntent.Open("server")); runCurrent() + val afterOpen = stores.transcripts.replaces + (2..6).forEach { revision -> + transport.records.emit(richRecord("server", "t-2", 1, revision.toLong(), "inprogress", "chunk $revision")) + runCurrent() + } + assertEquals(afterOpen, stores.transcripts.replaces) + advanceTimeBy(600); runCurrent() + assertEquals(afterOpen + 1, stores.transcripts.replaces) + transport.records.emit(richRecord("server", "t-2", 1, 7, "completed", "chunk done")); runCurrent() + assertEquals(afterOpen + 2, stores.transcripts.replaces) + assertEquals("chunk done", stores.transcripts.rows.getValue("device-a::server").last().text) + store.stop() + } + @Test fun streamDisconnectRetainsPersistedTranscriptAndRecovers() = runTest { val stores = MemoryPersistence() @@ -433,9 +473,10 @@ private class MemorySessions : RemoteSessionListStore { private class MemoryTranscripts : RemoteTranscriptStore { val rows = mutableMapOf>() val cursors = mutableMapOf() + var replaces = 0 override fun load(deviceKey: String, sessionId: String) = rows["$deviceKey::$sessionId"].orEmpty() override fun append(deviceKey: String, sessionId: String, startSeq: Int, messages: List) = Unit - override fun replace(deviceKey: String, sessionId: String, messages: List) { rows["$deviceKey::$sessionId"] = messages } + override fun replace(deviceKey: String, sessionId: String, messages: List) { replaces++; rows["$deviceKey::$sessionId"] = messages } override fun loadCursor(deviceKey: String, sessionId: String) = cursors["$deviceKey::$sessionId"] override fun saveCursor(deviceKey: String, sessionId: String, cursor: PersistedRemoteCursor) { cursors["$deviceKey::$sessionId"] = cursor } override fun delete(deviceKey: String, sessionId: String) { diff --git a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplicaTest.kt b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplicaTest.kt index 4e987dbf25..f7b8c85dee 100644 --- a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplicaTest.kt +++ b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/SessionRecordReplicaTest.kt @@ -24,6 +24,35 @@ class SessionRecordReplicaTest { assertEquals("hello world", messages[1].text) assertEquals("completed", messages[1].status) } + @Test fun turnRecordUserMessageOutlivesTheHeadersItsChildrenRepeat() { + val replica = SessionRecordReplica("s") + replica.apply(buildJsonObject { + put("sessionId", "s"); put("id", "turn/t"); put("revision", 1) + put("turn", buildJsonObject { + put("sessionId", "s"); put("turnId", "t"); put("turnIndex", 0); put("status", "inprogress") + put("userMessage", buildJsonObject { + put("id", "u"); put("content", "look at this"); put("timestamp", 1) + put("metadata", buildJsonObject { + put("images", buildJsonArray { + add(buildJsonObject { put("name", "shot.png"); put("data_url", "data:image/png;base64,AA") }) + add(buildJsonObject { put("name", "unreachable.png"); put("image_path", "/Users/dev/gone.png") }) + }) + }) + }) + }) + }) + // Attachments are drawn from the pixels recorded with the turn; one that + // kept only a host path has none to hand a client that cannot reach it. + assertEquals(listOf("shot.png"), replica.messages()[0].images.orEmpty().map { it.name }) + + // A round or item record repeats the turn as a parent header, without + // those pixels. It must not replace the turn record it descends from. + replica.apply(record(5, "inprogress", "on it")) + val messages = replica.messages() + assertEquals("look at this", messages[0].text) + assertEquals(listOf("data:image/png;base64,AA"), messages[0].images.orEmpty().map { it.dataUrl }) + assertEquals("on it", messages[1].text) + } @Test fun tombstonesPreventOldReplayResurrection() { val replica = SessionRecordReplica("s") replica.apply(record(3, "completed", "final")) diff --git a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentationTest.kt b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentationTest.kt index 9ef487527c..b0d80f010f 100644 --- a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentationTest.kt +++ b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/shell/RemoteSidebarPresentationTest.kt @@ -5,6 +5,9 @@ import com.openbitfun.mobile.core.domain.RemoteSession import com.openbitfun.mobile.core.domain.SelectedWorkspace import com.openbitfun.mobile.core.feature.session.RemoteSessionUiState import com.openbitfun.mobile.core.feature.session.SessionAgentFilter +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryEntry +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryStatus +import com.openbitfun.mobile.core.feature.session.WorkspaceSessionDirectoryUiState import com.openbitfun.mobile.core.feature.workspace.RemoteFilePreviewUiState import com.openbitfun.mobile.core.feature.workspace.RemoteWorkspaceUiState import kotlin.test.Test @@ -65,6 +68,96 @@ class RemoteSidebarPresentationTest { assertTrue(RemoteSidebarPresentation.workspacesForSessions(empty, sessions).isEmpty()) } + @Test + fun undisclosedBranchesReadAsUnaskedRatherThanEmpty() { + // `list_sessions` answers for the selected workspace only, so a branch + // nobody expanded has no rows for a reason the reader has to be told. + val rows = RemoteSidebarPresentation.workspacesWithDirectory( + workspaceState = workspaceReady( + selected = selected("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/repo/b", "Beta"), + recent = listOf(recent("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/repo/a", "Alpha"), recent("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/repo/b", "Beta")), + ), + sessionState = sessionReady(listOf(session("b-1", "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/repo/b", "code"))), + directory = WorkspaceSessionDirectoryUiState(emptyList()), + ) + + assertEquals(RemoteSidebarWorkspaceLoad.IDLE, rows.first().load) + assertTrue(rows.first().sessions.isEmpty()) + // The selected branch is the one that was asked about; it is not IDLE. + assertEquals(RemoteSidebarWorkspaceLoad.READY, rows.last().load) + assertEquals(listOf("b-1"), rows.last().sessions.map { it.id }) + } + + @Test + fun disclosedBranchTakesItsRowsAndStatusFromTheDirectory() { + val workspace = workspaceReady( + selected = selected("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/repo/b", "Beta"), + recent = listOf(recent("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/repo/a", "Alpha"), recent("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/repo/b", "Beta")), + ) + val sessions = sessionReady(listOf(session("b-1", "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/repo/b", "code"))) + + val loading = RemoteSidebarPresentation.workspacesWithDirectory( + workspace, sessions, + directory(entry("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/repo/a", WorkspaceSessionDirectoryStatus.LOADING, emptyList())), + ) + assertEquals(RemoteSidebarWorkspaceLoad.LOADING, loading.first().load) + + val ready = RemoteSidebarPresentation.workspacesWithDirectory( + workspace, sessions, + directory(entry("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/repo/a", WorkspaceSessionDirectoryStatus.READY, listOf(session("a-1", "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/repo/a", "chat")))), + ) + assertEquals(RemoteSidebarWorkspaceLoad.READY, ready.first().load) + assertEquals(listOf("a-1"), ready.first().sessions.map { it.id }) + + val failed = RemoteSidebarPresentation.workspacesWithDirectory( + workspace, sessions, + directory(entry("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/repo/a", WorkspaceSessionDirectoryStatus.FAILED, emptyList())), + ) + assertEquals(RemoteSidebarWorkspaceLoad.FAILED, failed.first().load) + } + + @Test + fun directoryRowsDoNotLeakAcrossHostsSharingAPath() { + val a = recent("/repo", "A").copy(remoteConnectionId = "saved", remoteSshHost = "host-a") + val b = a.copy(name = "B", remoteSshHost = "host-b") + val ready = workspaceReady( + selected("/repo", "Selected").copy(remoteConnectionId = "saved", remoteSshHost = "host-b"), + listOf(a, b), + ) + val rows = RemoteSidebarPresentation.workspacesWithDirectory( + ready, sessionReady(emptyList()), + directory( + entry("/repo", WorkspaceSessionDirectoryStatus.READY, listOf(session("a-1", "/repo", "code"))) + .copy(remoteConnectionId = "saved", remoteSshHost = "host-a"), + ), + ) + + assertEquals(listOf("a-1"), rows.first().sessions.map { it.id }) + assertEquals(RemoteSidebarWorkspaceLoad.READY, rows.first().load) + // The other host shares the path but was never disclosed. + assertEquals(RemoteSidebarWorkspaceLoad.READY, rows.last().load) + assertTrue(rows.last().sessions.isEmpty()) + } + + @Test + fun theDirectorylessProjectionStaysReady() { + val rows = RemoteSidebarPresentation.workspaces( + workspaceState = workspaceReady(selected("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/repo/b", "Beta"), listOf(recent("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/repo/a", "Alpha"))), + sessionState = sessionReady(emptyList()), + ) + + assertEquals(listOf(RemoteSidebarWorkspaceLoad.READY), rows.map { it.load }) + } + + private fun directory(vararg entries: WorkspaceSessionDirectoryEntry) = + WorkspaceSessionDirectoryUiState(entries.toList()) + + private fun entry( + path: String, + status: WorkspaceSessionDirectoryStatus, + sessions: List, + ) = WorkspaceSessionDirectoryEntry(path, status, sessions, null, null) + private fun workspaceReady( selected: SelectedWorkspace, recent: List, diff --git a/src/apps/mobile/shared/core-protocol/src/commonMain/kotlin/com/openbitfun/mobile/core/protocol/SessionDtos.kt b/src/apps/mobile/shared/core-protocol/src/commonMain/kotlin/com/openbitfun/mobile/core/protocol/SessionDtos.kt index b72b75deba..edf290b64f 100644 --- a/src/apps/mobile/shared/core-protocol/src/commonMain/kotlin/com/openbitfun/mobile/core/protocol/SessionDtos.kt +++ b/src/apps/mobile/shared/core-protocol/src/commonMain/kotlin/com/openbitfun/mobile/core/protocol/SessionDtos.kt @@ -39,6 +39,9 @@ public data class SessionItemResponse( val messageCount: Int? = null, val workspacePath: String? = null, val workspaceName: String? = null, + /** Set when the session hangs off another one (btw/review/miniapp/subagent). */ + val parentSessionId: String? = null, + val relationshipKind: String? = null, ) public object SessionItemResponseSerializer : KSerializer { @@ -56,6 +59,8 @@ public object SessionItemResponseSerializer : KSerializer { messageCount = json.wireInt("message_count"), workspacePath = json.wireString("workspace_path"), workspaceName = json.wireString("workspace_name"), + parentSessionId = json.wireString("parent_session_id"), + relationshipKind = json.wireString("relationship_kind"), ) } @@ -77,6 +82,8 @@ public object SessionItemResponseSerializer : KSerializer { value.messageCount?.let { put("message_count", it) } value.workspacePath?.let { put("workspace_path", it) } value.workspaceName?.let { put("workspace_name", it) } + value.parentSessionId?.let { put("parent_session_id", it) } + value.relationshipKind?.let { put("relationship_kind", it) } }, ) } diff --git a/src/apps/mobile/shared/core-protocol/src/commonTest/kotlin/com/openbitfun/mobile/core/protocol/SessionItemContractTest.kt b/src/apps/mobile/shared/core-protocol/src/commonTest/kotlin/com/openbitfun/mobile/core/protocol/SessionItemContractTest.kt index 90495f8e96..0ed234982b 100644 --- a/src/apps/mobile/shared/core-protocol/src/commonTest/kotlin/com/openbitfun/mobile/core/protocol/SessionItemContractTest.kt +++ b/src/apps/mobile/shared/core-protocol/src/commonTest/kotlin/com/openbitfun/mobile/core/protocol/SessionItemContractTest.kt @@ -27,7 +27,7 @@ class SessionItemContractTest { fun fixtureCoversMoreThanTheHappyPath() { // A fixture file that shrinks to one canonical case stops being a drift // guard, so the count is part of the contract. - assertTrue(cases.size >= 6, "expected the alias and fallthrough cases, got ${cases.size}") + assertTrue(cases.size >= 7, "expected the alias and fallthrough cases, got ${cases.size}") } @Test @@ -56,6 +56,16 @@ class SessionItemContractTest { decoded.workspaceName, "$name: workspaceName", ) + assertEquals( + expected.optionalString("parentSessionId"), + decoded.parentSessionId, + "$name: parentSessionId", + ) + assertEquals( + expected.optionalString("relationshipKind"), + decoded.relationshipKind, + "$name: relationshipKind", + ) } } diff --git a/src/crates/assembly/core/src/service_agent_runtime.rs b/src/crates/assembly/core/src/service_agent_runtime.rs index 03d218939a..95b2968afd 100644 --- a/src/crates/assembly/core/src/service_agent_runtime.rs +++ b/src/crates/assembly/core/src/service_agent_runtime.rs @@ -614,15 +614,40 @@ async fn load_remote_session_metadata_for_workspace( format!("Failed to list sessions for workspace: {error}") })?; + use openbitfun_services_core::session::SessionRelationshipKind; + Ok(metadata .into_iter() - .map(|session| RemoteSessionMetadata { - session_id: session.session_id, - name: session.session_name, - agent_type: session.agent_type, - created_at_ms: session.created_at, - last_active_at_ms: session.last_active_at, - turn_count: session.turn_count, + .map(|session| { + // Legacy sessions keep their lineage in `custom_metadata`, so the + // normalized view is the only source that sees every child session. + let relationship = + openbitfun_services_core::session::normalized_session_relationship(&session); + let relationship_kind = relationship + .as_ref() + .and_then(|relationship| relationship.kind.as_ref()) + .map(|kind| { + match kind { + SessionRelationshipKind::Btw => "btw", + SessionRelationshipKind::Review => "review", + SessionRelationshipKind::DeepReview => "deep_review", + SessionRelationshipKind::Miniapp => "miniapp", + SessionRelationshipKind::Subagent => "subagent", + } + .to_string() + }); + let parent_session_id = + relationship.and_then(|relationship| relationship.parent_session_id); + RemoteSessionMetadata { + session_id: session.session_id, + name: session.session_name, + agent_type: session.agent_type, + created_at_ms: session.created_at, + last_active_at_ms: session.last_active_at, + turn_count: session.turn_count, + parent_session_id, + relationship_kind, + } }) .collect()) } @@ -660,24 +685,58 @@ fn remote_model_capability_fact(capability: ModelCapability) -> RemoteModelCapab } } +/// An attachment recorded before pixels travelled inline holds a host path and +/// nothing else, and only this host can resolve it. Above this size the image is +/// left as a name: reading it would cost more than the thumbnail is worth. +#[cfg(feature = "remote-connect")] +const MAX_REMOTE_CHAT_IMAGE_SOURCE_BYTES: u64 = 16 * 1024 * 1024; + +/// Read the pixels behind an attachment path so clients that cannot reach this +/// filesystem still see the picture. A path that no longer resolves is not an +/// error here — the attachment simply keeps travelling as a name. +#[cfg(feature = "remote-connect")] +fn read_remote_chat_image_pixels(image_path: &str) -> Option> { + let path = std::path::Path::new(image_path); + // Turn metadata records where the user took the file from, always as an + // absolute host path. A relative one would resolve against whatever + // directory this process happens to be in, which is never what was meant. + if !path.is_absolute() { + return None; + } + let metadata = std::fs::metadata(path).ok()?; + if !metadata.is_file() || metadata.len() > MAX_REMOTE_CHAT_IMAGE_SOURCE_BYTES { + return None; + } + std::fs::read(path).ok() +} + /// Convert persisted turns into mobile ChatMessages. /// This is the same data source the desktop frontend uses. #[cfg(feature = "remote-connect")] -fn remote_chat_messages_from_turns(turns: &[DialogTurnData]) -> Vec { +fn remote_chat_messages_from_turns( + turns: &[DialogTurnData], + read_image_pixels: &dyn Fn(&str) -> Option>, +) -> Vec { let projected_turns = turns .iter() .filter(|turn| turn.kind.is_model_visible()) - .map(remote_chat_history_turn_from_core_turn) + .map(|turn| remote_chat_history_turn_from_core_turn(turn, read_image_pixels)) .collect::>(); build_remote_chat_messages(projected_turns) } #[cfg(feature = "remote-connect")] -fn remote_chat_history_turn_from_core_turn(turn: &DialogTurnData) -> RemoteChatHistoryTurn { +fn remote_chat_history_turn_from_core_turn( + turn: &DialogTurnData, + read_image_pixels: &dyn Fn(&str) -> Option>, +) -> RemoteChatHistoryTurn { let prompt_visible_content = crate::agentic::core::strip_prompt_markup(&turn.user_message.content); - let user_projection = - project_remote_chat_user(turn.user_message.metadata.as_ref(), &prompt_visible_content); + let user_projection = project_remote_chat_user( + turn.user_message.metadata.as_ref(), + &prompt_visible_content, + read_image_pixels, + ); let rounds = turn .model_rounds @@ -1811,7 +1870,11 @@ impl CoreServiceAgentRuntime { .ok_or_else(||anyhow::anyhow!("Session storage is unavailable on this host"))?; let coordinator=get_global_coordinator().ok_or_else(||anyhow::anyhow!("Runtime is unavailable"))?; let turns=coordinator.load_relay_session_turns(&directory,session_id,turn_id).await?; - openbitfun_services_integrations::remote_connect::session_records::records_from_turns(&turns) + // Attachment paths are read and images re-encoded here, so the work + // leaves the reactor rather than stalling the session's other events. + tokio::task::spawn_blocking(move||{ + openbitfun_services_integrations::remote_connect::session_records::records_from_turns(&turns,&read_remote_chat_image_pixels) + }).await? }).await.map_err(|error|error.to_string()) } @@ -1827,7 +1890,15 @@ impl CoreServiceAgentRuntime { .load_visible_persisted_session_turns(session_storage_dir, session_id) .await .map_err(|error| error.to_string())?; - Ok((remote_chat_messages_from_turns(&turns), false)) + // Projecting a history decodes and re-encodes every attachment, and now + // reads the ones stored as a path, so it does not belong on the thread + // driving the remote-connect server. + let messages = tokio::task::spawn_blocking(move || { + remote_chat_messages_from_turns(&turns, &read_remote_chat_image_pixels) + }) + .await + .map_err(|error| format!("Remote history projection failed: {error}"))?; + Ok((messages, false)) } #[cfg(feature = "remote-connect")] @@ -3365,6 +3436,7 @@ mod tests { use std::collections::HashSet; use openbitfun_runtime_ports::SessionTranscriptReader; + use openbitfun_services_integrations::remote_connect::no_host_image_pixels; use super::*; use crate::service::session::{ @@ -3919,7 +3991,7 @@ mod tests { })), ); - let messages = remote_chat_messages_from_turns(&[turn]); + let messages = remote_chat_messages_from_turns(&[turn], &no_host_image_pixels); assert_eq!(messages.len(), 2); assert_eq!(messages[0].role, "user"); @@ -3948,7 +4020,7 @@ mod tests { fn core_service_agent_runtime_owner_preserves_in_progress_remote_assistant_history() { let turn = remote_history_test_turn(TurnStatus::InProgress, None); - let messages = remote_chat_messages_from_turns(&[turn]); + let messages = remote_chat_messages_from_turns(&[turn], &no_host_image_pixels); assert_eq!(messages.len(), 2); assert_eq!(messages[0].role, "user"); @@ -3963,7 +4035,7 @@ mod tests { let mut turn = remote_history_test_turn(TurnStatus::InProgress, None); turn.model_rounds.clear(); - let messages = remote_chat_messages_from_turns(&[turn]); + let messages = remote_chat_messages_from_turns(&[turn], &no_host_image_pixels); assert_eq!(messages.len(), 1); assert_eq!(messages[0].role, "user"); @@ -3975,7 +4047,7 @@ mod tests { let mut turn = remote_history_test_turn(TurnStatus::Error, None); turn.error = Some("AI client could not reach the configured proxy".to_string()); - let messages = remote_chat_messages_from_turns(&[turn]); + let messages = remote_chat_messages_from_turns(&[turn], &no_host_image_pixels); assert_eq!(messages.len(), 2); assert_eq!(messages[1].turn_id.as_deref(), Some("turn-1")); @@ -3992,7 +4064,7 @@ mod tests { turn.user_message.content = "User uploaded a file.\nUser's question:\n explain this ".to_string(); - let messages = remote_chat_messages_from_turns(&[turn]); + let messages = remote_chat_messages_from_turns(&[turn], &no_host_image_pixels); assert_eq!(messages[0].content, "explain this"); } diff --git a/src/crates/contracts/runtime-ports/src/remote_workspace_ports.rs b/src/crates/contracts/runtime-ports/src/remote_workspace_ports.rs index ed5b7022a9..0c02c883e7 100644 --- a/src/crates/contracts/runtime-ports/src/remote_workspace_ports.rs +++ b/src/crates/contracts/runtime-ports/src/remote_workspace_ports.rs @@ -103,6 +103,20 @@ pub struct RemoteSessionMetadata { pub created_at_ms: u64, pub last_active_at_ms: u64, pub turn_count: usize, + /// Parent session id for child sessions (btw/review/miniapp/subagent). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_session_id: Option, + /// Relationship kind as the snake_case tag persisted by Services + /// (`btw`, `review`, `deep_review`, `miniapp`, `subagent`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relationship_kind: Option, +} + +impl RemoteSessionMetadata { + /// Child sessions belong under their parent, not in a flat session list. + pub fn is_child_session(&self) -> bool { + self.parent_session_id.is_some() + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -245,6 +259,8 @@ mod tests { created_at_ms: 10, last_active_at_ms: 20, turn_count: 3, + parent_session_id: None, + relationship_kind: None, }; assert_eq!(workspace.kind.as_wire_str(), "remote"); diff --git a/src/crates/services/services-integrations/src/remote_connect.rs b/src/crates/services/services-integrations/src/remote_connect.rs index 48ab52b2d6..0a95b3fa93 100644 --- a/src/crates/services/services-integrations/src/remote_connect.rs +++ b/src/crates/services/services-integrations/src/remote_connect.rs @@ -28,8 +28,8 @@ pub mod session_store; pub mod session_subscriber; pub use chat_projection::{ - agent_input_attachment_from_remote_image_context, project_remote_chat_user, - RemoteChatUserProjection, + agent_input_attachment_from_remote_image_context, no_host_image_pixels, + project_remote_chat_user, RemoteChatUserProjection, }; pub use device::DeviceIdentity; pub use encryption::{decrypt_from_base64, encrypt_to_base64, KeyPair}; @@ -1117,9 +1117,24 @@ pub fn remote_session_info( message_count: metadata.turn_count, workspace_path: workspace_path.map(ToOwned::to_owned), workspace_name: workspace_name.map(ToOwned::to_owned), + parent_session_id: metadata.parent_session_id.clone(), + relationship_kind: metadata.relationship_kind.clone(), } } +/// Child sessions (btw/review/miniapp/subagent) belong under their parent. +/// +/// Desktop groups them through `build_session_metadata_page`, but the remote +/// command path is flat, so they would surface as standalone conversations. +/// Drop them here, before pagination, so `has_more` counts what is actually +/// sent. +pub fn visible_remote_sessions(metadata: Vec) -> Vec { + metadata + .into_iter() + .filter(|session| !session.is_child_session()) + .collect() +} + pub fn remote_session_list_response( metadata: Vec, workspace_path: Option<&str>, @@ -1127,6 +1142,7 @@ pub fn remote_session_list_response( limit: usize, offset: usize, ) -> RemoteResponse { + let metadata = visible_remote_sessions(metadata); let page_size = limit.min(100); let total = metadata.len(); let has_more = offset.saturating_add(page_size) < total; @@ -1251,6 +1267,7 @@ where let (sessions, has_more) = if let Some(path) = workspace_path.as_deref() { match host.list_session_metadata(path, workspace_identity).await { Ok(metadata) => { + let metadata = visible_remote_sessions(metadata); let total = metadata.len(); let page_size = 100usize; ( @@ -2017,6 +2034,11 @@ pub struct SessionInfo { pub workspace_path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub workspace_name: Option, + /// Set when this session hangs off another one (btw/review/miniapp/subagent). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relationship_kind: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -4316,6 +4338,8 @@ mod tests { created_at_ms: 1_000, last_active_at_ms: 2_000, turn_count: 3, + parent_session_id: None, + relationship_kind: None, }, RemoteSessionMetadata { session_id: "session-b".to_string(), @@ -4324,6 +4348,8 @@ mod tests { created_at_ms: 1_000, last_active_at_ms: 2_000, turn_count: 1, + parent_session_id: None, + relationship_kind: None, }, ]) } diff --git a/src/crates/services/services-integrations/src/remote_connect/chat_projection.rs b/src/crates/services/services-integrations/src/remote_connect/chat_projection.rs index b68befc834..e8fa3ceddb 100644 --- a/src/crates/services/services-integrations/src/remote_connect/chat_projection.rs +++ b/src/crates/services/services-integrations/src/remote_connect/chat_projection.rs @@ -16,9 +16,13 @@ pub struct RemoteChatUserProjection { pub images: Vec, } +/// `read_image_pixels` resolves an attachment path against the runtime host's +/// filesystem. Everything else here is a pure transform over turn metadata, so +/// the one part that needs the host is passed in rather than reached for. pub fn project_remote_chat_user( metadata: Option<&serde_json::Value>, prompt_visible_content: &str, + read_image_pixels: &dyn Fn(&str) -> Option>, ) -> RemoteChatUserProjection { let original_text = metadata .and_then(|metadata| metadata.get("original_text")) @@ -26,16 +30,20 @@ pub fn project_remote_chat_user( RemoteChatUserProjection { content: remote_chat_user_display_content(original_text, prompt_visible_content), - images: remote_chat_user_images_from_metadata(metadata), + images: remote_chat_user_images_from_metadata(metadata, read_image_pixels), } } +/// Projection for callers with no filesystem of their own to offer: a path-only +/// attachment stays path-only and clients name it instead of drawing it. +pub fn no_host_image_pixels(_image_path: &str) -> Option> { + None +} + /// Compress a base64 data-URL image to a small thumbnail for mobile display. /// Falls back to the original if decoding/compression fails or the image is /// already within the mobile thumbnail budget. fn compress_remote_chat_data_url_for_mobile(data_url: &str) -> String { - const MAX_THUMBNAIL_DIM: u32 = 400; - let Some(comma_pos) = data_url.find(',') else { return data_url.to_string(); }; @@ -49,9 +57,18 @@ fn compress_remote_chat_data_url_for_mobile(data_url: &str) -> String { return data_url.to_string(); }; - let Ok(img) = image::load_from_memory(&raw_bytes) else { - return data_url.to_string(); - }; + compress_image_bytes_for_mobile(&raw_bytes).unwrap_or_else(|| data_url.to_string()) +} + +/// Shrink raw image bytes into a data URL within the mobile thumbnail budget. +/// `None` means the bytes could not be decoded or re-encoded, so the caller +/// keeps whatever it already had. +fn compress_image_bytes_for_mobile(raw_bytes: &[u8]) -> Option { + compress_decoded_image_for_mobile(image::load_from_memory(raw_bytes).ok()?) +} + +fn compress_decoded_image_for_mobile(img: image::DynamicImage) -> Option { + const MAX_THUMBNAIL_DIM: u32 = 400; let resized = if img.width() > MAX_THUMBNAIL_DIM || img.height() > MAX_THUMBNAIL_DIM { img.resize(MAX_THUMBNAIL_DIM, MAX_THUMBNAIL_DIM, FilterType::Triangle) @@ -70,16 +87,17 @@ fn compress_remote_chat_data_url_for_mobile(data_url: &str) -> String { if let Some(buf) = encode_jpeg(&resized, quality) { if buf.len() <= REMOTE_CHAT_MOBILE_IMAGE_MAX_BYTES || quality == 30 { let b64 = BASE64.encode(&buf); - return format!("data:image/jpeg;base64,{b64}"); + return Some(format!("data:image/jpeg;base64,{b64}")); } } } - data_url.to_string() + None } fn remote_chat_user_images_from_metadata( metadata: Option<&serde_json::Value>, + read_image_pixels: &dyn Fn(&str) -> Option>, ) -> Vec { metadata .and_then(|metadata| metadata.get("images")) @@ -89,8 +107,22 @@ fn remote_chat_user_images_from_metadata( .iter() .filter_map(|image| { let name = image.get("name")?.as_str()?.to_string(); - let raw_url = image.get("data_url")?.as_str()?; - let data_url = compress_remote_chat_data_url_for_mobile(raw_url); + // An attachment may have been stored as a host path only. + // Read it back where the path still resolves; when that + // fails, keep the attachment in the timeline anyway so + // clients name it instead of drawing a blank frame, which is + // the difference between "no image here" and "this image did + // not arrive". + let data_url = image + .get("data_url") + .and_then(|value| value.as_str()) + .filter(|raw_url| !raw_url.is_empty()) + .map(compress_remote_chat_data_url_for_mobile) + .or_else(|| inline_host_path_pixels(image, read_image_pixels)) + .unwrap_or_default(); + if name.trim().is_empty() && data_url.is_empty() { + return None; + } Some(ChatImageAttachment { name, data_url }) }) .collect() @@ -98,6 +130,41 @@ fn remote_chat_user_images_from_metadata( .unwrap_or_default() } +/// Conversations recorded before attachments carried their pixels hold a host +/// path and nothing else. This is the host that wrote that path, so read it and +/// give clients that cannot reach this filesystem the picture rather than a name. +pub(super) fn inline_host_path_pixels( + image: &serde_json::Value, + read_image_pixels: &dyn Fn(&str) -> Option>, +) -> Option { + let image_path = image + .get("image_path") + .and_then(|value| value.as_str()) + .filter(|path| !path.is_empty())?; + let raw_bytes = read_image_pixels(image_path).filter(|bytes| !bytes.is_empty())?; + + // An attachment path is data that travelled in from a paired device, and + // this read is not bounded by the workspace root the way the remote file + // commands are. Decoding is the proof that the path really points at an + // image, so a path aimed at a private file cannot come back as base64. + // It also settles the type from the bytes rather than from a recorded + // `mime_type` that may not describe them. + let format = image::guess_format(&raw_bytes).ok()?; + let decoded = image::load_from_memory_with_format(&raw_bytes, format).ok()?; + + // An image already inside the mobile budget travels as it was recorded, so + // a screenshot keeps its transparency instead of being flattened to JPEG. + if raw_bytes.len() <= REMOTE_CHAT_MOBILE_IMAGE_MAX_BYTES { + return Some(format!( + "data:{};base64,{}", + format.to_mime_type(), + BASE64.encode(&raw_bytes) + )); + } + + compress_decoded_image_for_mobile(decoded) +} + fn remote_chat_user_display_content( original_text: Option<&str>, prompt_visible_content: &str, diff --git a/src/crates/services/services-integrations/src/remote_connect/session_records.rs b/src/crates/services/services-integrations/src/remote_connect/session_records.rs index 645d8ad8c0..6881ff3797 100644 --- a/src/crates/services/services-integrations/src/remote_connect/session_records.rs +++ b/src/crates/services/services-integrations/src/remote_connect/session_records.rs @@ -5,6 +5,8 @@ use anyhow::Result; use openbitfun_services_core::session::DialogTurnData; use serde_json::{json, Value}; +use super::chat_projection::inline_host_path_pixels; + /// Transcript facts and interaction controls have different persistence roles. /// Provider chunks are not durable messages; completed runtime blocks are read /// from the canonical turn store. Approval controls retain their full payload. @@ -46,14 +48,24 @@ pub fn session_event_publication(name: &str, payload: &Value) -> SessionEventPub } } -pub fn records_from_turns(turns: &[DialogTurnData]) -> Result> { +/// `read_image_pixels` resolves an attachment path against the host filesystem; +/// it is passed in because records are otherwise a pure transform over turns. +pub fn records_from_turns( + turns: &[DialogTurnData], + read_image_pixels: &dyn Fn(&str) -> Option>, +) -> Result> { let mut records = Vec::new(); for source in turns { let mut turn = serde_json::to_value(source)?; turn.as_object_mut() .expect("turn serializes as object") .remove("modelRounds"); - records.push(json!({"sessionId":source.session_id,"id":format!("turn/{}",source.turn_id),"turn":turn})); + // The turn record is the one clients build the user message from, so the + // pixels are inlined there alone; the parent headers below stay as they + // were recorded rather than repeating an image once per item. + let mut turn_record = turn.clone(); + inline_turn_attachment_pixels(&mut turn_record, read_image_pixels); + records.push(json!({"sessionId":source.session_id,"id":format!("turn/{}",source.turn_id),"turn":turn_record})); for source_round in &source.model_rounds { let mut round = serde_json::to_value(source_round)?; let object = round.as_object_mut().expect("round serializes as object"); @@ -79,6 +91,36 @@ pub fn records_from_turns(turns: &[DialogTurnData]) -> Result> { Ok(records) } +/// An attachment recorded before pixels travelled inline holds a host path and +/// nothing else, and only this host can still resolve it. Clients read images +/// straight out of the recorded metadata, so the pixels join it there. +fn inline_turn_attachment_pixels( + turn: &mut Value, + read_image_pixels: &dyn Fn(&str) -> Option>, +) { + let Some(images) = turn + .pointer_mut("/userMessage/metadata/images") + .and_then(|images| images.as_array_mut()) + else { + return; + }; + for image in images { + let recorded_pixels = image + .get("data_url") + .and_then(|value| value.as_str()) + .is_some_and(|data_url| !data_url.is_empty()); + if recorded_pixels { + continue; + } + let Some(data_url) = inline_host_path_pixels(image, read_image_pixels) else { + continue; + }; + if let Some(image) = image.as_object_mut() { + image.insert("data_url".to_string(), Value::String(data_url)); + } + } +} + #[cfg(test)] mod publication_tests { use super::*; diff --git a/src/crates/services/services-integrations/tests/remote_connect_contracts.rs b/src/crates/services/services-integrations/tests/remote_connect_contracts.rs index 0a87802e25..00465be120 100644 --- a/src/crates/services/services-integrations/tests/remote_connect_contracts.rs +++ b/src/crates/services/services-integrations/tests/remote_connect_contracts.rs @@ -15,15 +15,15 @@ use openbitfun_services_integrations::remote_connect::{ build_remote_image_submission_request, build_remote_model_catalog, build_remote_session_create_request, build_remote_submission_request, cancel_remote_task, handle_remote_command, handle_remote_workspace_file_command, make_slim_tool_params, - normalize_remote_model_selection, normalize_remote_session_model_id, project_remote_chat_user, - project_remote_plan_tool, read_remote_workspace_file, read_remote_workspace_file_chunk, - read_remote_workspace_file_info, remote_answer_question_response, - remote_assistant_list_response, remote_assistant_updated_response, - remote_dialog_steer_response, remote_dialog_submit_outcome_from_scheduler, - remote_dialog_submit_response, remote_file_chunk_response, remote_file_content_response, - remote_file_display_name, remote_file_info_response, remote_initial_sync_response, - remote_interaction_accepted_response, remote_messages_response, - remote_model_catalog_poll_delta, remote_model_selection_needs_config, + no_host_image_pixels, normalize_remote_model_selection, normalize_remote_session_model_id, + project_remote_chat_user, project_remote_plan_tool, read_remote_workspace_file, + read_remote_workspace_file_chunk, read_remote_workspace_file_info, + remote_answer_question_response, remote_assistant_list_response, + remote_assistant_updated_response, remote_dialog_steer_response, + remote_dialog_submit_outcome_from_scheduler, remote_dialog_submit_response, + remote_file_chunk_response, remote_file_content_response, remote_file_display_name, + remote_file_info_response, remote_initial_sync_response, remote_interaction_accepted_response, + remote_messages_response, remote_model_catalog_poll_delta, remote_model_selection_needs_config, remote_no_change_poll_response, remote_persisted_poll_response, remote_plan_build_content, remote_recent_workspaces_response, remote_session_created_response, remote_session_deleted_response, remote_session_info, remote_session_list_response, @@ -266,6 +266,10 @@ fn remote_chat_projection_owner_extracts_images_and_display_text() { "name": "raw-image", "data_url": "not-a-data-url" }, + { + "name": "dropped-on-the-desktop.png", + "image_path": "/Users/dev/Pictures/dropped-on-the-desktop.png" + }, { "name": "", "ignored": true @@ -273,7 +277,8 @@ fn remote_chat_projection_owner_extracts_images_and_display_text() { ] }); - let projection = project_remote_chat_user(Some(&metadata), "fallback question"); + let projection = + project_remote_chat_user(Some(&metadata), "fallback question", &no_host_image_pixels); assert_eq!( projection.images, @@ -286,6 +291,12 @@ fn remote_chat_projection_owner_extracts_images_and_display_text() { name: "raw-image".to_string(), data_url: "not-a-data-url".to_string(), }, + // A path-only attachment keeps its place in the timeline so clients + // can say the image did not arrive instead of showing nothing. + ChatImageAttachment { + name: "dropped-on-the-desktop.png".to_string(), + data_url: String::new(), + }, ] ); assert_eq!(projection.content, " original question "); @@ -293,6 +304,7 @@ fn remote_chat_projection_owner_extracts_images_and_display_text() { project_remote_chat_user( Some(&serde_json::json!({ "original_text": " keep exact question text " })), "fallback question", + &no_host_image_pixels, ) .content, " keep exact question text " @@ -301,16 +313,176 @@ fn remote_chat_projection_owner_extracts_images_and_display_text() { project_remote_chat_user( None, "User uploaded a file.\nUser's question:\n explain this ", + &no_host_image_pixels, ) .content, "explain this" ); assert_eq!( - project_remote_chat_user(None, " keep fallback spacing ").content, + project_remote_chat_user(None, " keep fallback spacing ", &no_host_image_pixels).content, " keep fallback spacing " ); } +const TINY_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + +/// A 1x1 PNG, small enough to travel within the mobile thumbnail budget. +const TINY_PNG: &[u8] = &[ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, + 0x89, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x44, 0x41, 0x54, 0x78, 0xda, 0x63, 0xfc, 0xcf, 0xc0, 0x50, + 0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, + 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, +]; + +#[test] +fn remote_chat_projection_owner_recovers_pixels_behind_an_attachment_path() { + let metadata = serde_json::json!({ + "original_text": "look at this", + "images": [ + { + "name": "dropped-on-the-desktop.png", + "image_path": "/Users/dev/Pictures/dropped-on-the-desktop.png", + "mime_type": "image/png" + }, + { + "name": "private-key", + "image_path": "/Users/dev/.ssh/id_ed25519", + "mime_type": "image/png" + }, + { + "name": "moved-away.png", + "image_path": "/Users/dev/Pictures/moved-away.png", + "mime_type": "image/png" + } + ] + }); + + let read_pixels = |image_path: &str| match image_path { + "/Users/dev/Pictures/dropped-on-the-desktop.png" => Some(TINY_PNG.to_vec()), + // Split so the repository's secret scanner does not read this fixture + // as a real leaked key. There is no key material here, only the header + // a private key would start with. + "/Users/dev/.ssh/id_ed25519" => Some( + concat!("-----BEGIN OPENSSH ", "PRIVATE KEY-----") + .as_bytes() + .to_vec(), + ), + _ => None, + }; + let projection = project_remote_chat_user(Some(&metadata), "fallback", &read_pixels); + + assert_eq!( + projection.images, + vec![ + // The pixels behind the path travel, typed from the bytes rather + // than from the recorded mime, so an old conversation shows its + // image on a client that cannot reach this filesystem. + ChatImageAttachment { + name: "dropped-on-the-desktop.png".to_string(), + data_url: format!("data:image/png;base64,{TINY_PNG_BASE64}"), + }, + // A path aimed at something that is not an image never comes back + // as base64, whatever the attachment claims its type is. + ChatImageAttachment { + name: "private-key".to_string(), + data_url: String::new(), + }, + // A path that no longer resolves still names the attachment. + ChatImageAttachment { + name: "moved-away.png".to_string(), + data_url: String::new(), + }, + ] + ); +} + +#[test] +fn relay_records_carry_attachment_pixels_on_the_turn_record_alone() { + let turn: openbitfun_services_core::session::DialogTurnData = serde_json::from_value( + serde_json::json!({ + "turnId": "turn-1", + "turnIndex": 0, + "sessionId": "session-1", + "timestamp": 1_700_000_000_u64, + "startTime": 1_700_000_000_u64, + "status": "completed", + "userMessage": { + "id": "message-user-1", + "content": "look at this", + "timestamp": 1_700_000_000_u64, + "metadata": { + "original_text": "look at this", + "images": [ + {"name": "dropped.png", "image_path": "/Users/dev/Pictures/dropped.png", "mime_type": "image/png"}, + {"name": "already-inline.png", "image_path": "/Users/dev/Pictures/dropped.png", "data_url": "data:image/png;base64,recorded"} + ] + } + }, + "modelRounds": [{ + "id": "round-1", + "turnId": "turn-1", + "roundIndex": 0, + "timestamp": 1_700_000_000_u64, + "startTime": 1_700_000_000_u64, + "status": "completed", + "textItems": [{"id": "item-1", "roundId": "round-1", "turnId": "turn-1", "content": "on it", "timestamp": 1_700_000_000_u64, "orderIndex": 0, "status": "completed", "isStreaming": false}], + "thinkingItems": [], + "toolItems": [] + }] + }), + ) + .expect("turn fixture deserializes"); + + let read_pixels = |image_path: &str| match image_path { + "/Users/dev/Pictures/dropped.png" => Some(TINY_PNG.to_vec()), + _ => None, + }; + let records = + openbitfun_services_integrations::remote_connect::session_records::records_from_turns( + std::slice::from_ref(&turn), + &read_pixels, + ) + .expect("records build"); + + let images_of = |record: &serde_json::Value| { + record["turn"]["userMessage"]["metadata"]["images"] + .as_array() + .expect("attachments survive the record") + .iter() + .map(|image| image["data_url"].as_str().unwrap_or_default().to_string()) + .collect::>() + }; + + // The turn record is what clients build the user message from, so the pixels + // behind a recorded path join it there and a recorded data URL is left alone. + let turn_record = records + .iter() + .find(|record| record["id"] == "turn/turn-1") + .expect("turn record is published"); + assert_eq!( + images_of(turn_record), + vec![ + format!("data:image/png;base64,{TINY_PNG_BASE64}"), + "data:image/png;base64,recorded".to_string(), + ] + ); + + // Round and item records repeat the turn only as a parent header. Inlining + // there would send the same image once per item, so they stay as recorded. + for record in records + .iter() + .filter(|record| record["id"] != "turn/turn-1") + { + assert_eq!( + images_of(record), + vec![String::new(), "data:image/png;base64,recorded".to_string()], + "header of {} should not carry recovered pixels", + record["id"] + ); + } +} + #[test] fn remote_image_context_to_agent_attachment_preserves_metadata_contract() { let attachment = agent_input_attachment_from_remote_image_context(RemoteImageContext { @@ -2029,6 +2201,63 @@ fn remote_connect_workspace_response_helpers_own_wire_shape() { ); } +#[test] +fn remote_connect_session_list_hides_child_sessions_and_counts_only_what_it_sends() { + // Desktop nests btw/review/miniapp children under their parent. The remote + // command path is flat, so a child would read as a standalone conversation. + let metadata = vec![ + RemoteSessionMetadata { + session_id: "parent".to_string(), + name: "iOS sidebar spacing".to_string(), + agent_type: "Standard".to_string(), + created_at_ms: 1_700_000_000_000, + last_active_at_ms: 1_700_000_001_000, + turn_count: 2, + parent_session_id: None, + relationship_kind: None, + }, + RemoteSessionMetadata { + session_id: "btw".to_string(), + name: "why did it grep with bash".to_string(), + agent_type: "Standard".to_string(), + created_at_ms: 1_700_000_002_000, + last_active_at_ms: 1_700_000_003_000, + turn_count: 1, + parent_session_id: Some("parent".to_string()), + relationship_kind: Some("btw".to_string()), + }, + ]; + + let list = + remote_session_list_response(metadata, Some("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/workspace/project"), Some("project"), 1, 0); + let list_json = serde_json::to_value(list).expect("serialize session list"); + assert_eq!(list_json["sessions"].as_array().unwrap().len(), 1); + assert_eq!(list_json["sessions"][0]["session_id"], "parent"); + // The child is gone before pagination, so the page is the whole list. + assert_eq!(list_json["has_more"], false); + assert!(list_json["sessions"][0].get("parent_session_id").is_none()); +} + +#[test] +fn remote_connect_session_info_carries_child_lineage_to_clients() { + let child = RemoteSessionMetadata { + session_id: "btw".to_string(), + name: "why did it grep with bash".to_string(), + agent_type: "Standard".to_string(), + created_at_ms: 1_700_000_002_000, + last_active_at_ms: 1_700_000_003_000, + turn_count: 1, + parent_session_id: Some("parent".to_string()), + relationship_kind: Some("btw".to_string()), + }; + assert!(child.is_child_session()); + + let info = remote_session_info(&child, None, None); + let json = serde_json::to_value(info).expect("serialize session info"); + assert_eq!(json["parent_session_id"], "parent"); + assert_eq!(json["relationship_kind"], "btw"); +} + #[test] fn remote_connect_session_response_helpers_own_pagination_and_timestamps() { let metadata = vec![ @@ -2039,6 +2268,8 @@ fn remote_connect_session_response_helpers_own_pagination_and_timestamps() { created_at_ms: 1_700_000_000_000, last_active_at_ms: 1_700_000_001_000, turn_count: 3, + parent_session_id: None, + relationship_kind: None, }, RemoteSessionMetadata { session_id: "session-2".to_string(), @@ -2047,6 +2278,8 @@ fn remote_connect_session_response_helpers_own_pagination_and_timestamps() { created_at_ms: 1_700_000_002_000, last_active_at_ms: 1_700_000_003_000, turn_count: 5, + parent_session_id: None, + relationship_kind: None, }, RemoteSessionMetadata { session_id: "session-3".to_string(), @@ -2055,6 +2288,8 @@ fn remote_connect_session_response_helpers_own_pagination_and_timestamps() { created_at_ms: 1_700_000_004_000, last_active_at_ms: 1_700_000_005_000, turn_count: 8, + parent_session_id: None, + relationship_kind: None, }, ]; diff --git a/src/mobile-web/src/services/CloudAccountClient.ts b/src/mobile-web/src/services/CloudAccountClient.ts index 2eb87e1697..8ff9a55edf 100644 --- a/src/mobile-web/src/services/CloudAccountClient.ts +++ b/src/mobile-web/src/services/CloudAccountClient.ts @@ -16,6 +16,33 @@ export class CloudAccountRequestError extends Error { } } +/** The request never reached the relay, or its answer never came back. */ +export class CloudAccountTransportError extends Error { + constructor(message: string) { + super(message); + this.name = 'CloudAccountTransportError'; + } +} + +/** + * Whether a failed sign-in poll should be tried again inside the window. + * + * Retry what the next tick could plausibly get past: a transport failure, and + * a relay that is rate limiting or briefly unavailable. Stop for anything the + * relay meant — a rejected transaction stays rejected however long the browser + * keeps asking. + * + * Kept in step with `harmonyos/.../CloudAccountClient.ets` and + * `shared/core-feature/.../AccountStore.kt`, which carry the same decision. + */ +export function retryableAuthorizationPollError(error: unknown): boolean { + if (error instanceof CloudAccountTransportError) return true; + if (error instanceof CloudAccountRequestError) { + return error.status === 429 || (error.status >= 500 && error.status < 600); + } + return false; +} + export function generateRequestId(): string { if (typeof crypto.randomUUID === 'function') return crypto.randomUUID(); const bytes = crypto.getRandomValues(new Uint8Array(16)); @@ -59,11 +86,16 @@ async function requestJson( : null, ); } - if (!parsed) throw new Error('Relay returned an empty account response.'); + if (!parsed) throw new CloudAccountTransportError('Relay returned an empty account response.'); return parsed as T; } catch (error: unknown) { if ((error as { name?: string })?.name === 'AbortError') { - throw new Error('Account request timed out.'); + throw new CloudAccountTransportError('Account request timed out.'); + } + // fetch() rejects with a TypeError for every connection-level failure: + // offline, DNS, TLS, a dropped socket mid-body. + if (error instanceof TypeError) { + throw new CloudAccountTransportError('Could not reach the account service.'); } throw error; } finally { @@ -94,17 +126,34 @@ export class CloudAccountClient { if (url.origin === 'https://auth.openbitfun.com') url.searchParams.set('locale', (typeof document === 'undefined' ? 'en-US' : document.documentElement.lang || 'en-US')); if (signal.aborted) throw new Error('Sign-in cancelled.'); popup.location.href = url.href; + let lastTransient: Error | null = null; while (!signal.aborted && Date.now() < start.expiresAt * 1000) { await new Promise((resolve) => setTimeout(resolve, Math.min(30, Math.max(1, start.pollIntervalSeconds)) * 1000)); if (signal.aborted) break; - const result = await requestJson<{ status: string; tokens?: { accessToken: string } }>( - this.relayUrl, '/api/auth/github/poll', { - transactionId: start.transactionId, transactionSecret: start.transactionSecret, - }, - ); + let result: { status: string; tokens?: { accessToken: string } }; + try { + result = await requestJson<{ status: string; tokens?: { accessToken: string } }>( + this.relayUrl, '/api/auth/github/poll', { + transactionId: start.transactionId, transactionSecret: start.transactionSecret, + }, + ); + } catch (cause) { + // The poll window spans the minutes the user spends in the popup, which + // on a phone browser is exactly when the tab is backgrounded, the radio + // sleeps, or the network hops. Ending the sign-in on the first hiccup + // would send them back to the start for something the next tick fixes + // by itself, so only a refusal the relay actually means stops the loop. + if (!retryableAuthorizationPollError(cause)) throw cause; + lastTransient = cause as Error; + continue; + } + lastTransient = null; if (result.status === 'authorized' && result.tokens?.accessToken) return result.tokens.accessToken; if (result.status === 'expired' || result.status === 'denied') break; } + // A window that ran out while every poll was failing is a network problem, + // not a rejected sign-in: report the one the user can act on. + if (!signal.aborted && lastTransient) throw lastTransient; throw new Error(signal.aborted ? 'Sign-in cancelled.' : 'Sign-in expired. Try again.'); } diff --git a/src/mobile-web/tests/account-login.test.mjs b/src/mobile-web/tests/account-login.test.mjs index cac08a9044..b140176499 100644 --- a/src/mobile-web/tests/account-login.test.mjs +++ b/src/mobile-web/tests/account-login.test.mjs @@ -317,3 +317,102 @@ test('tab navigation preserves explicit disconnect and migrates only a matching deviceId: 'desktop-a', disconnected: true, session: undefined, }); }); + +const encryptionStub = `data:text/javascript;base64,${Buffer.from( + 'export const toB64 = (bytes) => Buffer.from(bytes).toString("base64");', +).toString('base64')}`; +const accountClient = await loadSource('../src/services/CloudAccountClient.ts', { + './E2EEncryption': encryptionStub, + './pairingLink': links.url, +}); +const { + CloudAccountClient, CloudAccountRequestError, CloudAccountTransportError, + retryableAuthorizationPollError, +} = await import(accountClient.url); + +test('a sign-in poll retries what the next tick can fix and stops at what the relay meant', () => { + for (const retryable of [ + new CloudAccountTransportError('Account request timed out.'), + new CloudAccountRequestError('Slow down.', 429), + new CloudAccountRequestError('Relay is restarting.', 503), + ]) assert.equal(retryableAuthorizationPollError(retryable), true); + for (const fatal of [ + new CloudAccountRequestError('Unknown transaction.', 404), + new CloudAccountRequestError('Bad secret.', 401), + new Error('Untrusted account authorization URL.'), + ]) assert.equal(retryableAuthorizationPollError(fatal), false); +}); + +/** + * Drives authorize() against a scripted relay on a virtual clock: every wait + * returns at once and advances the poll window by what it asked to sleep, so + * the whole five-minute window plays out inside the test. The last scripted + * answer repeats for as long as the window lasts. + */ +async function runAuthorization(pollAnswers) { + const realSetTimeout = globalThis.setTimeout; + const realNow = Date.now; + const previousWindow = globalThis.window; + const previousFetch = globalThis.fetch; + let clock = realNow(); + const start = { + transactionId: 'tx-1', transactionSecret: 'secret-1', + authorizationUrl: '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/login/oauth/authorize?client_id=x', + expiresAt: Math.floor(clock / 1000) + 300, pollIntervalSeconds: 5, + }; + const polls = []; + Date.now = () => clock; + globalThis.setTimeout = (fn, ms) => { clock += ms ?? 0; return realSetTimeout(fn, 0); }; + globalThis.window = { setTimeout: () => 0, clearTimeout: () => {} }; + globalThis.fetch = async (url) => { + if (String(url).includes('/start')) { + return { ok: true, status: 200, text: async () => JSON.stringify(start) }; + } + const answer = pollAnswers[Math.min(polls.length, pollAnswers.length - 1)]; + polls.push(answer); + if (answer.throws) throw answer.throws; + return { + ok: (answer.httpStatus ?? 200) < 400, + status: answer.httpStatus ?? 200, + text: async () => JSON.stringify(answer.body ?? {}), + }; + }; + try { + const client = new CloudAccountClient('http://localhost:8787'); + const outcome = await client + .authorize({ location: { href: '' } }, new AbortController().signal) + .then((token) => ({ token }), (error) => ({ error })); + return { ...outcome, polls }; + } finally { + Date.now = realNow; + globalThis.setTimeout = realSetTimeout; + globalThis.window = previousWindow; + globalThis.fetch = previousFetch; + } +} + +test('a dropped connection mid-sign-in is retried instead of ending the attempt', async () => { + const flaky = await runAuthorization([ + { throws: new TypeError('Failed to fetch') }, + { httpStatus: 503, body: { error: 'Relay is restarting.' } }, + { body: { status: 'authorized', tokens: { accessToken: 'token-1' } } }, + ]); + assert.equal(flaky.token, 'token-1'); + assert.equal(flaky.polls.length, 3); + + const refused = await runAuthorization([{ httpStatus: 401, body: { error: 'Bad secret.' } }]); + assert.equal(refused.token, undefined); + assert.equal(refused.error.status, 401); + assert.equal(refused.polls.length, 1); + + const denied = await runAuthorization([{ body: { status: 'denied' } }]); + assert.match(denied.error.message, /Sign-in expired\. Try again\./); + assert.equal(denied.polls.length, 1); + + // A window that ran out while every poll was failing is a network problem, + // and must not be reported as a sign-in the user let expire. + const offline = await runAuthorization([{ throws: new TypeError('Failed to fetch') }]); + assert.ok(offline.error instanceof CloudAccountTransportError); + assert.equal(offline.error.message, 'Could not reach the account service.'); + assert.equal(offline.polls.length, 60); +}); diff --git a/src/shared/relay-protocol-contract-fixtures/session-item.json b/src/shared/relay-protocol-contract-fixtures/session-item.json index 6b233d4f92..0c82344ca0 100644 --- a/src/shared/relay-protocol-contract-fixtures/session-item.json +++ b/src/shared/relay-protocol-contract-fixtures/session-item.json @@ -107,6 +107,29 @@ "workspaceName": null } }, + { + "name": "child session carries its parent", + "wire": { + "id": "sess_btw", + "title": "why did it grep with bash", + "updated_at": "2026-08-06T11:04:22Z", + "parent_session_id": "sess_parent", + "relationship_kind": "btw" + }, + "expected": { + "id": "sess_btw", + "title": "why did it grep with bash", + "agentType": null, + "status": null, + "updatedAt": "2026-08-06T11:04:22Z", + "createdAt": "", + "messageCount": null, + "workspacePath": null, + "workspaceName": null, + "parentSessionId": "sess_parent", + "relationshipKind": "btw" + } + }, { "name": "unknown fields are ignored, not fatal", "wire": { diff --git a/src/web-ui/src/flow_chat/utils/imagePayload.test.ts b/src/web-ui/src/flow_chat/utils/imagePayload.test.ts index 634d265ca1..21d0c79920 100644 --- a/src/web-ui/src/flow_chat/utils/imagePayload.test.ts +++ b/src/web-ui/src/flow_chat/utils/imagePayload.test.ts @@ -21,6 +21,10 @@ const clipboard: ImageContext = { imageName: 'screen.png', fileSize: 3, mimeType: 'image/png', source: 'clipboard', isLocal: false, dataUrl: 'data:image/png;base64,YQ==', }; +const dropped: ImageContext = { + ...clipboard, id: 'dropped-1', isLocal: true, dataUrl: undefined, + imagePath: '/remote/workspace/chart.png', imageName: 'chart.png', +}; beforeEach(() => { mocks.surfaceId = 'local'; @@ -59,9 +63,29 @@ describe('image payload host ownership', () => { await expect(buildImagePayload([clipboard])).rejects.toThrow('surface changed'); }); - it('retains remote workspace paths when no inline pixels exist', async () => { - const result = await buildImagePayload([{ ...clipboard, isLocal: true, dataUrl: undefined, imagePath: '/remote/workspace/chart.png' }]); - expect(result?.imageContexts[0].image_path).toBe('/remote/workspace/chart.png'); + it('inlines pixels for a dropped local file so path-blind clients can draw it', async () => { + mocks.invoke.mockResolvedValue('YQ=='); + const result = await buildImagePayload([dropped]); + expect(mocks.invoke).toHaveBeenCalledWith('read_file_content', { + request: { filePath: dropped.imagePath, encoding: 'base64', remoteConnectionId: undefined }, + }); + expect(result?.imageContexts[0].image_path).toBe(dropped.imagePath); + expect(result?.imageContexts[0].data_url).toBe('data:image/png;base64,YQ=='); + expect(result?.imageDisplayData[0].dataUrl).toBe('data:image/png;base64,YQ=='); + expect(mocks.assertCurrent).toHaveBeenCalled(); + }); + + it('keeps the attachment on an unreadable path instead of failing the send', async () => { + mocks.invoke.mockRejectedValue(new Error('gone')); + const result = await buildImagePayload([dropped]); + expect(result?.imageContexts[0].image_path).toBe(dropped.imagePath); + expect(result?.imageContexts[0].data_url).toBeUndefined(); + }); + + it('leaves oversized local images as a path so turn storage stays bounded', async () => { + const result = await buildImagePayload([{ ...dropped, fileSize: 9 * 1024 * 1024 }]); + expect(mocks.invoke).not.toHaveBeenCalled(); + expect(result?.imageContexts[0].image_path).toBe(dropped.imagePath); expect(result?.imageContexts[0].data_url).toBeUndefined(); }); }); diff --git a/src/web-ui/src/flow_chat/utils/imagePayload.ts b/src/web-ui/src/flow_chat/utils/imagePayload.ts index 3b85da52f4..f24224dd6e 100644 --- a/src/web-ui/src/flow_chat/utils/imagePayload.ts +++ b/src/web-ui/src/flow_chat/utils/imagePayload.ts @@ -1,8 +1,18 @@ import type { ImageContext } from '@/shared/types/context'; import type { ImageContextData as ImageInputContextData } from '@/infrastructure/api/service-api/ImageContextTypes'; import { api } from '@/infrastructure/api/service-api/ApiClient'; +import { workspaceAPI } from '@/infrastructure/api/service-api/WorkspaceAPI'; import { getActiveSurfaceScope, isLocalSurface } from '@/infrastructure/peer-device/deviceSurface'; import { peerConnectionManager } from '@/infrastructure/peer-device/PeerConnectionManager'; +import { createLogger } from '@/shared/utils/logger'; +import { getMimeTypeFromFilename } from './imageUtils'; + +const log = createLogger('imagePayload'); + +// Above this the inline copy costs more than it is worth: the pixels would be +// duplicated into durable turn storage for every client that cannot read the +// path itself. Such images keep their path and render as a named placeholder. +const MAX_INLINE_LOCAL_IMAGE_BYTES = 8 * 1024 * 1024; export interface ImageDisplayData { id: string; @@ -56,13 +66,24 @@ export async function buildImagePayload(imageContexts: ImageContext[]): Promise< } } + // A dropped local file arrives as a path only, so clients that cannot reach + // this host's filesystem (mobile) would receive an attachment with nothing to + // draw. Read the pixels here, where the path is still resolvable, exactly the + // way the desktop renderer resolves them for display. + const inlinedLocalPixels = await inlineLocalImagePixels(imageContexts); + if (inlinedLocalPixels.size > 0) { + scope.assertCurrent('inline local image attachments'); + } + const pixelsFor = (ctx: ImageContext): string | undefined => + ctx.dataUrl || inlinedLocalPixels.get(ctx.id); + return { imageContexts: imageContexts.map(ctx => ({ id: ctx.id, image_path: ctx.isLocal ? ctx.imagePath : uploadedImagePaths.get(ctx.id), // Retain pixels for durable host storage and Detached Dispatch. Older // peer hosts still receive their upload path through the legacy branch. - data_url: ctx.dataUrl || undefined, + data_url: pixelsFor(ctx), mime_type: ctx.mimeType, metadata: { name: ctx.imageName, @@ -75,9 +96,37 @@ export async function buildImagePayload(imageContexts: ImageContext[]): Promise< imageDisplayData: imageContexts.map(ctx => ({ id: ctx.id, name: ctx.imageName || 'Image', - dataUrl: ctx.dataUrl, + dataUrl: pixelsFor(ctx), imagePath: ctx.isLocal ? ctx.imagePath : uploadedImagePaths.get(ctx.id), mimeType: ctx.mimeType, })), }; } + +async function inlineLocalImagePixels(imageContexts: ImageContext[]): Promise> { + const inlined = new Map(); + const pending = imageContexts.filter(ctx => + ctx.isLocal + && ctx.imagePath + && !ctx.dataUrl + && !(typeof ctx.fileSize === 'number' && ctx.fileSize > MAX_INLINE_LOCAL_IMAGE_BYTES) + ); + + await Promise.all(pending.map(async ctx => { + const imagePath = ctx.imagePath as string; + try { + // Attachment paths belong to the runtime host; the transport routes this + // read there, so never build a local asset URL from the path. + const base64 = await workspaceAPI.readFileContent(imagePath, 'base64'); + if (base64) { + inlined.set(ctx.id, `data:${ctx.mimeType || getMimeTypeFromFilename(imagePath)};base64,${base64}`); + } + } catch (cause) { + // The path still travels with the attachment, so hosts that can read it + // keep working. Only pixel-only clients lose the preview. + log.warn('Failed to inline local image pixels', { imagePath, error: cause }); + } + })); + + return inlined; +}