diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts index 0f039274e207..73884bb6f9e5 100644 --- a/apps/server/src/device/DeviceHubProxy.test.ts +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -139,3 +139,29 @@ it.each([ expect(await response.text()).not.toContain("private credential diagnostic"); expect(requests).toEqual([]); }); + +it.each([1, 3])( + "forwards fixed Duo display %s through the authenticated read proxy", + async (panel) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const route = `/vendor/serve-sim/helper/duo/panel/${panel}/stream.avcc`; + const response = await handler( + new Request(`http://t3.test/api/device-hub${route}?wsTicket=secret`), + ); + expect(response.status).toBe(200); + await response.text(); + expect(requests).toEqual([`http://hub.test${route}`]); + }, +); + +it.each(["/panel/2/stream.avcc", "/panel/1/webrtc/offer", "/panel/3/exec"])( + "rejects unsupported Duo route %s", + async (route) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request(`http://t3.test/api/device-hub/vendor/serve-sim/helper/duo${route}`), + ); + expect(response.status).toBe(404); + expect(requests).toEqual([]); + }, +); diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index dd048480b3dd..0fb49c68e281 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray = [ /^\/vendor\/serve-sim\/api\/screenshot$/, /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/, /^\/vendor\/serve-sim\/appstate$/, /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/, /^\/vendor\/serve-emu\/health$/, diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts index e8e7d5cae46a..f30960ed2212 100644 --- a/apps/server/src/device/DeviceToolchain.ts +++ b/apps/server/src/device/DeviceToolchain.ts @@ -26,7 +26,7 @@ import * as Semaphore from "effect/Semaphore"; import * as ProcessRunner from "../processRunner.ts"; const DEVICE_HUB_PACKAGE = "expo-device-hub"; -export const DEVICE_HUB_VERSION = "0.10.1"; +export const DEVICE_HUB_VERSION = "0.12.0"; const AGENT_DEVICE_PACKAGE = "agent-device"; export const AGENT_DEVICE_VERSION = "0.21.12"; diff --git a/apps/web/src/components/device/DeviceControlsRail.tsx b/apps/web/src/components/device/DeviceControlsRail.tsx index 443f111f01cc..3b9e8bf96c36 100644 --- a/apps/web/src/components/device/DeviceControlsRail.tsx +++ b/apps/web/src/components/device/DeviceControlsRail.tsx @@ -57,9 +57,9 @@ export function DeviceControlsRail(props: { ); } diff --git a/apps/web/src/components/device/DeviceDuoControls.tsx b/apps/web/src/components/device/DeviceDuoControls.tsx new file mode 100644 index 000000000000..1759ab07123b --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoControls.tsx @@ -0,0 +1,75 @@ +import { + DUO_POSES, + type DuoCommand, + type DuoControlState, +} from "@t3tools/client-runtime/device/duo-control"; +import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; +import { DeviceDuoGlyph } from "./DeviceDuoGlyph"; +import { Button } from "~/components/ui/button"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; + +/** Physical presets live beside the device. Pinching supplies continuous hinge control. */ +export function DeviceDuoControls(props: { + screen: DeviceScreenSize; + state: DuoControlState; + enabled: boolean; + onCommand: (command: DuoCommand) => void; +}) { + const angle = props.screen.hingeAngle; + const fold = angle == null ? null : angle === 0 ? "closed" : angle === 180 ? "open" : "book"; + const selected = (id: (typeof DUO_POSES)[number]["id"]) => + id === "laptop" || id === "tent" ? props.screen.hingePose === id : fold === id; + return ( +
+ {([DUO_POSES.slice(0, 3), DUO_POSES.slice(3)] as const).map((poses, index) => ( +
+ {poses.map((pose) => ( + + props.onCommand({ control: "pose", value: pose.id })} + /> + } + > + + + + {pose.label} + {pose.id === "book" ? " / bookshelf" : ""} + + + ))} +
+ ))} + {props.state.error ? ( + + + ! + + } + /> + {props.state.error} + + ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceDuoGlyph.tsx b/apps/web/src/components/device/DeviceDuoGlyph.tsx new file mode 100644 index 000000000000..a179e3afe260 --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoGlyph.tsx @@ -0,0 +1,43 @@ +import type { DuoPose } from "@t3tools/client-runtime/device/duo-control"; + +// Bitrig 0.25's toolbar glyphs. The fold outlines follow the native toolbar; +// the rounded stance contours are ported from SimulatorFoldingPoseGlyph's SwiftUI paths. +const stancePaths = { + laptop: + "M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z", + tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z", +} as const; + +export function DeviceDuoGlyph({ pose }: { pose: DuoPose }) { + const stance = pose === "laptop" || pose === "tent"; + return ( + + {stance ? : null} + {pose === "closed" ? ( + <> + + + + + ) : null} + {pose === "book" ? ( + + ) : null} + {pose === "open" ? ( + <> + + + + ) : null} + + ); +} diff --git a/apps/web/src/components/device/DeviceDuoViewport.tsx b/apps/web/src/components/device/DeviceDuoViewport.tsx new file mode 100644 index 000000000000..f168ceccc30d --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoViewport.tsx @@ -0,0 +1,205 @@ +import { useEffect, useRef, type RefObject } from "react"; +import type { DuoViewer } from "@t3tools/client-runtime/device/duo-viewer"; +import type { DeviceModelSource } from "@t3tools/client-runtime/device/model"; +import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction"; +import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; +import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream"; +import { createDuoPinch } from "@t3tools/client-runtime/device/duo-control"; +import { bindPhoneTrackpad } from "./phoneTrackpad"; + +const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer"); + +/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */ +export function DeviceDuoViewport(props: { + readonly model: DeviceModelSource; + readonly source: RefObject; + readonly onFrameListener: (listener: (() => void) | null) => void; + readonly onResetReady: (reset: (() => void) | null) => void; + readonly onInputCancel: (cancel: (() => void) | null) => void; + readonly client: RefObject; + readonly screen: DeviceScreenSize | null; + readonly hingePreview: number | null; + readonly controlError: string | null; + readonly onUnavailable: () => void; +}) { + const hostRef = useRef(null); + const canvasRef = useRef(null); + const viewerRef = useRef(null); + const interactionRef = useRef | null>(null); + const pinchRef = useRef | null>(null); + const trackpadRef = useRef | null>(null); + const screenRef = useRef(props.screen); + const modelRef = useRef(props.model); + const previewRef = useRef(props.hingePreview); + const { source, client, onInputCancel, onResetReady, onUnavailable, onFrameListener } = props; + + useEffect(() => { + screenRef.current = props.screen; + modelRef.current = props.model; + interactionRef.current?.end(); + viewerRef.current?.setScreen(props.screen); + }, [props.screen, props.model]); + + useEffect(() => { + if (props.controlError || !props.screen) { + trackpadRef.current?.cancel(); + viewerRef.current?.rejectOrientation(); + } + }, [props.controlError, props.screen]); + + useEffect(() => { + previewRef.current = props.hingePreview; + if (!pinchRef.current?.active) viewerRef.current?.setHingePreview(props.hingePreview); + }, [props.hingePreview]); + + useEffect(() => { + const host = hostRef.current; + const canvas = canvasRef.current; + const decoded = source.current; + if (!host || !canvas || !decoded) return; + let disposed = false; + let trackpad: ReturnType | null = null; + let stopTrackpadEnd: (() => void) | undefined; + const resize = () => { + const { width, height } = host.getBoundingClientRect(); + viewerRef.current?.resize(width, height, window.devicePixelRatio); + }; + const observer = new ResizeObserver(resize); + observer.observe(host); + const blur = () => { + interactionRef.current?.end(); + trackpad?.cancel(); + viewerRef.current?.cancelInput(); + }; + onInputCancel(blur); + window.addEventListener("blur", blur); + void loadDuoViewer() + .then(({ createDuoViewer }) => { + if (disposed) return; + const sources = { + 1: document.createElement("canvas"), + 3: document.createElement("canvas"), + }; + const viewer = createDuoViewer({ + canvas, + sources, + onUnavailable, + onPanelRequested: (panel) => + client.current?.controlDuo({ + control: "physical", + value: panel === 1 ? "facedown" : "faceup", + }), + onOrientationRequested: (value) => + client.current?.controlDuo({ control: "orientation", value }), + onModelError: (cause) => console.warn("Device 3D asset could not load", cause), + model: modelRef.current, + }); + viewerRef.current = viewer; + onResetReady(viewer.resetPose); + viewer.setScreen(screenRef.current); + const primaryFrame = () => { + const panel = screenRef.current?.screenId; + if (panel === 1 || panel === 3) viewer.frameUpdated(panel, decoded); + }; + onFrameListener(primaryFrame); + primaryFrame(); + viewer.setHingePreview(previewRef.current); + client.current?.setDuoPanels({ + onScreen(next) { + screenRef.current = next; + interactionRef.current?.end(); + viewer.setScreen(next); + }, + cover: createCanvasFrameSink(sources[1], () => viewer.frameUpdated(1)), + inner: createCanvasFrameSink(sources[3], () => viewer.frameUpdated(3)), + }); + interactionRef.current = createPhoneInteraction({ + screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured), + touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y), + orbit: viewer.orbit, + zoomBy: () => {}, + onInteractionActive: viewer.setInteractionActive, + }); + const pinch = createDuoPinch({ + angle: () => + previewRef.current ?? + screenRef.current?.hingeAngle ?? + (screenRef.current?.screenId === 1 ? 0 : 180), + contains: (x, y) => !!screenRef.current && viewer.beginHinge(x, y), + change: (angle) => { + viewer.setHingePreview(angle ?? previewRef.current); + if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle }); + }, + }); + pinchRef.current = pinch; + trackpad = bindPhoneTrackpad(canvas, interactionRef.current, { + begin(x, y) { + interactionRef.current?.end(); + return pinch.begin(x, y); + }, + move: pinch.move, + end: pinch.end, + }); + trackpadRef.current = trackpad; + stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit()); + resize(); + }) + .catch(() => { + if (!disposed) onUnavailable(); + }); + return () => { + disposed = true; + stopTrackpadEnd?.(); + trackpad?.dispose(); + trackpadRef.current = null; + pinchRef.current = null; + interactionRef.current?.end(); + interactionRef.current = null; + onInputCancel(null); + onResetReady(null); + onFrameListener(null); + client.current?.setDuoPanels(null); + observer.disconnect(); + window.removeEventListener("blur", blur); + viewerRef.current?.dispose(); + viewerRef.current = null; + }; + }, [onInputCancel, onResetReady, client, onUnavailable, source, onFrameListener]); + + const point = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + return { + x: (event.clientX - rect.left) / rect.width, + y: (event.clientY - rect.top) / rect.height, + }; + }; + return ( +
+
+
+ { + if (event.button !== 0) return; + if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return; + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus(); + }} + onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))} + onPointerUp={(event) => { + interactionRef.current?.move(event.pointerId, point(event)); + interactionRef.current?.end(event.pointerId); + }} + onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)} + onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)} + /> +
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index f5457efd0024..aa2d5886a13e 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -8,6 +8,9 @@ import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile"; import { deviceKeyboard, deviceModel } from "./deviceModels"; import { fitDeviceFrame } from "./deviceFrameLayout"; +import { DeviceDuoViewport } from "./DeviceDuoViewport"; +import { DeviceDuoControls } from "./DeviceDuoControls"; +import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control"; import { DevicePhoneViewport } from "./DevicePhoneViewport"; import { DeviceLoadingView } from "./DeviceLoadingView"; import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi"; @@ -29,6 +32,7 @@ export interface DeviceViewControls { readonly showPhone: () => void; readonly showFlat: () => void; readonly resetView: () => void; + readonly foldingControls?: ReactNode; readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null; } @@ -61,6 +65,13 @@ export function DeviceStreamView(props: { readonly onHandle?: (handle: DeviceStreamHandle | null) => void; readonly onScreen?: (screen: DeviceScreenSize | null) => void; }) { + const [duoControl, setDuoControl] = useState({ + pending: false, + requested: null, + error: null, + }); + const model = deviceModel(props.platform, props.deviceName ?? ""); + const isDuo = model?.id === "iphone-duo"; const [presentation, setPresentation] = useState<"phone" | "flat">("phone"); const [keyboardAttached, setKeyboardAttached] = useState(false); const [phoneUnavailable, setPhoneUnavailable] = useState(false); @@ -105,6 +116,8 @@ export function DeviceStreamView(props: { { platform: props.platform, deviceId: props.deviceId, access }, createCanvasFrameSink(canvas, () => frameListenerRef.current?.()), { + onDuoControl: setDuoControl, + onDuoUnavailable: onPhoneUnavailable, onStatus: (next, nextDetail) => { setStatus(next); setDetail(nextDetail); @@ -148,6 +161,7 @@ export function DeviceStreamView(props: { access, cancelPhoneInput, onHandle, + onPhoneUnavailable, onScreen, props.deviceId, props.environmentId, @@ -176,7 +190,8 @@ export function DeviceStreamView(props: { presentation === "phone" && !phoneUnavailable && !mjpegUrl && - !props.axOverlay; + !props.axOverlay && + (!isDuo || screen?.supportsHingeAngle === true); const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0; // The frame is the largest box at `aspect` that fits the container, so a @@ -275,20 +290,25 @@ export function DeviceStreamView(props: { return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; }; - const phoneUnavailableReason = phoneUnavailable - ? "3D is unavailable on this browser" - : mjpegUrl - ? "3D requires the H.264 stream" - : props.axOverlay - ? "Turn off accessibility frames to use 3D" - : null; + const phoneUnavailableReason = + isDuo && !screen?.supportsHingeAngle + ? "iPhone Duo 3D requires Device Hub 0.11.0 or newer" + : phoneUnavailable + ? "3D is unavailable on this browser" + : mjpegUrl + ? "3D requires the H.264 stream" + : props.axOverlay + ? "Turn off accessibility frames to use 3D" + : null; const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? ""); const resetView = useCallback(() => { - const orientation = keyboardAttached ? "landscape_right" : "portrait"; - if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation); + if (!isDuo) { + const orientation = keyboardAttached ? "landscape_right" : "portrait"; + if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation); + } resetViewRef.current?.(); - }, [keyboardAttached, screen?.orientation]); + }, [isDuo, keyboardAttached, screen?.orientation]); const profile = resolveDeviceShape({ platform: props.platform, name: props.deviceName ?? "", @@ -309,6 +329,18 @@ export function DeviceStreamView(props: { phone: !!showPhone, streaming: status === "streaming", phoneUnavailableReason, + foldingControls: + showPhone && isDuo && screen?.supportsHingeAngle ? ( + { + cancelPhoneInput(); + clientRef.current?.controlDuo(command); + }} + /> + ) : null, keyboard: showPhone && keyboardSource ? { @@ -413,7 +445,22 @@ export function DeviceStreamView(props: {
) : null} - {showPhone ? ( + {showPhone && isDuo && model ? ( + + ) : showPhone ? ( setPresentation("phone")} > 3D diff --git a/apps/web/src/components/device/deviceModels.ts b/apps/web/src/components/device/deviceModels.ts index 682e0e6de4d5..d9197a27e357 100644 --- a/apps/web/src/components/device/deviceModels.ts +++ b/apps/web/src/components/device/deviceModels.ts @@ -4,6 +4,7 @@ import { type DeviceModelSource, } from "@t3tools/client-runtime/device/model"; import type { DevicePlatform } from "@t3tools/contracts"; +import iphoneDuo from "./models/iphone-duo.glb?url"; import iphone18Pro from "./models/iphone-18-pro.glb?url"; import iphone18ProMax from "./models/iphone-18-pro-max.glb?url"; import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url"; @@ -11,6 +12,7 @@ import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url"; // Bundled URLs follow the client origin in local, desktop, hosted and remote sessions. const models: Record = { + "iphone-duo": { id: "iphone-duo", url: iphoneDuo }, "iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro }, "iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax }, "ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 }, diff --git a/apps/web/src/components/device/models/iphone-duo.glb b/apps/web/src/components/device/models/iphone-duo.glb new file mode 100644 index 000000000000..0269a1dc40e0 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-duo.glb differ diff --git a/apps/web/src/components/device/models/sources.json b/apps/web/src/components/device/models/sources.json index 513a56fc291d..766b65a7a759 100644 --- a/apps/web/src/components/device/models/sources.json +++ b/apps/web/src/components/device/models/sources.json @@ -6,7 +6,7 @@ "gltfTransform": "4.2.1", "changes": [ "Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.", - "Normalized portrait axes, centered the screen and uniformly scaled its height to 2.2 scene units.", + "Normalized conventional device portrait axes, centered their screen and uniformly scaled its height to 2.2 scene units. The Duo retains its upstream centimeter hinge rig.", "Replaced the baked marketing screen with a named placeholder for the live framebuffer.", "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry." ] @@ -44,6 +44,14 @@ "bodyNode": "zRrSLDpdYmKeRJQ", "screenNode": "lsDiIbtoSGSmWWZ", "accessoryNode": "PoBqSMmyhhcJsBX" + }, + { + "id": "iphone-duo", + "file": "iphone-duo.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-duo/2026/9305e4b9-72d9-4c05-9381-b572adadd5e5/ar/iPhone_Duo_e-sim_Star-White_Variant.usdz", + "sourceSha256": "5cab2ea636da0bc0b06c8abf4809843498f7c1d680042b4f95e383b5c6a718b2", + "upstreamConversionCommit": "bb265b11c13b395e5302d121458e2d42224a2e9f", + "conversion": "Preserved the centimeter split hinge rig and three display meshes from serve-sim; resized textures to 1024 pixels and encoded them as WebP without simplifying or flattening the rig." } ] } diff --git a/docs/user/devices.md b/docs/user/devices.md index 02e06d2bdffa..68f85ee214a6 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -39,6 +39,12 @@ iOS, and power off. Close the tab to stop watching; the device keeps running unless you power it off. Closed tabs stay closed after a reload. To watch the device again, choose it from **+ → Device**. +Choose **3D view** to inspect supported devices while the live screen stays +interactive. On iPhone Duo, use the fold and stance controls to change its +physical pose, or pinch over the device to adjust the hinge. Turning the model +to the other screen switches the live display and touch input to that screen. +**Restore 3D view** returns the device to a screen-facing position. + ## Tools The toolbar's **Tools** button opens a drawer for the open device. It shows the diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index dd6113ad6426..a8cbdd54350d 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -290,6 +290,14 @@ "./device/model": { "types": "./src/device/model.ts", "default": "./src/device/model.ts" + }, + "./device/duo-viewer": { + "types": "./src/device/duoViewer.ts", + "default": "./src/device/duoViewer.ts" + }, + "./device/duo-control": { + "types": "./src/device/duoControl.ts", + "default": "./src/device/duoControl.ts" } }, "scripts": { diff --git a/packages/client-runtime/src/device/duoControl.test.ts b/packages/client-runtime/src/device/duoControl.test.ts new file mode 100644 index 000000000000..019c383a56dd --- /dev/null +++ b/packages/client-runtime/src/device/duoControl.test.ts @@ -0,0 +1,109 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { createDuoControl, createDuoPinch, type DuoCommand } from "./duoControl.ts"; +afterEach(() => vi.useRealTimers()); + +it("keeps a failed send visible, including a disconnect while draining queued motion", () => { + const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => false); + const onChange = vi.fn(); + const queue = createDuoControl({ send, onChange }); + queue.enqueue({ control: "angle", value: 40 }); + expect(onChange).toHaveBeenLastCalledWith({ + pending: false, + requested: null, + error: "Device is disconnected.", + }); + send.mockReturnValueOnce(true); + queue.enqueue({ control: "pose", value: "book" }); + queue.enqueue({ control: "angle", value: 60 }); + queue.receive({ requestId: 2, ok: true }); + expect(onChange).toHaveBeenLastCalledWith({ + pending: false, + requested: null, + error: "Device is disconnected.", + }); +}); + +it("coalesces hinge edits behind acknowledgements and lets a preset replace queued edits", () => { + const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => true); + const onChange = vi.fn(); + const queue = createDuoControl({ send, onChange }); + queue.enqueue({ control: "angle", value: 40 }); + queue.enqueue({ control: "angle", value: 50 }); + queue.enqueue({ control: "angle", value: 60 }); + expect(send).toHaveBeenCalledTimes(1); + queue.receive({ requestId: 1, ok: true }); + expect(send.mock.calls[1]?.[0]).toEqual({ + requestId: 2, + command: { control: "angle", value: 60 }, + }); + queue.enqueue({ control: "angle", value: 100 }); + queue.enqueue({ control: "pose", value: "tent" }); + queue.receive({ requestId: 2, ok: true }); + expect(send.mock.calls[2]?.[0]).toEqual({ + requestId: 3, + command: { control: "pose", value: "tent" }, + }); + queue.receive({ requestId: 3, ok: true }); + expect(onChange).toHaveBeenLastCalledWith({ pending: false, requested: null, error: null }); + queue.clear(); +}); + +it("drops queued commands on failure, timeout and disconnect; late replies cannot acknowledge later work", () => { + vi.useFakeTimers(); + const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => true); + const onChange = vi.fn(); + const queue = createDuoControl({ send, onChange, timeoutMs: 100 }); + queue.enqueue({ control: "angle", value: 90 }); + queue.enqueue({ control: "angle", value: 100 }); + vi.advanceTimersByTime(100); + expect(onChange.mock.lastCall?.[0].error).toContain("timed out"); + queue.enqueue({ control: "pose", value: "open" }); + queue.receive({ requestId: 1, ok: true }); + expect(onChange.mock.lastCall?.[0].pending).toBe(true); + queue.enqueue({ control: "angle", value: 130 }); + queue.receive({ requestId: 2, ok: false, error: "native refused" }); + expect(onChange.mock.lastCall?.[0]).toEqual({ + pending: false, + requested: null, + error: "native refused", + }); + queue.enqueue({ control: "pose", value: "book" }); + queue.enqueue({ control: "pose", value: "closed" }); + queue.clear(); + queue.receive({ requestId: 3, ok: true }); + expect(send).toHaveBeenCalledTimes(3); + queue.enqueue({ control: "angle", value: Infinity }); + expect(send).toHaveBeenCalledTimes(3); + queue.clear(); +}); + +it("pinches only a hit device, accumulates independently of native readback, clamps and cancels", () => { + let confirmed = 90; + const change = vi.fn(); + const pinch = createDuoPinch({ + angle: () => confirmed, + contains: (x, y) => x > 0.2 && y > 0.2, + change, + }); + expect(pinch.begin(0.1, 0.5)).toBe(false); + pinch.move(1); + expect(change).not.toHaveBeenCalled(); + expect(pinch.begin(0.5, 0.5)).toBe(true); + pinch.move(0.25); + expect(change).toHaveBeenLastCalledWith(120); + confirmed = 100; + pinch.move(0.25); + expect(change).toHaveBeenLastCalledWith(150); + pinch.move(2); + expect(change).toHaveBeenLastCalledWith(180); + pinch.move(-3); + expect(change).toHaveBeenLastCalledWith(0); + pinch.move(NaN); + pinch.end(); + expect(change).toHaveBeenLastCalledWith(null); + const count = change.mock.calls.length; + pinch.move(1); + pinch.end(); + expect(change).toHaveBeenCalledTimes(count); + expect(pinch.active).toBe(false); +}); diff --git a/packages/client-runtime/src/device/duoControl.ts b/packages/client-runtime/src/device/duoControl.ts new file mode 100644 index 000000000000..b9f61ec997d9 --- /dev/null +++ b/packages/client-runtime/src/device/duoControl.ts @@ -0,0 +1,113 @@ +// @effect-diagnostics globalTimers:off - The stream owns this browser control queue and its timeout. +export const DUO_POSES = [ + { id: "closed", label: "Closed", angle: 0 }, + { id: "book", label: "Book", angle: 90 }, + { id: "open", label: "Open", angle: 180 }, + { id: "laptop", label: "Laptop", angle: 90 }, + { id: "tent", label: "Tent", angle: 80 }, +] as const; +export type DuoPose = (typeof DUO_POSES)[number]["id"]; +export type DuoOrientation = + | "portrait" + | "landscape_left" + | "portrait_upside_down" + | "landscape_right"; +export type DuoCommand = + | { control: "angle"; value: number } + | { control: "pose"; value: DuoPose } + | { control: "table"; value: boolean } + | { control: "physical"; value: "faceup" | "facedown" } + | { control: "orientation"; value: DuoOrientation }; +export type DuoControlState = { + pending: boolean; + requested: DuoCommand | null; + error: string | null; +}; + +/** One in-flight native transaction. Hinge motion coalesces; presets replace queued motion. Nothing replays after reconnect. */ +export function createDuoControl(options: { + send: (request: { requestId: number; command: DuoCommand }) => boolean; + onChange: (state: DuoControlState) => void; + timeoutMs?: number; +}) { + let nextId = 1; + let active: { requestId: number; command: DuoCommand } | null = null; + let queued: DuoCommand | null = null; + let timer: ReturnType | null = null; + const publish = (error: string | null = null) => + options.onChange({ pending: !!active, requested: queued ?? active?.command ?? null, error }); + const clear = (error: string | null = null) => { + if (timer) clearTimeout(timer); + timer = null; + active = null; + queued = null; + publish(error); + }; + const drain = () => { + if (active || !queued) return; + active = { requestId: nextId++, command: queued }; + queued = null; + const id = active.requestId; + timer = setTimeout(() => { + if (active?.requestId === id) clear("Device control timed out. Its position is unknown."); + }, options.timeoutMs ?? 5_000); + publish(); + if (!options.send(active)) clear("Device is disconnected."); + }; + return { + enqueue(command: DuoCommand) { + if ( + command.control === "angle" && + (!Number.isFinite(command.value) || command.value < 0 || command.value > 180) + ) + return; + queued = command; + if (active) publish(); + else drain(); + }, + receive(reply: { requestId: number; ok: boolean; error?: string }) { + if (!active || reply.requestId !== active.requestId) return; + if (timer) clearTimeout(timer); + timer = null; + active = null; + if (!reply.ok) { + clear(reply.error ?? "Device control failed. Its position is unknown."); + return; + } + if (queued) drain(); + else publish(); + }, + clear, + }; +} + +/** A pinch keeps its own accumulator across asynchronous native acknowledgements. */ +export function createDuoPinch(options: { + angle: () => number; + contains: (x: number, y: number) => boolean; + change: (angle: number | null) => void; +}) { + let angle: number | null = null; + return { + begin(x: number, y: number) { + if (!options.contains(x, y)) return false; + angle = Math.max(0, Math.min(180, options.angle())); + return true; + }, + move(logScale: number) { + if (angle === null || !Number.isFinite(logScale)) return; + const next = Math.max(0, Math.min(180, angle + logScale * 120)); + if (next === angle) return; + angle = next; + options.change(next); + }, + end() { + if (angle === null) return; + angle = null; + options.change(null); + }, + get active() { + return angle !== null; + }, + }; +} diff --git a/packages/client-runtime/src/device/duoScene.test.ts b/packages/client-runtime/src/device/duoScene.test.ts new file mode 100644 index 000000000000..4837bb6b71c3 --- /dev/null +++ b/packages/client-runtime/src/device/duoScene.test.ts @@ -0,0 +1,108 @@ +import { + BoxGeometry, + Group, + Mesh, + MeshBasicMaterial, + PerspectiveCamera, + PlaneGeometry, + Texture, + Vector3, +} from "three"; +import { expect, it } from "vite-plus/test"; +import { createDuoScene, duoDisplayKey, duoRawPoint } from "./duoScene.ts"; + +function fixture() { + const asset = new Group(); + const left = new Group(); + left.name = "left-half"; + const right = new Group(); + right.name = "right-half"; + const screen = (name: string, x: number, z: number, rear = false) => { + const mesh = new Mesh(new PlaneGeometry(1, 2), new MeshBasicMaterial()); + if (rear) mesh.geometry.rotateY(Math.PI); + mesh.geometry.translate(x, 0, z); + mesh.name = name; + return mesh; + }; + const innerLeft = screen("inner-display-left", -0.5, 0.06); + const innerRight = screen("inner-display-right", 0.5, 0.06); + const cover = screen("cover-display", -0.5, -0.06, true); + left.add(innerLeft, cover); + right.add(innerRight); + for (const [group, x] of [ + [left, -0.5], + [right, 0.5], + ] as const) { + const body = new Mesh(new BoxGeometry(1, 2, 0.1), new MeshBasicMaterial()); + body.geometry.translate(x, 0, 0); + group.add(body); + } + asset.add(left, right); + const scene = createDuoScene(asset, { 1: new Texture(), 3: new Texture() }); + const camera = new PerspectiveCamera(36, 1, 0.1, 50); + camera.position.z = 6; + camera.updateMatrixWorld(); + return { scene, camera, left, right, innerLeft, innerRight }; +} + +it("keeps one continuous inner UV map across the hinge and folds opposite leaves", () => { + const { scene, left, right, innerLeft, innerRight } = fixture(); + scene.setAngle(90); + expect(left.rotation.y).toBeCloseTo(Math.PI / 4); + expect(right.rotation.y).toBeCloseTo(-Math.PI / 4); + const values = (mesh: Mesh) => + Array.from({ length: mesh.geometry.getAttribute("uv").count }, (_, i) => + mesh.geometry.getAttribute("uv").getX(i), + ); + expect(Math.max(...values(innerLeft))).toBe(0.5); + expect(Math.min(...values(innerRight))).toBe(0.5); + scene.dispose(); +}); + +it("derives resting views from the hinged display planes independently of the inspection orbit", () => { + const { scene } = fixture(); + scene.setAngle(90); + const frames = scene.restFrames(3); + expect(frames.map((frame) => frame.face)).toEqual(["inside", "left", "right"]); + expect(frames[1]!.normal.x).toBeGreaterThan(0.5); + expect(frames[2]!.normal.x).toBeLessThan(-0.5); + scene.root.rotation.set(0.5, 1.2, -0.7); + scene.root.position.set(3, -2, 1); + const rotated = scene.restFrames(3); + for (let index = 0; index < frames.length; index++) { + expect(rotated[index]!.normal.distanceTo(frames[index]!.normal)).toBeLessThan(1e-6); + expect(rotated[index]!.center.distanceTo(frames[index]!.center)).toBeLessThan(1e-6); + } + scene.setAngle(180); + expect(scene.restFrames(3).map((frame) => frame.face)).toEqual(["inside"]); + expect(scene.restFrames(1).map((frame) => frame.face)).toEqual(["cover"]); + scene.dispose(); +}); + +it("maps active display input through hardware mounting and blocks rear, inactive and stale surfaces", () => { + const { scene, camera, innerLeft } = fixture(); + scene.setAngle(180); + const screen = { width: 2007, height: 2853, orientation: "portrait" as const, screenId: 3 }; + const key = duoDisplayKey(screen); + scene.root.updateMatrixWorld(); + const world = innerLeft.localToWorld(new Vector3(-0.5, -0.5, 0.06)).project(camera); + const x = (world.x + 1) / 2, + y = (1 - world.y) / 2; + const hit = scene.screenPoint(x, y, camera, screen, key); + expect(hit?.x).toBeCloseTo(0.75); + expect(hit?.y).toBeCloseTo(0.75); + expect( + scene.screenPoint( + x, + y, + camera, + { ...screen, screenId: 1 }, + duoDisplayKey({ ...screen, screenId: 1 }), + ), + ).toBeNull(); + expect(scene.screenPoint(x, y, camera, screen, "old")).toBeNull(); + expect(scene.screenPoint(x, y, camera, screen, key)).not.toBeNull(); + expect(scene.screenPoint(x, 1.1, camera, screen, key, true)?.x).toBeGreaterThan(1); + expect(duoRawPoint(1, 0.2, 0.7)).toEqual({ x: 0.2, y: 0.7 }); + scene.dispose(); +}); diff --git a/packages/client-runtime/src/device/duoScene.ts b/packages/client-runtime/src/device/duoScene.ts new file mode 100644 index 000000000000..eada17e5a8fc --- /dev/null +++ b/packages/client-runtime/src/device/duoScene.ts @@ -0,0 +1,244 @@ +import { + Box3, + BufferAttribute, + Group, + Mesh, + MeshBasicMaterial, + Matrix4, + Quaternion, + Plane, + Raycaster, + Triangle, + Vector2, + Vector3, + type PerspectiveCamera, + type Texture, +} from "three"; +import type { DuoRestFrame } from "./duoSnap.ts"; +import type { DeviceScreenSize } from "./stream.ts"; + +export type DuoPanelId = 1 | 3; +export type DuoHingeLeaf = "left" | "right"; +export type DuoFrameLayout = { width: number; height: number }; + +/** Hardware mounting is independent of app orientation and the model's orbit. The inverse is shared by taps and drags. */ +export function duoRawPoint(panel: DuoPanelId, x: number, y: number) { + return panel === 1 ? { x, y } : { x: y, y: 1 - x }; +} +export function duoFrameMatches(frame: DuoFrameLayout, screen: DeviceScreenSize) { + return ( + frame.width > 0 && + frame.height > 0 && + Math.abs(frame.width / frame.height - screen.width / screen.height) <= + 1 / frame.height + 1 / screen.height + ); +} +export function duoDisplayKey(screen: DeviceScreenSize | null) { + return screen ? `${screen.screenId}:${screen.width}:${screen.height}:${screen.orientation}` : ""; +} + +/** Asset resources belong to its model slot; this scene owns only live-screen replacement materials. */ +export function createDuoScene(asset: Group, textures: Record) { + const left = asset.getObjectByName("left-half"); + const right = asset.getObjectByName("right-half"); + const cover = asset.getObjectByName("cover-display"); + const innerLeft = asset.getObjectByName("inner-display-left"); + const innerRight = asset.getObjectByName("inner-display-right"); + if ( + !left || + !right || + !(cover instanceof Mesh) || + !(innerLeft instanceof Mesh) || + !(innerRight instanceof Mesh) + ) + throw new Error("Duo model must have two hinge groups and all three display meshes"); + const surfaces = { 1: [cover], 3: [innerLeft, innerRight] }; + const boundsByPanel = ([1, 3] as const).map((id) => { + const bounds = new Box3(); + for (const mesh of surfaces[id]) { + mesh.geometry.computeBoundingBox(); + bounds.union(mesh.geometry.boundingBox!); + } + const size = bounds.getSize(new Vector3()); + if (![size.x, size.y].every((value) => Number.isFinite(value) && value > 0)) + throw new Error("Invalid Duo display bounds"); + return { bounds, size }; + }); + const root = new Group(); + const content = new Group(); + content.add(asset); + root.add(content); + let currentAngle = Number.NaN; + const localBounds = new Box3(); + const relative = new Matrix4(); + const inverse = new Matrix4(); + const materials = { + 1: new MeshBasicMaterial({ map: textures[1], toneMapped: false }), + 3: new MeshBasicMaterial({ map: textures[3], toneMapped: false }), + }; + const originals = new Map(); + // The two inner meshes sample halves of one framebuffer. Cover UVs face outward on the rear leaf. + for (const id of [1, 3] as const) { + const { bounds, size } = boundsByPanel[id === 1 ? 0 : 1]!; + for (const mesh of surfaces[id]) { + const position = mesh.geometry.getAttribute("position"); + const uv = new Float32Array(position.count * 2); + for (let i = 0; i < position.count; i++) { + const x = (position.getX(i) - bounds.min.x) / size.x; + uv[i * 2] = id === 1 ? 1 - x : x; + uv[i * 2 + 1] = (position.getY(i) - bounds.min.y) / size.y; + } + mesh.geometry.setAttribute("uv", new BufferAttribute(uv, 2)); + originals.set(mesh, mesh.material); + mesh.material = materials[id]; + } + } + const ray = new Raycaster(); + let captured: { + panel: DuoPanelId; + plane: Plane; + triangle: Triangle; + uv: [Vector2, Vector2, Vector2]; + key: string; + } | null = null; + return { + root, + setAngle(angle: number) { + if (angle === currentAngle) return; + currentAngle = angle; + const radians = ((180 - Math.min(180, Math.max(0, angle))) * Math.PI) / 360; + left.rotation.y = radians; + right.rotation.y = -radians; + // Orbit the folded body's center, while retaining the authored hinge pivots. + content.position.set(0, 0, 0); + root.updateWorldMatrix(true, true); + inverse.copy(root.matrixWorld).invert(); + localBounds.makeEmpty(); + asset.traverse((object) => { + if (!(object instanceof Mesh)) return; + if (!object.geometry.boundingBox) object.geometry.computeBoundingBox(); + if (!object.geometry.boundingBox) return; + relative.multiplyMatrices(inverse, object.matrixWorld); + localBounds.union(object.geometry.boundingBox.clone().applyMatrix4(relative)); + }); + content.position.copy(localBounds.getCenter(new Vector3())).negate(); + }, + restFrames(panel: DuoPanelId): DuoRestFrame[] { + root.updateWorldMatrix(true, true); + inverse.copy(root.matrixWorld).invert(); + const frame = (meshes: Mesh[], face: DuoRestFrame["face"]): DuoRestFrame => { + const bounds = new Box3(); + const normal = new Vector3(); + const up = new Vector3(); + for (const mesh of meshes) { + if (!mesh.geometry.boundingBox) mesh.geometry.computeBoundingBox(); + relative.multiplyMatrices(inverse, mesh.matrixWorld); + bounds.union(mesh.geometry.boundingBox!.clone().applyMatrix4(relative)); + normal.add( + new Vector3() + .fromBufferAttribute(mesh.geometry.getAttribute("normal"), 0) + .transformDirection(relative), + ); + up.add(new Vector3(0, 1, 0).transformDirection(relative)); + } + return { + face, + center: bounds.getCenter(new Vector3()), + normal: normal.normalize(), + up: up.normalize(), + }; + }; + if (panel === 1) return [frame([cover], "cover")]; + const inside = frame([innerLeft, innerRight], "inside"); + // Leaf views only make sense when both displays form a useful open fold. + return currentAngle > 20 && currentAngle < 165 + ? [inside, frame([innerLeft], "left"), frame([innerRight], "right")] + : [inside]; + }, + leafRotation(leaf: DuoHingeLeaf) { + root.updateWorldMatrix(true, true); + relative.multiplyMatrices( + root.matrixWorld.clone().invert(), + (leaf === "left" ? left : right).matrixWorld, + ); + return new Quaternion().setFromRotationMatrix(relative.extractRotation(relative)); + }, + hingeLeafAt(x: number, y: number, camera: PerspectiveCamera): DuoHingeLeaf | null { + if (!Number.isFinite(x) || !Number.isFinite(y)) return null; + root.updateMatrixWorld(true); + camera.updateMatrixWorld(true); + ray.setFromCamera(new Vector2(x * 2 - 1, 1 - y * 2), camera); + let object = ray.intersectObject(root, true)[0]?.object; + while (object) { + if (object === left) return "left"; + if (object === right) return "right"; + object = object.parent ?? undefined; + } + return null; + }, + cancelInput() { + captured = null; + }, + screenPoint( + x: number, + y: number, + camera: PerspectiveCamera, + screen: DeviceScreenSize | null, + readyKey: string, + extend = false, + ) { + const key = duoDisplayKey(screen); + if (!screen || !readyKey || key !== readyKey) { + captured = null; + return null; + } + root.updateMatrixWorld(true); + camera.updateMatrixWorld(true); + ray.setFromCamera(new Vector2(x * 2 - 1, 1 - y * 2), camera); + if (extend && captured?.key === key) { + const point = ray.ray.intersectPlane(captured.plane, new Vector3()); + const bary = point ? captured.triangle.getBarycoord(point, new Vector3()) : null; + if (!bary) return null; + const uv = captured.uv[0] + .clone() + .multiplyScalar(bary.x) + .addScaledVector(captured.uv[1], bary.y) + .addScaledVector(captured.uv[2], bary.z); + return duoRawPoint(captured.panel, uv.x, 1 - uv.y); + } + // The chassis occludes rear displays. Only the first visible hit can own a contact. + const hit = ray.intersectObject(root, true)[0]; + if (!hit?.uv || !hit.face || !(hit.object instanceof Mesh)) return null; + const panel: DuoPanelId | null = surfaces[1].includes(hit.object) + ? 1 + : surfaces[3].includes(hit.object) + ? 3 + : null; + if (!panel || panel !== screen.screenId) return null; + const indices = [hit.face.a, hit.face.b, hit.face.c] as const; + const positions = hit.object.geometry.getAttribute("position"); + const triangle = new Triangle( + ...(indices.map((index) => + new Vector3().fromBufferAttribute(positions, index).applyMatrix4(hit.object.matrixWorld), + ) as [Vector3, Vector3, Vector3]), + ); + const hitUvs = hit.object.geometry.getAttribute("uv"); + const uv = indices.map((index) => new Vector2().fromBufferAttribute(hitUvs, index)) as [ + Vector2, + Vector2, + Vector2, + ]; + captured = { panel, triangle, uv, plane: triangle.getPlane(new Plane()), key }; + return duoRawPoint(panel, hit.uv.x, 1 - hit.uv.y); + }, + dispose() { + for (const [mesh, material] of originals) mesh.material = material; + for (const material of Object.values(materials)) { + material.map = null; + material.dispose(); + } + content.remove(asset); + captured = null; + }, + }; +} diff --git a/packages/client-runtime/src/device/duoSnap.test.ts b/packages/client-runtime/src/device/duoSnap.test.ts new file mode 100644 index 000000000000..9b312a4fb45e --- /dev/null +++ b/packages/client-runtime/src/device/duoSnap.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from "vite-plus/test"; +import { Quaternion, Vector3 } from "three"; +import { duoViewSnaps, nearestDuoView } from "./duoSnap.ts"; + +const frames = [ + { + face: "inside" as const, + normal: new Vector3(0, 0, 1), + up: new Vector3(0, 1, 0), + center: new Vector3(), + }, +]; +const rotation = (x: number, y: number, z = 0) => + new Quaternion().setFromAxisAngle(new Vector3(x, y, z).normalize(), Math.hypot(x, y, z)); + +it("chooses a leaf-focused seated view when it is closer than the middle of the fold", () => { + const leaf = { + face: "right" as const, + normal: new Vector3(0.6, 0, 0.8), + up: new Vector3(0, 1, 0), + center: new Vector3(1, 0, 0), + }; + const candidates = duoViewSnaps([...frames, leaf], 3); + const seat = candidates.find( + (candidate) => candidate.face === "right" && candidate.orientation === "portrait", + )!; + const released = seat.rotation.clone().premultiply(rotation(0.04, 0.05)); + const chosen = nearestDuoView(released, candidates)!; + expect(chosen.face).toBe("right"); + expect(chosen.orientation).toBe("portrait"); + expect(chosen.center.x).toBe(1); +}); + +it("a seated view exposes the base, and upright yaw keeps both inner displays facing the camera", () => { + const folded = [ + frames[0]!, + ...(["left", "right"] as const).map((face) => ({ + face, + normal: new Vector3(face === "left" ? Math.SQRT1_2 : -Math.SQRT1_2, 0, Math.SQRT1_2), + up: new Vector3(0, 1, 0), + center: new Vector3(face === "left" ? -1 : 1, 0, 0), + })), + ]; + const candidates = duoViewSnaps(folded, 3); + expect(candidates.filter((candidate) => candidate.face === "right")).toHaveLength(1); + for (const candidate of candidates) { + for (const frame of folded.slice(1)) + expect(frame.normal.clone().applyQuaternion(candidate.rotation).z).toBeGreaterThan(0.2); + const side = candidate.rotation.clone().premultiply(rotation(0, 1.5)); + const chosen = nearestDuoView(side, [candidate])!; + for (const frame of folded.slice(1)) + expect(frame.normal.clone().applyQuaternion(chosen.rotation).z).toBeGreaterThan(0.02); + } +}); diff --git a/packages/client-runtime/src/device/duoSnap.ts b/packages/client-runtime/src/device/duoSnap.ts new file mode 100644 index 000000000000..1e0e3f46fd0c --- /dev/null +++ b/packages/client-runtime/src/device/duoSnap.ts @@ -0,0 +1,86 @@ +import { Matrix4, Quaternion, Vector3 } from "three"; +import { nearestDeviceView } from "./deviceViewSnap.ts"; +import type { DeviceScreenSize } from "./stream.ts"; + +export type DuoRestFace = "cover" | "inside" | "left" | "right"; +export type DuoRestFrame = { face: DuoRestFace; normal: Vector3; up: Vector3; center: Vector3 }; +export type DuoViewSnap = { + rotation: Quaternion; + face: DuoRestFace; + orientation: DeviceScreenSize["orientation"]; + center: Vector3; + yawLimit: number; +}; +const orientations = [ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +] as const; +const z = new Vector3(0, 0, 1); +const y = new Vector3(0, 1, 0); + +/** Build views from the actual hinged display planes, rather than model-specific Euler offsets. */ +export function duoViewSnaps(frames: readonly DuoRestFrame[], panel: 1 | 3) { + const snaps: DuoViewSnap[] = []; + for (const frame of frames) { + const normal = frame.normal.clone().normalize(); + if (normal.lengthSq() < 0.5) continue; + const right = frame.up.clone().cross(normal).normalize(); + const up = normal.clone().cross(right).normalize(); + const faceRotation = new Quaternion() + .setFromRotationMatrix(new Matrix4().makeBasis(right, up, normal)) + .invert(); + for (let index = 0; index < orientations.length; index++) { + // Only these rolls put the partner below the focused lid. Other quarter + // turns belong to the upright, whole-display family. + if (frame.face === "right" && index !== 0) continue; + if (frame.face === "left" && index !== 2) continue; + const roll = (panel === 3 ? Math.PI / 2 : 0) - (index * Math.PI) / 2; + const rotation = faceRotation.clone().premultiply(new Quaternion().setFromAxisAngle(z, roll)); + // A leaf-focused view looks slightly down onto its partner, as in a seated laptop. + if (frame.face === "left" || frame.face === "right") + rotation.premultiply(new Quaternion().setFromAxisAngle(new Vector3(1, 0, 0), Math.PI / 12)); + if ( + panel === 3 && + frames.some( + (leaf) => + (leaf.face === "left" || leaf.face === "right") && + leaf.normal.clone().applyQuaternion(rotation).z < 0.04, + ) + ) + continue; + // Bound yaw using both real leaf normals. A narrow fold cannot tolerate + // the same side view as a fully open display. + let yawLimit = frame.face === "cover" ? Math.PI / 9 : Math.PI / 3; + if (panel === 3) { + for (const sign of [-1, 1]) { + for (let step = 1; step <= 60; step++) { + const turn = new Quaternion().setFromAxisAngle(y, (sign * step * Math.PI) / 180); + if ( + frames.some( + (leaf) => + (leaf.face === "left" || leaf.face === "right") && + leaf.normal.clone().applyQuaternion(rotation).applyQuaternion(turn).z < 0.04, + ) + ) { + yawLimit = Math.min(yawLimit, ((step - 1) * Math.PI) / 180); + break; + } + } + } + } + snaps.push({ + rotation, + face: frame.face, + orientation: orientations[index]!, + center: frame.center.clone(), + yawLimit, + }); + } + } + return snaps; +} + +/** Fold-specific candidates use the base viewer's nearest-family selection. */ +export const nearestDuoView = nearestDeviceView; diff --git a/packages/client-runtime/src/device/duoStream.test.ts b/packages/client-runtime/src/device/duoStream.test.ts new file mode 100644 index 000000000000..f290bef1712d --- /dev/null +++ b/packages/client-runtime/src/device/duoStream.test.ts @@ -0,0 +1,285 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { createDeviceStreamClient, type DeviceScreenSize } from "./stream.ts"; +afterEach(() => vi.unstubAllGlobals()); + +it("switches to fixed authenticated feeds while retaining one HID socket, routes the active panel and rejects superseded decoded output", async () => { + const feeds: { + url: string; + signal: AbortSignal; + controller: ReadableStreamDefaultController; + }[] = []; + const outputs: VideoFrameOutputCallback[] = []; + const errors: VideoDecoderInit["error"][] = []; + let supported = true; + let panelHttpStatus = 200; + let expected = 0; + let decoderReady = () => {}; + const waitDecoders = (count: number) => { + expected = count; + return new Promise((resolve) => { + decoderReady = resolve; + }); + }; + class Decoder { + static isConfigSupported = async () => ({ supported }); + static instances: Decoder[] = []; + state = "unconfigured"; + constructor(options: VideoDecoderInit) { + outputs.push(options.output); + errors.push(options.error); + Decoder.instances.push(this); + if (outputs.length === expected) decoderReady(); + } + configure() { + this.state = "configured"; + } + close() { + this.state = "closed"; + } + } + let socketReady = () => {}; + const socketConstructed = new Promise((resolve) => { + socketReady = resolve; + }); + class Socket { + static OPEN = 1; + static instances: Socket[] = []; + readyState = 1; + binaryType = "arraybuffer"; + onopen?: () => void; + onmessage?: (event: { data: ArrayBuffer }) => void; + onclose?: (event: { code: number; reason: string }) => void; + send = vi.fn(); + close = vi.fn(); + constructor() { + Socket.instances.push(this); + socketReady(); + } + } + vi.stubGlobal("VideoDecoder", Decoder); + vi.stubGlobal("EncodedVideoChunk", vi.fn()); + vi.stubGlobal("WebSocket", Socket); + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, options: { signal: AbortSignal }) => { + if (!url.includes("avcc")) return new Response("prime"); + if (url.includes("/panel/") && panelHttpStatus !== 200) + return new Response("unsupported panel", { status: panelHttpStatus }); + const stream = new ReadableStream({ + start(controller) { + feeds.push({ url, signal: options.signal, controller }); + options.signal.addEventListener("abort", () => + controller.error(new DOMException("Aborted", "AbortError")), + ); + }, + }); + return new Response(stream); + }), + ); + const present = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true), + cover = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true), + inner = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true); + let panelFailed = () => {}; + const panelFailure = new Promise((resolve) => { + panelFailed = resolve; + }); + const onStatus = vi.fn(); + const onDuoControl = vi.fn(); + const onDuoUnavailable = vi.fn((_detail?: string) => panelFailed()); + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "duo", + access: { + httpBase: "https://t3.test/api/device-hub", + wsBase: "wss://t3.test/api/device-hub", + credentials: true, + query: { hostId: "remote", wsTicket: "ticket" }, + }, + }, + { present }, + { + onStatus, + onDuoControl, + onDuoUnavailable, + onScreen: vi.fn(), + onInputConnected: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + }, + ); + client.start(); + await socketConstructed; + const ws = Socket.instances[0]!; + const config = ( + id: number, + orientation: DeviceScreenSize["orientation"] = "portrait", + physical = false, + ) => { + const json = new TextEncoder().encode( + JSON.stringify({ + width: id === 1 ? 1398 : 2007, + height: id === 1 ? 2034 : 2853, + orientation, + screenId: id, + supportsHingeAngle: true, + supportsPhysicalOrientation: physical, + hingePose: "open", + }), + ); + const packet = new Uint8Array(json.length + 1); + packet[0] = 0x82; + packet.set(json, 1); + ws.onmessage?.({ data: packet.buffer }); + }; + config(3); + const requestedOrientation = () => + JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1))).orientation; + client.rotate(); + expect(requestedOrientation()).toBe("landscape_left"); + config(3); // An orientation-locked app keeps its framebuffer orientation after the sensor rotates. + expect(onDuoControl.mock.lastCall?.[0].error).toBeNull(); + client.rotate(); + expect(requestedOrientation()).toBe("portrait_upside_down"); + config(3, "portrait_upside_down"); + client.rotate(); + expect(requestedOrientation()).toBe("landscape_right"); + config(3); // An external native orientation change becomes authoritative again. + client.rotate(); + expect(requestedOrientation()).toBe("landscape_left"); + config(3, "landscape_left"); + expect(onDuoControl.mock.lastCall?.[0].pending).toBe(false); + client.controlDuo({ control: "angle", value: 40 }); + const angleRequest = JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1))); + const before = ws.send.mock.calls.length; + client.controlDuo({ control: "orientation", value: "portrait" }); + expect(ws.send.mock.calls.length).toBe(before); + config(3, "landscape_left"); // The angle's config precedes its receipt; it cannot acknowledge a queued rotation. + const reply = new TextEncoder().encode( + JSON.stringify({ requestId: angleRequest.requestId, ok: true }), + ); + const receipt = new Uint8Array(reply.length + 1); + receipt[0] = 0x90; + receipt.set(reply, 1); + ws.onmessage?.({ data: receipt.buffer }); + expect(requestedOrientation()).toBe("portrait"); + expect(ws.send.mock.lastCall?.[0][0]).toBe(0x07); + client.controlDuo({ control: "angle", value: 55 }); + const orientationSends = ws.send.mock.calls.length; + expect(onDuoControl.mock.lastCall?.[0].pending).toBe(true); + config(3); + expect(ws.send.mock.calls.length).toBe(orientationSends + 1); + const after = JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1))); + expect(after.command).toEqual({ control: "angle", value: 55 }); + const finalReply = new TextEncoder().encode( + JSON.stringify({ requestId: after.requestId, ok: true }), + ); + const finalReceipt = new Uint8Array(finalReply.length + 1); + finalReceipt[0] = 0x90; + finalReceipt.set(finalReply, 1); + ws.onmessage?.({ data: finalReceipt.buffer }); + expect(onDuoControl.mock.lastCall?.[0].pending).toBe(false); + const description = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]); + let ready = waitDecoders(1); + feeds[0]!.controller.enqueue(description); + await ready; + client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } }); + expect(feeds[0]!.signal.aborted).toBe(true); + expect(Socket.instances).toHaveLength(1); + for (const [index, id] of [ + [1, 1], + [2, 3], + ]) { + const url = new URL(feeds[index!]!.url); + expect(url.pathname.endsWith(`/panel/${id}/stream.avcc`)).toBe(true); + expect(url.searchParams.get("hostId")).toBe("remote"); + expect(url.searchParams.get("wsTicket")).toBe("ticket"); + } + ready = waitDecoders(3); + feeds[1]!.controller.enqueue(description); + feeds[2]!.controller.enqueue(description); + await ready; + const frame = { + displayWidth: 2007, + displayHeight: 2853, + close: vi.fn(), + } as unknown as VideoFrame; + outputs[0]!(frame); + expect(present).not.toHaveBeenCalled(); + outputs[1]!(frame); + expect(cover).not.toHaveBeenCalled(); + outputs[2]!(frame); + expect(inner).toHaveBeenCalledOnce(); + expect(present).toHaveBeenCalledOnce(); + config(1); + outputs[2]!(frame); + expect(inner).toHaveBeenCalledOnce(); + outputs[1]!(frame); + expect(cover).toHaveBeenCalledOnce(); + client.setDuoPanels(null); + expect(feeds[1]!.signal.aborted).toBe(true); + expect(feeds[2]!.signal.aborted).toBe(true); + expect(feeds[3]!.url).not.toContain("/panel/"); + outputs[1]!(frame); + expect(cover).toHaveBeenCalledOnce(); + ready = waitDecoders(4); + feeds[3]!.controller.enqueue(description); + await ready; + errors[0]!(new DOMException("Late decoder failure")); + expect(Decoder.instances[3]!.state).toBe("configured"); + config(1, "portrait", true); + client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } }); + expect(feeds).toHaveLength(5); // One active feed replaces the two fixed feeds. + expect(feeds[4]!.url).not.toContain("/panel/"); + expect(new URL(feeds[4]!.url).searchParams.get("hostId")).toBe("remote"); + ready = waitDecoders(5); + feeds[4]!.controller.enqueue(description); + await ready; + const primary = { ...frame, displayWidth: 1398, displayHeight: 2034 } as VideoFrame; + const painted = present.mock.calls.length; + outputs[4]!(primary); + expect(present).toHaveBeenCalledTimes(painted + 1); + expect(present.mock.lastCall?.[0]).toBe(primary); + expect(cover).toHaveBeenCalledOnce(); // The main sink owns primary texture delivery. + config(3, "portrait", true); + expect(feeds[4]!.signal.aborted).toBe(true); + expect(feeds).toHaveLength(6); + ready = waitDecoders(6); + feeds[5]!.controller.enqueue(description); + await ready; + outputs[4]!(primary); // The old elected display cannot paint after handoff. + expect(present).toHaveBeenCalledTimes(painted + 1); + outputs[5]!(frame); + expect(present).toHaveBeenCalledTimes(painted + 2); + expect(present.mock.lastCall?.[0]).toBe(frame); + config(3, "portrait", true); + expect(feeds).toHaveLength(6); // Duplicate native readback does not reconnect. + supported = false; + client.setDuoPanels(null); + client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } }); + expect(feeds[7]!.url).not.toContain("/panel/"); + feeds[7]!.controller.enqueue(description); + await panelFailure; + expect(onDuoUnavailable.mock.lastCall?.[0]).toContain("cannot decode"); + config(3); + const fixedPanelFailure = new Promise((resolve) => { + panelFailed = resolve; + }); + client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } }); + feeds[8]!.controller.enqueue(description); + await fixedPanelFailure; + expect(onDuoUnavailable).toHaveBeenCalled(); + expect(onStatus.mock.calls.some(([status]) => status === "error")).toBe(false); + panelHttpStatus = 404; + const missingPanel = new Promise((resolve) => { + panelFailed = resolve; + }); + client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } }); + await missingPanel; + expect(onDuoUnavailable.mock.lastCall?.[0]).toContain("does not provide fixed Duo"); + expect(Socket.instances).toHaveLength(1); + client.stop(); + expect(feeds[3]!.signal.aborted).toBe(true); + expect(ws.close).toHaveBeenCalledOnce(); + expect(frame.close).toHaveBeenCalledTimes(9); +}); diff --git a/packages/client-runtime/src/device/duoViewer.test.ts b/packages/client-runtime/src/device/duoViewer.test.ts new file mode 100644 index 000000000000..b85d306bcfd6 --- /dev/null +++ b/packages/client-runtime/src/device/duoViewer.test.ts @@ -0,0 +1,703 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import type { Scene, PerspectiveCamera, Box3, Quaternion } from "three"; + +const gpu = vi.hoisted(() => ({ + instances: [] as { + blank: boolean; + allocations: number; + frames: Scene[]; + views: { camera: PerspectiveCamera; bounds: Box3; quaternion: Quaternion }[]; + dispose: ReturnType; + forceContextLoss: ReturnType; + }[], + environmentDispose: vi.fn(), +})); +vi.mock("three", async () => { + const actual = await vi.importActual("three"); + return { + ...actual, + WebGLRenderer: class { + state = { + blank: true, + allocations: 0, + frames: [] as Scene[], + views: [] as { camera: PerspectiveCamera; bounds: Box3; quaternion: Quaternion }[], + dispose: vi.fn(), + forceContextLoss: vi.fn(), + }; + constructor() { + gpu.instances.push(this.state); + } + setDrawingBufferSize() { + this.state.blank = true; + this.state.allocations++; + } + render(scene: Scene, camera: PerspectiveCamera) { + this.state.blank = false; + this.state.frames.push(scene); + const root = scene.children.find((child) => child instanceof actual.Group)!; + root.updateMatrixWorld(true); + camera.updateMatrixWorld(true); + this.state.views.push({ + camera: camera.clone(), + bounds: new actual.Box3().setFromObject(root), + quaternion: root.quaternion.clone(), + }); + } + dispose() { + this.state.dispose(); + } + forceContextLoss() { + this.state.forceContextLoss(); + } + }, + PMREMGenerator: class { + fromScene() { + return { texture: new actual.Texture(), dispose: gpu.environmentDispose }; + } + dispose() {} + }, + }; +}); +const models = vi.hoisted(() => ({ + resolve: (_model: { asset: import("three").Group; dispose: () => void }) => {}, + signal: null as AbortSignal | null, +})); +vi.mock("./modelScene.ts", () => ({ + loadDeviceModel: (_source: unknown, signal: AbortSignal) => { + models.signal = signal; + return new Promise((resolve) => { + models.resolve = resolve; + }); + }, +})); +import { + Group, + Mesh, + MeshBasicMaterial, + PlaneGeometry, + Vector3, + Euler, + Quaternion as Rotation, +} from "three"; +import { createDuoViewer } from "./duoViewer.ts"; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + gpu.instances.length = 0; + gpu.environmentDispose.mockClear(); +}); + +function fixture( + reduced = true, + onOrientationRequested = vi.fn(), + onPanelRequested?: (panel: 1 | 3) => void, +) { + const pending = new Map(); + let id = 0; + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + pending.set(++id, callback); + return id; + }); + vi.stubGlobal("cancelAnimationFrame", (id: number) => pending.delete(id)); + vi.stubGlobal("matchMedia", () => ({ matches: reduced })); + const drawImage = vi.fn(); + vi.stubGlobal("document", { + createElement: () => ({ + width: 0, + height: 0, + getContext: () => ({ + fillRect() {}, + save() {}, + restore() {}, + translate() {}, + rotate() {}, + drawImage, + getImageData: () => ({ data: new Uint8ClampedArray(8 * 8 * 4).fill(255) }), + }), + }), + }); + const canvas = new EventTarget() as HTMLCanvasElement; + const viewer = createDuoViewer({ + canvas, + sources: { + 1: { width: 1398, height: 2034 } as HTMLCanvasElement, + 3: { width: 2007, height: 2853 } as HTMLCanvasElement, + }, + model: { id: "iphone-duo", url: "/duo.glb" }, + onUnavailable: vi.fn(), + onOrientationRequested, + ...(onPanelRequested ? { onPanelRequested } : {}), + }); + const draw = () => { + const callbacks = [...pending.values()]; + pending.clear(); + callbacks.forEach((callback) => callback(0)); + }; + return { viewer, draw, pending, state: gpu.instances[0]!, onOrientationRequested, drawImage }; +} + +function asset() { + const group = new Group(); + for (const [name, names] of [ + ["left-half", ["cover-display", "inner-display-left"]], + ["right-half", ["inner-display-right"]], + ] as const) { + const leaf = new Group(); + leaf.name = name; + for (const name of names) { + const mesh = new Mesh(new PlaneGeometry(1, 2), new MeshBasicMaterial()); + if (name === "cover-display") mesh.geometry.rotateY(Math.PI); + mesh.geometry.translate( + name === "inner-display-right" ? 0.5 : -0.5, + 0, + name === "cover-display" ? -0.05 : 0.05, + ); + mesh.name = name; + leaf.add(mesh); + } + group.add(leaf); + } + return group; +} + +it("coalesces resize with redraw, retains the renderer and scene, and settles without an idle animation loop", async () => { + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw, pending, state } = fixture(); + const dispose = vi.fn(); + models.resolve({ asset: asset(), dispose }); + await Promise.resolve(); + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "portrait", + screenId: 1, + hingeAngle: 0, + }); + viewer.resize(500, 700, 2); + draw(); + const scene = state.frames.at(-1); + viewer.resize(450, 700, 2); + viewer.resize(400, 700, 2); + viewer.orbit(0.1, 0.04); + expect(state.blank).toBe(false); + expect(state.allocations).toBe(1); + expect(pending.size).toBe(1); + draw(); + expect(state.blank).toBe(false); + expect(state.allocations).toBe(2); + expect(state.frames.at(-1)).toBe(scene); + expect(gpu.instances).toHaveLength(1); + now += 1000; + draw(); + draw(); + expect(pending.size).toBe(0); + viewer.dispose(); + viewer.dispose(); + expect(dispose).toHaveBeenCalledOnce(); + expect(state.dispose).toHaveBeenCalledOnce(); + expect(gpu.environmentDispose).toHaveBeenCalledOnce(); + expect(pending.size).toBe(0); +}); + +it("cancels loading and disposes a late model after unmount without installing or rendering it", async () => { + const { viewer, draw, state } = fixture(); + viewer.resize(400, 700, 2); + viewer.dispose(); + expect(models.signal?.aborted).toBe(true); + const dispose = vi.fn(); + models.resolve({ asset: asset(), dispose }); + await Promise.resolve(); + draw(); + expect(dispose).toHaveBeenCalledOnce(); + expect(state.frames).toHaveLength(0); +}); + +it("keeps the chosen release view when an app locks orientation, freezes it during contact, and stops drawing after release", async () => { + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw, state, pending, onOrientationRequested } = fixture(); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + const screen = { + width: 1398, + height: 2034, + orientation: "portrait" as const, + screenId: 1, + hingeAngle: 0, + }; + viewer.setScreen(screen); + viewer.resize(500, 700, 2); + draw(); + const initial = state.views.at(-1)!.quaternion.clone(); + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(0, Math.PI / 3); + draw(); + const held = state.views.at(-1)!.quaternion.clone(); + now = 1000; + viewer.frameUpdated(1); + draw(); + expect(state.views.at(-1)!.quaternion.angleTo(held)).toBeLessThan(1e-6); + expect(onOrientationRequested).not.toHaveBeenCalled(); + viewer.setInteractionActive(false, "orbit"); + draw(); + expect(onOrientationRequested).toHaveBeenCalledOnce(); + const chosen = state.views.at(-1)!.quaternion.clone(); + expect(chosen.angleTo(initial)).toBeGreaterThan(1); + viewer.setScreen({ ...screen }); // Native config confirms the sensor command, even if the app stays portrait. + draw(); + expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6); + expect(pending.size).toBe(0); + viewer.resetPose(); + draw(); + expect(onOrientationRequested).toHaveBeenCalledTimes(2); + expect(onOrientationRequested.mock.lastCall?.[0]).toBe("portrait"); + viewer.setScreen({ ...screen }); + draw(); + expect(pending.size).toBe(0); + viewer.dispose(); +}); + +it("settles a hinge transition after a throttled frame instead of stretching time", async () => { + let now = 100; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw, pending } = fixture(false); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(400, 700, 2); + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "portrait", + screenId: 1, + hingeAngle: 0, + }); + draw(); + viewer.setScreen({ + width: 2007, + height: 2853, + orientation: "portrait", + screenId: 3, + hingeAngle: 180, + }); + now += 1_000; + draw(); + draw(); + expect(pending.size).toBe(0); + viewer.dispose(); +}); + +it("faces a display handed off by native rotation without requesting another sensor rotation", async () => { + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw, pending, onOrientationRequested } = fixture(); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + viewer.setScreen({ + width: 2007, + height: 2853, + orientation: "portrait", + screenId: 3, + hingeAngle: 90, + }); + draw(); + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(0, Math.PI / 6); + viewer.setInteractionActive(false, "orbit"); + now = 1000; + draw(); + const requests = onOrientationRequested.mock.calls.length; + expect(requests).toBe(1); + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "landscape_left", + screenId: 1, + hingeAngle: 90, + }); + draw(); + draw(); + const cover = loaded.getObjectByName("cover-display") as Mesh; + const normal = new Vector3() + .fromBufferAttribute(cover.geometry.getAttribute("normal"), 0) + .transformDirection(cover.matrixWorld); + expect(normal.z).toBeGreaterThan(0.45); + expect(onOrientationRequested).toHaveBeenCalledTimes(requests); + expect(pending.size).toBe(0); + viewer.dispose(); +}); + +it.each(["book", "laptop", "open"] as const)( + "faces native cover readback for %s even when configuration arrives before the model", + async (hingePose) => { + const { viewer, draw } = fixture(); + viewer.resize(500, 700, 2); + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "landscape_left", + screenId: 1, + hingeAngle: hingePose === "open" ? 180 : 90, + hingePose, + }); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + draw(); + draw(); + const cover = loaded.getObjectByName("cover-display") as Mesh; + const normal = new Vector3() + .fromBufferAttribute(cover.geometry.getAttribute("normal"), 0) + .transformDirection(cover.matrixWorld); + expect(normal.z).toBeGreaterThan(0.45); + viewer.dispose(); + }, +); + +it("keeps default-zoom orbits framed across folds and fits the current assembly", async () => { + const { viewer, draw, state } = fixture(); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(380, 620, 2); + viewer.resetPose(); + viewer.setInteractionActive(true, "orbit"); + let minimumDistance = Infinity; + let maximumDistance = 0; + for (const hingeAngle of [0, 30, 90, 127, 180]) { + viewer.setScreen({ + width: 2007, + height: 2853, + orientation: "portrait", + screenId: 3, + hingeAngle, + }); + for (let turn = 0; turn < 16; turn++) { + viewer.orbit(0.1, turn % 2 ? -0.08 : 0.08); + draw(); + const { camera, bounds } = state.views.at(-1)!; + const center = bounds.getCenter(new Vector3()).project(camera); + expect(Math.abs(center.x)).toBeLessThan(0.18); + expect(Math.abs(center.y)).toBeLessThan(0.18); + state.frames.at(-1)!.traverse((object) => { + if (!(object instanceof Mesh)) return; + const positions = object.geometry.getAttribute("position"); + for (let index = 0; index < positions.count; index++) { + const point = new Vector3() + .fromBufferAttribute(positions, index) + .applyMatrix4(object.matrixWorld) + .project(camera); + expect(Math.abs(point.x)).toBeLessThan(1); + expect(Math.abs(point.y)).toBeLessThan(1); + } + }); + minimumDistance = Math.min(minimumDistance, camera.position.z); + maximumDistance = Math.max(maximumDistance, camera.position.z); + } + } + expect(maximumDistance).toBeGreaterThan(minimumDistance); + viewer.dispose(); +}); + +it("previews hinge articulation while preserving the laptop lid orientation and blocking unconfirmed input", async () => { + const { viewer, draw, pending } = fixture(); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + viewer.setScreen({ + width: 2853, + height: 2007, + orientation: "landscape_left", + screenId: 3, + hingeAngle: 90, + hingePose: "laptop", + }); + draw(); + const lid = loaded.getObjectByName("right-half")!; + const presentation = lid.getWorldQuaternion(new Rotation()); + for (const value of [127, 50, 170]) { + viewer.setHingePreview(value); + draw(); + expect(loaded.getObjectByName("left-half")!.rotation.y).toBeCloseTo( + ((180 - value) * Math.PI) / 360, + ); + expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001); + expect(viewer.screenPoint(0.5, 0.5)).toBeNull(); + viewer.setScreen({ + width: 2007, + height: 2853, + orientation: "portrait", + screenId: 3, + hingeAngle: value, + hingePose: null, + }); + draw(); + expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001); + } + viewer.setHingePreview(null); + draw(); + expect(pending.size).toBe(0); + expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001); + viewer.dispose(); +}); + +it("does not restart an animated preset on duplicate native configurations", async () => { + let now = 100; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw, pending } = fixture(false); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(400, 700, 2); + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "portrait", + screenId: 1, + hingeAngle: 0, + hingePose: "closed", + }); + draw(); + const next = { + width: 2007, + height: 2853, + orientation: "portrait" as const, + screenId: 3, + hingeAngle: 180, + hingePose: "open" as const, + }; + viewer.setScreen(next); + now += 1_000; + viewer.setScreen(next); + draw(); + now += 1000; + draw(); + draw(); + expect(pending.size).toBe(0); + viewer.dispose(); +}); + +it("lets standalone rotation leave Laptop and Tent and stand a closed device upright after hinge edits", async () => { + const { viewer, draw, state } = fixture(); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + viewer.resetPose(); + for (const hingePose of ["laptop", "tent"] as const) { + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "landscape_left", + screenId: 1, + hingeAngle: 90, + hingePose, + }); + draw(); + viewer.setHingePreview(0); + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "landscape_left", + screenId: 1, + hingeAngle: 0, + hingePose: null, + }); + viewer.setHingePreview(null); + draw(); + viewer.cancelInput(); // Explicit toolbar rotation ends folding ownership. + viewer.setScreen({ + width: 1398, + height: 2034, + orientation: "portrait", + screenId: 1, + hingeAngle: 0, + hingePose: null, + }); + draw(); + const upright = new Rotation().setFromEuler(new Euler(0, Math.PI / 2, 0, "YXZ")); + expect(state.views.at(-1)!.quaternion.angleTo(upright)).toBeLessThan(0.00001); + const bounds = state.views.at(-1)!.bounds; + expect(bounds.max.y - bounds.min.y).toBeGreaterThan(bounds.max.x - bounds.min.x); + } + viewer.dispose(); +}); + +it("snaps onto the opposite screen, requests native handoff once and retains the chosen view on readback", async () => { + vi.useFakeTimers(); + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const onPanelRequested = vi.fn(); + const { viewer, draw, state, pending, onOrientationRequested } = fixture( + true, + vi.fn(), + onPanelRequested, + ); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + const inner = { + width: 2007, + height: 2853, + orientation: "portrait" as const, + screenId: 3, + hingeAngle: 90, + supportsPhysicalOrientation: true, + }; + viewer.setScreen(inner); + draw(); + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(Math.PI / 3, 0); + draw(); + viewer.setInteractionActive(false, "orbit"); + draw(); + expect(onPanelRequested).toHaveBeenCalledExactlyOnceWith(1); + expect(onOrientationRequested).not.toHaveBeenCalled(); + expect(viewer.screenPoint(0.5, 0.5)).toBeNull(); + const chosen = state.views.at(-1)!.quaternion.clone(); + viewer.setScreen({ ...inner, hingePose: null }); // Command acknowledgement precedes sensor readback. + draw(); + expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6); + viewer.setScreen({ ...inner, width: 1398, height: 2034, screenId: 1 }); + draw(); + expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6); + vi.advanceTimersByTime(5000); + expect(onPanelRequested).toHaveBeenCalledTimes(1); + expect(pending.size).toBe(0); + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(-Math.PI / 3, 0); + draw(); + viewer.setInteractionActive(false, "orbit"); + draw(); + expect(onPanelRequested).toHaveBeenLastCalledWith(3); + const reverse = state.views.at(-1)!.quaternion.clone(); + viewer.setScreen(inner); + draw(); + expect(state.views.at(-1)!.quaternion.angleTo(reverse)).toBeLessThan(1e-6); + vi.advanceTimersByTime(5000); + draw(); + expect(pending.size).toBe(0); + expect(onPanelRequested).toHaveBeenCalledTimes(2); + viewer.dispose(); + vi.useRealTimers(); +}); + +it("rolls an unconfirmed opposite-screen snap back to the native active screen after timeout", async () => { + vi.useFakeTimers(); + const request = vi.fn(); + const { viewer, draw, state, pending } = fixture(true, vi.fn(), request); + models.resolve({ asset: asset(), dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + viewer.setScreen({ + width: 2007, + height: 2853, + orientation: "portrait", + screenId: 3, + hingeAngle: 90, + supportsPhysicalOrientation: true, + }); + draw(); + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(Math.PI / 3, 0); + draw(); + viewer.setInteractionActive(false, "orbit"); + draw(); + const unconfirmed = state.views.at(-1)!.quaternion.clone(); + expect(request).toHaveBeenCalledExactlyOnceWith(1); + vi.advanceTimersByTime(5000); + draw(); + expect(state.views.at(-1)!.quaternion.angleTo(unconfirmed)).toBeGreaterThan(0.5); + expect(request).toHaveBeenCalledOnce(); + draw(); + expect(pending.size).toBe(0); + viewer.dispose(); +}); + +it("uses the elected primary frame during handoff and never lets a stale fixed-panel feed overwrite it", async () => { + const { viewer, drawImage } = fixture(); + const cover = { + width: 1398, + height: 2034, + orientation: "portrait" as const, + screenId: 1, + hingeAngle: 90, + }; + viewer.setScreen(cover); + const primary = { width: 1398, height: 2034 } as HTMLCanvasElement; + viewer.frameUpdated(1, primary); + expect(drawImage.mock.calls.at(-1)?.[0]).toBe(primary); + const uploads = drawImage.mock.calls.length; + viewer.frameUpdated(1); + expect(drawImage).toHaveBeenCalledTimes(uploads); + viewer.frameUpdated(3, primary); + expect(drawImage).toHaveBeenCalledTimes(uploads); + viewer.setScreen({ ...cover, width: 2007, height: 2853, screenId: 3 }); + const inner = { width: 2007, height: 2853 } as HTMLCanvasElement; + viewer.frameUpdated(3, inner); + expect(drawImage.mock.calls.at(-1)?.[0]).toBe(inner); + viewer.setScreen(cover); + viewer.frameUpdated(1); + expect(drawImage.mock.calls.length).toBeGreaterThan(uploads + 1); + viewer.dispose(); +}); + +it.each(["left", "right"] as const)( + "keeps the primary surface oriented when pinching over the %s half through closure and reopening", + async (face) => { + const { viewer, draw, state, pending, onOrientationRequested } = fixture(); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + const inner = { + width: 2007, + height: 2853, + orientation: "portrait" as const, + screenId: 3, + hingeAngle: 180, + hingePose: "open" as const, + }; + viewer.setScreen(inner); + draw(); + expect(viewer.beginHinge(-1, -1)).toBe(false); + const leaf = loaded.getObjectByName("right-half")!; + const mesh = loaded.getObjectByName(`inner-display-${face}`) as Mesh; + const center = mesh.geometry.boundingBox!.getCenter(new Vector3()); + const point = mesh.localToWorld(center).project(state.views.at(-1)!.camera); + expect(viewer.beginHinge((point.x + 1) / 2, (1 - point.y) / 2)).toBe(true); + const orientation = leaf.getWorldQuaternion(new Rotation()); + const partner = loaded.getObjectByName("left-half")!; + const partnerStart = partner.getWorldQuaternion(new Rotation()); + for (const value of [150, 90, 30, 0, 30, 90, 180]) { + viewer.setHingePreview(value); + draw(); + expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6); + if (value === 90) + expect(partner.getWorldQuaternion(new Rotation()).angleTo(partnerStart)).toBeGreaterThan(1); + const next = + value === 0 + ? { ...inner, width: 1398, height: 2034, screenId: 1, hingeAngle: value, hingePose: null } + : { ...inner, hingeAngle: value, hingePose: null }; + viewer.setScreen(next); + viewer.setHingePreview(null); + draw(); + viewer.setScreen({ ...next }); // Late native readback cannot change the view. + draw(); + expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6); + expect(viewer.screenPoint(0.5, 0.5)).toBeNull(); // No fresh native frame yet. + if (value === 0) { + const cover = loaded.getObjectByName("cover-display") as Mesh; + const center = cover.geometry.boundingBox!.getCenter(new Vector3()); + const hit = cover.localToWorld(center).project(state.views.at(-1)!.camera); + expect(viewer.beginHinge((hit.x + 1) / 2, (1 - hit.y) / 2)).toBe(true); + } + } + expect(onOrientationRequested).not.toHaveBeenCalled(); + draw(); + expect(pending.size).toBe(0); + viewer.dispose(); + }, +); diff --git a/packages/client-runtime/src/device/duoViewer.ts b/packages/client-runtime/src/device/duoViewer.ts new file mode 100644 index 000000000000..7feca7452e0d --- /dev/null +++ b/packages/client-runtime/src/device/duoViewer.ts @@ -0,0 +1,566 @@ +// @effect-diagnostics globalTimers:off - Native display handoff has a bounded acknowledgement window. +import { + AmbientLight, + Box3, + CanvasTexture, + DirectionalLight, + Euler, + LinearFilter, + PerspectiveCamera, + PMREMGenerator, + Quaternion, + Scene, + SRGBColorSpace, + Vector3, + WebGLRenderer, +} from "three"; +import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js"; +import { + createDuoScene, + duoDisplayKey, + duoFrameMatches, + type DuoPanelId, + type DuoHingeLeaf, +} from "./duoScene.ts"; +import { loadDeviceModel } from "./modelScene.ts"; +import { createDeviceModelSlot, type DeviceModelSource } from "./model.ts"; +import { createDeviceFraming } from "./deviceFraming.ts"; +import { createDeviceMotion } from "./deviceMotion.ts"; +import { duoViewSnaps, nearestDuoView, type DuoRestFace } from "./duoSnap.ts"; +import { createRenderScheduler } from "./renderScheduler.ts"; +import type { DeviceScreenSize } from "./stream.ts"; +import type { DuoPose } from "./duoControl.ts"; + +export interface DuoViewer { + readonly setScreen: (screen: DeviceScreenSize | null) => void; + readonly setHingePreview: (angle: number | null) => void; + readonly setInteractionActive: (active: boolean, mode?: "touch" | "orbit") => void; + readonly rejectOrientation: () => void; + readonly frameUpdated: (panel: DuoPanelId, primary?: HTMLCanvasElement) => void; + readonly resize: (width: number, height: number, pixelRatio: number) => void; + readonly screenPoint: ( + x: number, + y: number, + captured?: boolean, + ) => { x: number; y: number } | null; + readonly orbit: (x: number, y: number) => void; + readonly beginHinge: (x: number, y: number) => boolean; + readonly resetPose: () => void; + readonly cancelInput: () => void; + readonly dispose: () => void; +} + +/** On-demand renderer for the articulated body. One inner framebuffer spans both leaves; HID belongs to the stream. */ +export function createDuoViewer(options: { + canvas: HTMLCanvasElement; + sources: Record; + model: DeviceModelSource; + onUnavailable: () => void; + onModelError?: (cause: unknown) => void; + onPanelRequested?: (panel: DuoPanelId) => void; + onOrientationRequested?: (orientation: DeviceScreenSize["orientation"]) => void; +}): DuoViewer { + const surfaces = ([1, 3] as const) + .map((id) => { + const canvas = document.createElement("canvas"); + canvas.width = id === 1 ? 784 : 1600; + canvas.height = id === 1 ? 1140 : 1125; + const context = canvas.getContext("2d", { alpha: false }); + if (!context) throw new Error("Duo display canvas is unavailable"); + context.fillStyle = "#080a10"; + context.fillRect(0, 0, canvas.width, canvas.height); + return { id, canvas, context }; + }) + .map(({ id, canvas, context }) => { + const texture = new CanvasTexture(canvas); + texture.colorSpace = SRGBColorSpace; + texture.minFilter = texture.magFilter = LinearFilter; + texture.generateMipmaps = false; + return { id, canvas, context, texture }; + }); + const renderer = new WebGLRenderer({ + canvas: options.canvas, + alpha: true, + antialias: true, + powerPreference: "low-power", + }); + renderer.outputColorSpace = SRGBColorSpace; + const scene = new Scene(); + const environment = (() => { + const generator = new PMREMGenerator(renderer); + const room = new RoomEnvironment(); + try { + return generator.fromScene(room, 0.04); + } catch (cause) { + for (const surface of surfaces) surface.texture.dispose(); + renderer.dispose(); + renderer.forceContextLoss(); + throw cause; + } finally { + room.dispose(); + generator.dispose(); + } + })(); + scene.environment = environment.texture; + const camera = new PerspectiveCamera(36, 1, 0.1, 150); + const key = new DirectionalLight(0xffffff, 4); + key.position.set(-6, 10, 14); + const fill = new DirectionalLight(0xc7dcff, 2); + fill.position.set(8, -3, -5); + scene.add(new AmbientLight(0xffffff, 2.4), key, fill); + let model: ReturnType | null = null; + let screen: DeviceScreenSize | null = null; + let readyKey = ""; + let primaryKey = ""; + let activationAt = 0; + let disposed = false; + let viewport = { width: 0, height: 0, ratio: 1 }; + let buffer = { width: 0, height: 0, ratio: 0 }; + let restFace: DuoRestFace = "inside"; + let requestedOrientation: DeviceScreenSize["orientation"] | null = null; + let viewOrientation: DeviceScreenSize["orientation"] | null = null; + const pivot = new Vector3(); + const targetPivot = new Vector3(); + let requestedPanel: DuoPanelId | null = null; + let handoffTimer: ReturnType | null = null; + const clearHandoff = () => { + if (handoffTimer) clearTimeout(handoffTimer); + handoffTimer = null; + requestedPanel = null; + }; + const activeSnaps = () => + model + ? duoViewSnaps( + model.restFrames(screen?.screenId === 1 ? 1 : 3), + screen?.screenId === 1 ? 1 : 3, + ) + : []; + const snaps = () => { + if (!model || !screen?.supportsPhysicalOrientation || !options.onPanelRequested) + return activeSnaps(); + const cover = duoViewSnaps(model.restFrames(1), 1); + // A shut inner display is occluded and cannot become a useful rest view. + return angle > 20 && angle < 180 + ? [...duoViewSnaps(model.restFrames(3), 3), ...cover] + : activeSnaps(); + }; + const orbit = createDeviceMotion({ + choose(rotation) { + const snap = nearestDuoView(rotation, snaps()); + if (!snap) return rotation; + restFace = snap.face; + const panel = snap.face === "cover" ? 1 : 3; + if (screen && panel !== (requestedPanel ?? screen.screenId) && options.onPanelRequested) { + clearHandoff(); + requestedPanel = panel; + model?.cancelInput(); + options.onPanelRequested(panel); + handoffTimer = setTimeout(() => { + clearHandoff(); + const confirmed = nearestDuoView(orbit.rotation, activeSnaps()); + if (confirmed) { + restFace = confirmed.face; + orbit.setPose(confirmed.rotation, performance.now()); + } + scheduler.invalidate(); + }, 5000); + } else if ( + screen && + requestedPanel === null && + snap.orientation !== viewOrientation && + options.onOrientationRequested + ) { + viewOrientation = snap.orientation; + requestedOrientation = snap.orientation; + options.onOrientationRequested(snap.orientation); + } + return snap.rotation; + }, + }); + let angle = 180; + let targetAngle = 180; + let previewAngle: number | null = null; + let hingeLeaf: DuoHingeLeaf | null = null; + let appliedAngle = Number.NaN; + let interactionActive = false; + const framing = createDeviceFraming(); + const framingBounds = new Box3(); + let firstPose = true; + let physicalPose: DuoPose = "open"; + let presentationAngle = 180; + let targetPresentation = new Quaternion(); + let lastTime = 0; + const reduced = + typeof matchMedia === "function" && matchMedia("(prefers-reduced-motion: reduce)").matches; + const applyPose = () => { + if (!model) return; + const before = hingeLeaf && appliedAngle !== angle ? model.leafRotation(hingeLeaf) : null; + model.setAngle(angle); + appliedAngle = angle; + if (before && hingeLeaf) { + // The primary surface stays in camera space; its partner supplies the fold. + // Rebase both the displayed rotation and its rest target so releasing a + // pinch cannot resume the pre-fold body rotation. + const correction = before.multiply(model.leafRotation(hingeLeaf).invert()); + if (correction.angleTo(new Quaternion()) > 1e-8) + orbit.setPose(orbit.rotation.clone().multiply(correction), performance.now(), true); + } + model.root.quaternion.copy(orbit.rotation); + model.root.position.set(0, 0, 0); + const frames = model.restFrames(restFace === "cover" ? 1 : 3); + const frame = frames.find((frame) => frame.face === restFace) ?? frames[0]; + targetPivot.copy(frame?.center ?? new Vector3()); + model.root.position.copy(pivot).applyQuaternion(model.root.quaternion).negate(); + }; + const updateCamera = () => { + camera.position.set(framing.center.x, framing.center.y, framing.distance()); + camera.lookAt(framing.center.x, framing.center.y, 0); + camera.updateProjectionMatrix(); + }; + const fit = (immediate = false) => { + if (!model || !viewport.width || !viewport.height) return; + camera.aspect = viewport.width / viewport.height; + framing.setBounds( + framingBounds.setFromObject(model.root), + (camera.fov * Math.PI) / 360, + camera.aspect, + performance.now(), + immediate, + ); + updateCamera(); + }; + const moving = () => + Math.abs(angle - targetAngle) > 0.01 || pivot.distanceTo(targetPivot) > 0.001; + const scheduler = createRenderScheduler(() => { + if (disposed || !model || !viewport.width || !viewport.height) return; + try { + const now = performance.now(); + const elapsed = Math.max(0, (now - lastTime) / 1000); + const amount = reduced + ? 1 + : 1 - Math.exp(-14 * Math.max(0, (now - lastTime) / 1000 || 0.016)); + lastTime = now; + const inMotion = moving(); + const orbitChanged = orbit.advance(now, reduced); + if (inMotion) { + angle += (targetAngle - angle) * amount; + + if (Math.abs(angle - targetAngle) <= 0.01) angle = targetAngle; + applyPose(); + pivot.lerp(targetPivot, amount); + if (pivot.distanceTo(targetPivot) <= 0.001) pivot.copy(targetPivot); + applyPose(); + fit(reduced || elapsed > 0.5); + } + if (orbitChanged) { + applyPose(); + fit(reduced || elapsed > 0.5); + } + if (framing.advance(now, reduced)) updateCamera(); + if ( + buffer.width !== viewport.width || + buffer.height !== viewport.height || + buffer.ratio !== viewport.ratio + ) { + renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.ratio); + buffer = viewport; + } + renderer.render(scene, camera); + if (inMotion || moving() || orbit.needsFrame() || framing.needsFrame()) + scheduler.invalidate(); + } catch { + options.onUnavailable(); + } + }); + const slot = createDeviceModelSlot({ + load: loadDeviceModel, + onError(cause) { + options.onModelError?.(cause); + options.onUnavailable(); + }, + install(loaded) { + const next = loaded + ? createDuoScene(loaded.asset, { 1: surfaces[0]!.texture, 3: surfaces[1]!.texture }) + : null; + if (model) { + scene.remove(model.root); + model.dispose(); + } + model = next; + appliedAngle = Number.NaN; + if (!model || disposed) return; + scene.add(model.root); + applyPose(); + if (screen?.screenId === 1) { + const snap = nearestDuoView(orbit.rotation, activeSnaps()); + if (snap) { + restFace = snap.face; + orbit.setPose(snap.rotation, performance.now(), true); + applyPose(); + } + } + pivot.copy(targetPivot); + applyPose(); + fit(); + scheduler.invalidate(); + }, + }); + const lost = (event: Event) => { + event.preventDefault(); + options.onUnavailable(); + }; + options.canvas.addEventListener("webglcontextlost", lost); + slot.set(options.model); + return { + setScreen(next) { + if (disposed) return; + const wasMoving = moving() || orbit.needsFrame(); + if (duoDisplayKey(screen) !== duoDisplayKey(next)) { + readyKey = ""; + activationAt = performance.now(); + model?.cancelInput(); + } + const previous = screen; + const ownedHandoff = requestedPanel !== null; + if (!next || next.screenId === requestedPanel) clearHandoff(); + const ownedRotation = requestedOrientation !== null && next?.screenId === previous?.screenId; + const changedDisplay = next?.screenId !== previous?.screenId; + requestedOrientation = null; + screen = next; + const changedPose = next?.hingePose && next.hingePose !== previous?.hingePose; + const folding = hingeLeaf !== null && !changedPose; + const rotated = + next?.screenId === previous?.screenId && + next?.hingeAngle === previous?.hingeAngle && + next?.orientation !== previous?.orientation; + const leftPhysicalPose = + rotated && + !ownedRotation && + !ownedHandoff && + !folding && + !next?.hingePose && + previewAngle === null; + if (firstPose || changedPose || leftPhysicalPose) { + physicalPose = next?.hingePose ?? (next?.screenId === 1 ? "closed" : "open"); + presentationAngle = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180); + } + // A command reply/config confirms hinge state. Display identity, never an angle heuristic, owns input. + targetAngle = previewAngle ?? next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180); + // Native angle commands clear hingePose. They change articulation only; + // preserve the viewing pose, including laptop/tent and user orbit. A + // separate rotation clears the physical preset and follows panel orientation. + if (firstPose || changedPose || (rotated && !ownedRotation && !ownedHandoff && !folding)) { + hingeLeaf = null; + viewOrientation = next?.orientation ?? null; + const poseAngle = presentationAngle; + const fold = ((180 - poseAngle) * Math.PI) / 360; + let roll = physicalPose === "closed" ? 0 : Math.PI / 2; + if (next && next.width < next.height) { + if (next.orientation === "landscape_left") roll -= Math.PI / 2; + if (next.orientation === "landscape_right") roll += Math.PI / 2; + } + if (next?.orientation === "portrait_upside_down") roll -= Math.PI; + const physical = + physicalPose === "laptop" + ? new Euler(-fold + Math.PI / 9, -Math.PI / 9, Math.PI / 2, "YXZ") + : physicalPose === "tent" + ? new Euler(Math.PI / 2 + Math.PI / 18, -Math.PI / 9, -Math.PI / 2, "YXZ") + : new Euler(0, (Math.PI / 2) * Math.pow(1 - poseAngle / 180, 3), 0, "YXZ"); + targetPresentation = new Quaternion().setFromEuler(physical); + if (physicalPose !== "laptop" && physicalPose !== "tent") + targetPresentation.premultiply( + new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), roll), + ); + if (next?.screenId === 1) { + // Native readback can retain the cover even for an open physical preset. + // A screen-facing rest must use that display's actual hinged normal. + const snap = nearestDuoView(targetPresentation, activeSnaps()); + if (snap) targetPresentation.copy(snap.rotation); + } + if (firstPose) angle = targetAngle; + restFace = next?.screenId === 1 ? "cover" : physicalPose === "laptop" ? "right" : "inside"; + orbit.setPose(targetPresentation, performance.now(), firstPose); + firstPose = false; + } else if (next && changedDisplay && !ownedHandoff && !folding) { + // A sensor rotation can hand ownership to the other native display. + // Face that display without sending another rotation and creating a feedback loop. + const snap = nearestDuoView(orbit.rotation, activeSnaps()); + if (snap) { + restFace = snap.face; + orbit.setPose(snap.rotation, performance.now()); + } + } + if (!wasMoving) lastTime = performance.now(); + applyPose(); + fit(); + scheduler.invalidate(); + }, + setHingePreview(next) { + if (disposed || (next !== null && (!Number.isFinite(next) || next < 0 || next > 180))) return; + if (!moving()) lastTime = performance.now(); + previewAngle = next; + if (next !== null && !hingeLeaf && model) { + clearHandoff(); + requestedOrientation = null; + hingeLeaf = screen?.screenId === 1 && angle > 20 ? "left" : "right"; + orbit.setPose(orbit.rotation.clone(), performance.now(), true); + } + orbit.hold(interactionActive || next !== null, performance.now()); + targetAngle = next ?? screen?.hingeAngle ?? (screen?.screenId === 1 ? 0 : 180); + if (next !== null) { + angle = next; + applyPose(); + fit(); + } + model?.cancelInput(); + scheduler.invalidate(); + }, + setInteractionActive(active, mode = "touch") { + if (disposed) return; + if (mode === "orbit") { + if (active) hingeLeaf = null; + orbit.dragActive(active, performance.now()); + } else { + interactionActive = active; + orbit.hold(active || previewAngle !== null, performance.now()); + framing.hold(active, performance.now()); + } + scheduler.invalidate(); + }, + rejectOrientation() { + if (disposed || (requestedOrientation === null && requestedPanel === null)) return; + clearHandoff(); + requestedOrientation = null; + viewOrientation = screen?.orientation ?? null; + const confirmed = activeSnaps().filter((snap) => snap.orientation === screen?.orientation); + const snap = nearestDuoView(orbit.rotation, confirmed); + if (snap) { + restFace = snap.face; + orbit.setPose(snap.rotation, performance.now()); + } + scheduler.invalidate(); + }, + frameUpdated(id, primary) { + if (disposed || screen?.screenId !== id) return; + const key = duoDisplayKey(screen); + // The elected native feed is authoritative during handoff. Fixed-panel + // encoders can retain an inactive blank until that surface changes again. + if (!primary && primaryKey === key) return; + const source = primary ?? options.sources[id]; + if (!duoFrameMatches(source, screen)) return; + const surface = surfaces[id === 1 ? 0 : 1]!; + const { context, canvas } = surface; + // Ignore native shutdown blanks only while waiting for an activation. Steady black application content remains valid. + if (!primary && !readyKey && performance.now() - activationAt < 1500) { + const probe = document.createElement("canvas"); + probe.width = probe.height = 8; + const probeContext = probe.getContext("2d", { willReadFrequently: true }); + if (probeContext) { + probeContext.drawImage(source, 0, 0, 8, 8); + if ( + !probeContext + .getImageData(0, 0, 8, 8) + .data.some((value, index) => index % 4 !== 3 && value > 3) + ) + return; + } + } + context.save(); + context.fillStyle = "#080a10"; + context.fillRect(0, 0, canvas.width, canvas.height); + context.translate(canvas.width / 2, canvas.height / 2); + // The inner panel is mounted a quarter turn from the native framebuffer. + const rotate = id === 3; + if (rotate) context.rotate(Math.PI / 2); + const scale = Math.min( + canvas.width / (rotate ? source.height : source.width), + canvas.height / (rotate ? source.width : source.height), + ); + context.drawImage( + source, + (-source.width * scale) / 2, + (-source.height * scale) / 2, + source.width * scale, + source.height * scale, + ); + context.restore(); + surface.texture.needsUpdate = true; + readyKey = duoDisplayKey(screen); + if (primary) primaryKey = readyKey; + scheduler.invalidate(); + }, + resize(width, height, ratio) { + if (disposed || ![width, height, ratio].every(Number.isFinite) || width <= 0 || height <= 0) + return; + viewport = { width, height, ratio: Math.min(2, Math.max(1, ratio)) }; + fit(true); + scheduler.invalidate(); + }, + screenPoint(x, y, captured = false) { + if ( + disposed || + moving() || + previewAngle !== null || + requestedOrientation !== null || + requestedPanel !== null + ) + return null; + applyPose(); + return model?.screenPoint(x, y, camera, screen, readyKey, captured) ?? null; + }, + cancelInput() { + hingeLeaf = null; + model?.cancelInput(); + }, + orbit(x, y) { + if (disposed) return; + hingeLeaf = null; + orbit.orbit(x * viewport.width, y * viewport.height, performance.now()); + scheduler.invalidate(); + }, + beginHinge(x, y) { + if (disposed || !model || !screen) return false; + applyPose(); + const leaf = model.hingeLeafAt(x, y, camera); + if (!leaf) return false; + clearHandoff(); + requestedOrientation = null; + // The right inner leaf and the shut cover share the front-facing plane. + // Holding that leaf lets the cover replace the inner image without a + // half turn, even when the pinch starts over the moving partner. + hingeLeaf = screen.screenId === 1 && angle > 20 ? "left" : "right"; + orbit.setPose(orbit.rotation.clone(), performance.now(), true); + model.cancelInput(); + return true; + }, + resetPose() { + if (disposed) return; + hingeLeaf = null; + requestedOrientation = null; + clearHandoff(); + const snap = activeSnaps().find((snap) => snap.orientation === screen?.orientation); + if (snap && snap.orientation !== viewOrientation && options.onOrientationRequested) { + viewOrientation = snap.orientation; + requestedOrientation = snap.orientation; + options.onOrientationRequested(snap.orientation); + } + orbit.reset(snap?.rotation ?? targetPresentation, performance.now()); + applyPose(); + fit(); + scheduler.invalidate(); + }, + dispose() { + clearHandoff(); + if (disposed) return; + disposed = true; + scheduler.dispose(); + slot.dispose(); + options.canvas.removeEventListener("webglcontextlost", lost); + for (const surface of surfaces) surface.texture.dispose(); + scene.environment = null; + environment.dispose(); + renderer.dispose(); + renderer.forceContextLoss(); + }, + }; +} diff --git a/packages/client-runtime/src/device/model.test.ts b/packages/client-runtime/src/device/model.test.ts index 94db7942176b..7bd1aa231376 100644 --- a/packages/client-runtime/src/device/model.test.ts +++ b/packages/client-runtime/src/device/model.test.ts @@ -24,6 +24,7 @@ function fixture() { it("matches only exact supported hardware, including iPad size and generation", () => { expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id); expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id); + expect(resolveDeviceModelId("ios", "iPhone Duo")).toBe("iphone-duo"); expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5"); for (const name of [ "iPhone 17 Pro", diff --git a/packages/client-runtime/src/device/model.ts b/packages/client-runtime/src/device/model.ts index 1f6524d4277c..c1a66f33365e 100644 --- a/packages/client-runtime/src/device/model.ts +++ b/packages/client-runtime/src/device/model.ts @@ -1,6 +1,6 @@ import type { DevicePlatform } from "@t3tools/contracts"; -export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5"; +export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5" | "iphone-duo"; export interface DeviceModelSource { readonly id: DeviceModelId; @@ -18,6 +18,7 @@ export type DeviceAssetSource = DeviceModelSource | DeviceAccessorySource; /** Match actual hardware, never stretch an available model to impersonate another device. */ export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId | null { if (platform !== "ios") return null; + if (/^iPhone Duo$/i.test(name)) return "iphone-duo"; if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max"; if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro"; if (/^iPad Pro 13-inch \(M5\)$/i.test(name)) return "ipad-pro-13-m5"; diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts index cc849ea3e15e..6b1ff6158879 100644 --- a/packages/client-runtime/src/device/stream.ts +++ b/packages/client-runtime/src/device/stream.ts @@ -19,6 +19,14 @@ * The decoder only runs while frames arrive and the viewer is attached; a * hidden panel calls `stop()` so an idle device costs nothing on the GPU. */ +import { + createDuoControl, + type DuoCommand, + type DuoControlState, + type DuoPose, +} from "./duoControl.ts"; +import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts"; import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; import type { DevicePlatform } from "@t3tools/contracts"; @@ -29,9 +37,55 @@ export interface DeviceScreenSize { readonly width: number; readonly height: number; readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; + readonly screenId?: number; + readonly supportsHingeAngle?: boolean; + readonly supportsPhysicalOrientation?: boolean; + readonly hingeAngle?: number; + readonly hingePose?: DuoPose | null; + readonly tableMode?: boolean; + readonly tableModeAvailable?: boolean; +} + +const screenConfigSchema = Schema.Struct({ + width: Schema.Finite.check(Schema.isGreaterThan(0)), + height: Schema.Finite.check(Schema.isGreaterThan(0)), + orientation: Schema.Literals([ + "portrait", + "portrait_upside_down", + "landscape_left", + "landscape_right", + ]), + screenId: Schema.optionalKey(Schema.Number), + supportsHingeAngle: Schema.optionalKey(Schema.Boolean), + supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean), + hingeAngle: Schema.optionalKey( + Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })), + ), + hingePose: Schema.optionalKey( + Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])), + ), + tableMode: Schema.optionalKey(Schema.Boolean), + tableModeAvailable: Schema.optionalKey(Schema.Boolean), +}); +const controlReplySchema = Schema.Struct({ + requestId: Schema.Int, + ok: Schema.Boolean, + error: Schema.optionalKey(Schema.String), +}); +const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema); +const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema); + +export interface DuoPanelSinks { + readonly cover: DeviceFrameSink; + readonly inner: DeviceFrameSink; + /** Invalidate captured input synchronously, before React can commit the new layout. */ + readonly onScreen?: (screen: DeviceScreenSize) => void; } export interface DeviceStreamEvents { + readonly onDuoControl?: (state: DuoControlState) => void; + /** A fixed panel cannot be decoded; the owner should return to the active flat feed. */ + readonly onDuoUnavailable?: (detail?: string) => void; readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; readonly onScreen: (screen: DeviceScreenSize) => void; /** The proxy rejected the credential; the owner should refresh access and reconnect. */ @@ -52,6 +106,9 @@ export interface DeviceStreamTarget { readonly access: DeviceHubAccess; /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */ readonly preferMjpeg?: boolean; + /** Internal fixed-panel feeds share their parent's input session. */ + readonly panelId?: 1 | 3; + readonly videoOnly?: boolean; } export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; @@ -219,6 +276,11 @@ export interface DeviceStreamClient { readonly pressButton: (button: DeviceHardwareButton) => void; readonly rotate: () => void; readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void; + readonly controlDuo: (command: DuoCommand) => void; + /** Switch between one active feed and two fixed-panel feeds without replacing HID. */ + readonly setDuoPanels: (panels: DuoPanelSinks | null) => void; + /** Model UVs already map to the hardware framebuffer. */ + readonly sendRawTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; } const HID_USAGE_BY_CODE: Readonly> = { @@ -314,6 +376,37 @@ export function createDeviceStreamClient( let videoReadTimer: ReturnType | null = null; let mjpegImage: HTMLImageElement | null = null; let releaseImage: (() => void) | null = null; + let videoGeneration = 0; + let panelClients: DeviceStreamClient[] = []; + let panelSinks: DuoPanelSinks | null = null; + let rotationCursor: DeviceScreenSize["orientation"] | null = null; + let pendingOrientation: { requestId: number } | null = null; + const duoControl = createDuoControl({ + send(request) { + if (socket?.readyState !== WebSocket.OPEN || !screen?.supportsHingeAngle) return false; + if (request.command.control === "physical" && !screen.supportsPhysicalOrientation) + return false; + try { + pendingOrientation = null; + if (request.command.control === "orientation") { + const value = request.command.value; + pendingOrientation = { requestId: request.requestId }; + rotationCursor = value; + // Upstream serializes orientation with hinge commands, then broadcasts config. + // An orientation-locked app can keep its framebuffer orientation after the sensor rotates. + socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value })); + } else socket.send(taggedJson(0x10, request)); + return true; + } catch { + return false; + } + }, + onChange(state) { + if (!state.pending) pendingOrientation = null; + events.onDuoControl?.(state); + }, + }); + const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`; const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); @@ -437,16 +530,26 @@ export function createDeviceStreamClient( }; const makeDecoder = () => { + const feedGeneration = videoGeneration; const decoder = new VideoDecoder({ output: (frame) => { try { - if (videoDecoder === decoder) paint(frame, frame.displayWidth, frame.displayHeight); + if ( + videoDecoder === decoder && + (platform !== "ios" || feedGeneration === videoGeneration) + ) + paint(frame, frame.displayWidth, frame.displayHeight); } finally { frame.close(); } }, error: () => { - if (stopped || videoDecoder !== decoder) return; + if ( + stopped || + videoDecoder !== decoder || + (platform === "ios" && feedGeneration !== videoGeneration) + ) + return; recoverDecoder(); }, }); @@ -456,7 +559,7 @@ export function createDeviceStreamClient( /** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */ const configureDecoder = async ( config: VideoDecoderConfig, - isCurrent: () => boolean, + isCurrent = () => !stopped, ): Promise => { const epoch = decoderEpoch; const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; @@ -525,19 +628,32 @@ export function createDeviceStreamClient( // iOS video: fetch the AVCC body and demux into the decoder. const readIosVideo = async () => { + const lifecycle = generation; + const feedGeneration = ++videoGeneration; const demuxer = new AvccDemuxer(); - const session = generation; const videoController = new AbortController(); controller = videoController; - const isCurrent = () => !stopped && generation === session && controller === videoController; + const isCurrent = () => + !stopped && + generation === lifecycle && + videoGeneration === feedGeneration && + controller === videoController; let retryDetail: string | undefined; try { - const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { + const response = await fetch(httpUrl(videoPath), { signal: videoController.signal, credentials: access.credentials ? "include" : "same-origin", }); - if (!isCurrent()) return; + if (!isCurrent()) { + await response.body?.cancel(); + return; + } if (response.status === 401 || response.status === 403) return handleUnauthorized(); + if (target.panelId && [400, 404, 405, 410].includes(response.status)) { + await response.body?.cancel(); + setStatus("error", "This Device Hub does not provide fixed Duo display feeds."); + return; + } if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); const reader = response.body.getReader(); for (;;) { @@ -579,7 +695,12 @@ export function createDeviceStreamClient( if (!configured) { await reader.cancel().catch(() => {}); if (!isCurrent()) return; - fallBackToMjpeg(); + if (target.videoOnly) + setStatus( + "error", + `This browser cannot decode the Duo panel's ${avcCodecString(chunk.payload)} stream.`, + ); + else fallBackToMjpeg(); return; } break; @@ -630,6 +751,42 @@ export function createDeviceStreamClient( } }; + const startDuoVideo = (panels: DuoPanelSinks) => { + for (const panel of panelClients) panel.stop(); + // Physical handoff elects a native surface. Fixed-panel encoders can keep + // an inactive shutdown frame after election, so this build uses one active + // feed instead of decoding a third stream alongside the two fixed feeds. + const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const); + panelClients = ids.map((id) => { + const output = id === 1 ? panels.cover : panels.inner; + return createDeviceStreamClient( + { ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true }, + { + present(source, width, height) { + if (id === null) { + return sink.present(source, width, height); + } + // An inactive native LCD can emit its shutdown black frame. Retain its last useful image. + if (screen?.screenId !== id) return true; + const retained = output.present(source, width, height); + const primary = sink.present(source, width, height); + return retained && primary; + }, + }, + { + onStatus: (status, detail) => { + if (status === "error") events.onDuoUnavailable?.(detail); + }, + onScreen: () => {}, + onInputConnected: () => {}, + onMjpegFallback: () => {}, + onUnauthorized: handleUnauthorized, + }, + ); + }); + for (const panel of panelClients) panel.start(); + }; + // iOS input socket; also carries the screen config the helper pushes. const connectIosInput = async () => { if (stopped) return; @@ -648,12 +805,42 @@ export function createDeviceStreamClient( if (stopped || socket !== ws) return; if (!(event.data instanceof ArrayBuffer)) return; const bytes = new Uint8Array(event.data); - if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; + if (socket !== ws || stopped || bytes.length < 1) return; try { - const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; - if (config.width > 0 && config.height > 0) { - screen = config; - events.onScreen(config); + const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1))); + if (bytes[0] === 0x90) { + const reply = decodeControlReply(payload); + if (Option.isSome(reply)) duoControl.receive(reply.value); + } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) { + const config = decodeScreenConfig(payload); + if (Option.isSome(config)) { + const previous = screen; + screen = config.value; + if (screen.hingePose && screen.hingePose !== previous?.hingePose) + rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait"; + else if (screen.orientation !== previous?.orientation) + rotationCursor = screen.orientation; + panelSinks?.onScreen?.(screen); + events.onScreen(screen); + // A surface election can leave an existing decoder on the former + // encoder description. Reopen only video to acquire the elected + // surface's seed and codec configuration; HID and the viewer stay. + if ( + panelSinks && + screen.supportsPhysicalOrientation && + previous && + screen.screenId !== previous.screenId + ) + startDuoVideo(panelSinks); + if (pendingOrientation) { + const receipt = pendingOrientation; + pendingOrientation = null; + duoControl.receive({ + requestId: receipt.requestId, + ok: true, + }); + } + } } } catch { // Ignore malformed config frames. @@ -662,6 +849,8 @@ export function createDeviceStreamClient( ws.onclose = (event) => { if (socket !== ws) return; socket = null; + duoControl.clear(); + rotationCursor = null; if (stopped) return; events.onInputConnected( false, @@ -757,7 +946,7 @@ export function createDeviceStreamClient( configuring = false; connecting(); if (platform === "ios") { - void connectIosInput(); + if (!target.videoOnly) void connectIosInput(); if (useWebCodecs) void readIosVideo(); else fallBackToMjpeg(); } else if (useWebCodecs) { @@ -771,6 +960,12 @@ export function createDeviceStreamClient( if (stopped) return; stopped = true; generation++; + videoGeneration++; + duoControl.clear(); + rotationCursor = null; + for (const panel of panelClients) panel.stop(); + panelClients = []; + panelSinks = null; mjpeg = false; clearFrameTimer(); if (videoReadTimer !== null) clearTimeout(videoReadTimer); @@ -814,6 +1009,29 @@ export function createDeviceStreamClient( start, stop, setMjpegImage, + controlDuo: duoControl.enqueue, + sendRawTouch: (phase, x, y) => { + if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y })); + }, + setDuoPanels(panels) { + if (platform !== "ios" || target.videoOnly || stopped || panelSinks === panels) return; + if (panels && !screen?.supportsHingeAngle) return; + panelSinks = panels; + videoGeneration++; + controller?.abort(); + controller = null; + closeDecoder(); + const retry = retryTimers.get("video"); + if (retry) clearTimeout(retry); + retryTimers.delete("video"); + for (const panel of panelClients) panel.stop(); + panelClients = []; + if (!panels) { + if (useWebCodecs) void readIosVideo(); + return; + } + startDuoVideo(panels); + }, sendTouch: (phase, x, y) => { if (platform === "ios") { send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); @@ -856,10 +1074,15 @@ export function createDeviceStreamClient( }, rotate: () => { if (platform !== "ios") return; - const current = screen?.orientation ?? "portrait"; + const current = screen?.supportsHingeAngle + ? (rotationCursor ?? screen.orientation) + : (screen?.orientation ?? "portrait"); const next = IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; - send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); + if (screen?.supportsHingeAngle) { + rotationCursor = next; + duoControl.enqueue({ control: "orientation", value: next }); + } else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); }, setOrientation: (orientation) => { if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));