From 8932e4293def7243496077ffd7a13b70259a0854 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 20 Sep 2026 17:09:45 -0700 Subject: [PATCH 01/17] feat(web): prototype iPhone Duo on a pinned simulator build --- apps/server/src/device/DeviceHubProxy.test.ts | 26 ++ apps/server/src/device/DeviceHubProxy.ts | 1 + .../server/src/device/DeviceToolchain.test.ts | 74 +++- apps/server/src/device/DeviceToolchain.ts | 72 +++- .../components/device/DeviceDuoControls.tsx | 79 ++++ .../components/device/DeviceDuoViewport.tsx | 150 ++++++++ .../components/device/DeviceStreamView.tsx | 70 +++- .../web/src/components/device/deviceModels.ts | 2 + .../components/device/models/iphone-duo.glb | Bin 0 -> 3097724 bytes .../src/components/device/models/sources.json | 10 +- packages/client-runtime/package.json | 8 + .../src/device/duoControl.test.ts | 78 ++++ .../client-runtime/src/device/duoControl.ts | 75 ++++ .../src/device/duoScene.test.ts | 88 +++++ .../client-runtime/src/device/duoScene.ts | 166 +++++++++ .../src/device/duoStream.test.ts | 208 +++++++++++ .../src/device/duoViewer.test.ts | 207 +++++++++++ .../client-runtime/src/device/duoViewer.ts | 338 ++++++++++++++++++ .../client-runtime/src/device/model.test.ts | 1 + packages/client-runtime/src/device/model.ts | 3 +- packages/client-runtime/src/device/stream.ts | 251 ++++++++++++- scripts/build-duo-device-hub.ts | 73 ++++ 22 files changed, 1930 insertions(+), 50 deletions(-) create mode 100644 apps/web/src/components/device/DeviceDuoControls.tsx create mode 100644 apps/web/src/components/device/DeviceDuoViewport.tsx create mode 100644 apps/web/src/components/device/models/iphone-duo.glb create mode 100644 packages/client-runtime/src/device/duoControl.test.ts create mode 100644 packages/client-runtime/src/device/duoControl.ts create mode 100644 packages/client-runtime/src/device/duoScene.test.ts create mode 100644 packages/client-runtime/src/device/duoScene.ts create mode 100644 packages/client-runtime/src/device/duoStream.test.ts create mode 100644 packages/client-runtime/src/device/duoViewer.test.ts create mode 100644 packages/client-runtime/src/device/duoViewer.ts create mode 100644 scripts/build-duo-device-hub.ts diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts index 0f039274e207..73884bb6f9e5 100644 --- a/apps/server/src/device/DeviceHubProxy.test.ts +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -139,3 +139,29 @@ it.each([ expect(await response.text()).not.toContain("private credential diagnostic"); expect(requests).toEqual([]); }); + +it.each([1, 3])( + "forwards fixed Duo display %s through the authenticated read proxy", + async (panel) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const route = `/vendor/serve-sim/helper/duo/panel/${panel}/stream.avcc`; + const response = await handler( + new Request(`http://t3.test/api/device-hub${route}?wsTicket=secret`), + ); + expect(response.status).toBe(200); + await response.text(); + expect(requests).toEqual([`http://hub.test${route}`]); + }, +); + +it.each(["/panel/2/stream.avcc", "/panel/1/webrtc/offer", "/panel/3/exec"])( + "rejects unsupported Duo route %s", + async (route) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request(`http://t3.test/api/device-hub/vendor/serve-sim/helper/duo${route}`), + ); + expect(response.status).toBe(404); + expect(requests).toEqual([]); + }, +); diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index dd048480b3dd..0fb49c68e281 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray = [ /^\/vendor\/serve-sim\/api\/screenshot$/, /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/, /^\/vendor\/serve-sim\/appstate$/, /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/, /^\/vendor\/serve-emu\/health$/, diff --git a/apps/server/src/device/DeviceToolchain.test.ts b/apps/server/src/device/DeviceToolchain.test.ts index d7812a92b48e..cf3981af352e 100644 --- a/apps/server/src/device/DeviceToolchain.test.ts +++ b/apps/server/src/device/DeviceToolchain.test.ts @@ -4,6 +4,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import * as ProcessRunner from "../processRunner.ts"; @@ -13,6 +14,7 @@ import { ensureDeviceHub, isDeviceHubInstalled, } from "./DeviceToolchain.ts"; +const encodeManifest = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); it.effect("failed installation cleans staging and exposes only a safe failure message", () => Effect.gen(function* () { @@ -43,7 +45,6 @@ it.effect("failed installation cleans staging and exposes only a safe failure me expect(yield* fs.readDirectory(path.join(baseDir, "tools", "expo-device-hub"))).toEqual([]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("inventory reports only completed versions without installing the required version", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -97,3 +98,74 @@ it.effect("unreadable inventory stays unknown instead of reporting no installs", Effect.provide(NodeServices.layer), ), ); +it.effect( + "a Duo archive installs separately from the released hub and verifies its source revision before publishing", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-duo-install-" }); + const archive = path.join(baseDir, "prototype.tgz"); + const previous = process.env.T3CODE_DEVICE_HUB_ARCHIVE; + process.env.T3CODE_DEVICE_HUB_ARCHIVE = archive; + const { DEVICE_HUB_DUO_VERSION, DEVICE_HUB_DUO_COMMIT, DEVICE_HUB_DUO_PATCH } = + yield* Effect.promise(() => import("./DeviceToolchain.ts")); + const installs: string[][] = []; + let commit = "wrong-revision"; + const run: ProcessRunner.ProcessRunner["Service"]["run"] = (request) => + Effect.gen(function* () { + const args = [...(request.args ?? [])]; + installs.push(args); + const staging = args[args.indexOf("--prefix") + 1]!; + const pkg = path.join(staging, "node_modules", "expo-device-hub"); + yield* fs.makeDirectory(path.join(pkg, "dist", "server"), { recursive: true }); + yield* fs.writeFileString(path.join(pkg, "dist", "server", "cli.mjs"), "// entry"); + yield* fs.writeFileString( + path.join(pkg, "package.json"), + yield* encodeManifest({ + version: DEVICE_HUB_DUO_VERSION, + t3DeviceHubBuild: { + serveSimCommit: commit, + physicalOrientationPatchSha256: DEVICE_HUB_DUO_PATCH, + }, + }), + ); + return { + code: ChildProcessSpawner.ExitCode(0), + stdout: "", + stderr: "", + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + }).pipe(Effect.orDie); + yield* Effect.gen(function* () { + const failed = yield* ensureDeviceHub(baseDir).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { run }), + Effect.flip, + ); + expect(failed.step).toBe("verifying the pinned Duo build"); + expect(yield* isDeviceHubInstalled(baseDir)).toBe(false); + commit = DEVICE_HUB_DUO_COMMIT; + const installed = yield* ensureDeviceHub(baseDir).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { run }), + ); + expect(installed.installDir).toContain(DEVICE_HUB_DUO_VERSION); + expect(yield* isDeviceHubInstalled(baseDir)).toBe(true); + expect(installs.every((args) => args.at(-1) === archive)).toBe(true); + yield* ensureDeviceHub(baseDir).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { run }), + ); + expect(installs).toHaveLength(2); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + if (previous === undefined) delete process.env.T3CODE_DEVICE_HUB_ARCHIVE; + else process.env.T3CODE_DEVICE_HUB_ARCHIVE = previous; + }), + ), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts index e8e7d5cae46a..a5637f28131b 100644 --- a/apps/server/src/device/DeviceToolchain.ts +++ b/apps/server/src/device/DeviceToolchain.ts @@ -27,6 +27,11 @@ import * as ProcessRunner from "../processRunner.ts"; const DEVICE_HUB_PACKAGE = "expo-device-hub"; export const DEVICE_HUB_VERSION = "0.10.1"; +const DEVICE_HUB_DUO_BASE_VERSION = "0.9.0"; +export const DEVICE_HUB_DUO_COMMIT = "bb265b11c13b395e5302d121458e2d42224a2e9f"; +export const DEVICE_HUB_DUO_PATCH = + "ed67a0803952b7554dae0466125c915850013d9042470d87e3d25cbb9827f01a"; +export const DEVICE_HUB_DUO_VERSION = `${DEVICE_HUB_DUO_BASE_VERSION}-duo.${DEVICE_HUB_DUO_COMMIT.slice(0, 12)}.physical.${DEVICE_HUB_DUO_PATCH.slice(0, 8)}`; const AGENT_DEVICE_PACKAGE = "agent-device"; export const AGENT_DEVICE_VERSION = "0.21.12"; @@ -64,12 +69,17 @@ interface ToolSpec { readonly name: string; readonly version: string; readonly entry: ReadonlyArray; + readonly archive?: string; } -const HUB_SPEC: ToolSpec = { - name: DEVICE_HUB_PACKAGE, - version: DEVICE_HUB_VERSION, - entry: ["dist", "server", "cli.mjs"], +const hubSpec = (): ToolSpec => { + const archive = process.env.T3CODE_DEVICE_HUB_ARCHIVE; + return { + name: DEVICE_HUB_PACKAGE, + version: archive ? DEVICE_HUB_DUO_VERSION : DEVICE_HUB_VERSION, + ...(archive ? { archive } : {}), + entry: ["dist", "server", "cli.mjs"], + }; }; const AGENT_DEVICE_SPEC: ToolSpec = { @@ -87,8 +97,12 @@ const toolPaths = (path: Path.Path, baseDir: string, spec: ToolSpec): DeviceTool }; }; -const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchainPaths => ({ - hub: toolPaths(path, baseDir, HUB_SPEC), +const deviceToolchainPaths = ( + path: Path.Path, + baseDir: string, + hub = hubSpec(), +): DeviceToolchainPaths => ({ + hub: toolPaths(path, baseDir, hub), agentDevice: toolPaths(path, baseDir, AGENT_DEVICE_SPEC), }); @@ -118,6 +132,10 @@ const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( const fail = (step: string) => (cause: unknown) => new DeviceToolchainInstallError({ tool: spec.name, step, cause }); + if (spec.archive && !path.isAbsolute(spec.archive)) + return yield* fail("validating the Duo archive path")( + new Error("An absolute archive path is required"), + ); if (yield* isInstalled(fs, paths, spec.version)) return paths; const parentDir = path.dirname(paths.installDir); @@ -138,7 +156,7 @@ const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( stagingDir, "--no-fund", "--no-audit", - `${spec.name}@${spec.version}`, + spec.archive ?? `${spec.name}@${spec.version}`, ]; const result = yield* runner .run({ command: "npm", args: installArgs, timeout: INSTALL_TIMEOUT }) @@ -171,6 +189,28 @@ const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( step: "verifying the installed entry point", }); } + if (spec.archive) { + const manifest = yield* fs + .readFileString(path.join(stagingDir, "node_modules", spec.name, "package.json")) + .pipe( + Effect.flatMap( + Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ + version: Schema.Literal(DEVICE_HUB_DUO_VERSION), + t3DeviceHubBuild: Schema.Struct({ + serveSimCommit: Schema.Literal(DEVICE_HUB_DUO_COMMIT), + physicalOrientationPatchSha256: Schema.Literal(DEVICE_HUB_DUO_PATCH), + }), + }), + ), + ), + ), + Effect.mapError(fail("verifying the pinned Duo build")), + ); + if (manifest.version !== spec.version) + return yield* fail("verifying the pinned Duo build")(manifest); + } yield* fs .writeFileString(path.join(stagingDir, ".install-complete"), `${spec.version}\n`) .pipe(Effect.mapError(fail("recording the completed install"))); @@ -196,12 +236,16 @@ const ensureTool = Effect.fn("DeviceToolchain.ensureTool")(function* ( select: (paths: DeviceToolchainPaths) => DeviceToolPaths, ) { const path = yield* Path.Path; - const paths = deviceToolchainPaths(path, baseDir); + const paths = deviceToolchainPaths( + path, + baseDir, + spec.name === DEVICE_HUB_PACKAGE ? spec : hubSpec(), + ); return yield* installLock.withPermit(installTool(spec, select(paths))); }); export const ensureDeviceHub = (baseDir: string) => - ensureTool(baseDir, HUB_SPEC, (paths) => paths.hub); + ensureTool(baseDir, hubSpec(), (paths) => paths.hub); export const ensureAgentDevice = (baseDir: string) => ensureTool(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); @@ -213,12 +257,16 @@ const isToolInstalled = Effect.fn("DeviceToolchain.isToolInstalled")(function* ( ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const paths = deviceToolchainPaths(path, baseDir); + const paths = deviceToolchainPaths( + path, + baseDir, + spec.name === DEVICE_HUB_PACKAGE ? spec : hubSpec(), + ); return yield* isInstalled(fs, select(paths), spec.version); }); export const isDeviceHubInstalled = (baseDir: string) => - isToolInstalled(baseDir, HUB_SPEC, (paths) => paths.hub); + isToolInstalled(baseDir, hubSpec(), (paths) => paths.hub); export const isAgentDeviceInstalled = (baseDir: string) => isToolInstalled(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); @@ -260,7 +308,7 @@ export const deviceToolVersions = Effect.fn("DeviceToolchain.versions")(function }); return yield* Effect.gen(function* () { return { - hub: yield* inspect(HUB_SPEC), + hub: yield* inspect(hubSpec()), agent: yield* inspect(AGENT_DEVICE_SPEC), } satisfies DeviceToolVersions; }).pipe(Effect.orElseSucceed(() => undefined)); diff --git a/apps/web/src/components/device/DeviceDuoControls.tsx b/apps/web/src/components/device/DeviceDuoControls.tsx new file mode 100644 index 000000000000..88fc6f6c743b --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoControls.tsx @@ -0,0 +1,79 @@ +import { + DUO_POSES, + type DuoCommand, + type DuoControlState, +} from "@t3tools/client-runtime/device/duo-control"; +import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; +import { Button } from "~/components/ui/button"; + +export function DeviceDuoControls(props: { + screen: DeviceScreenSize; + state: DuoControlState; + enabled: boolean; + onCommand: (command: DuoCommand) => void; +}) { + const requested = props.state.requested; + const angle = + requested?.control === "angle" + ? requested.value + : requested?.control === "pose" + ? DUO_POSES.find((pose) => pose.id === requested.value)?.angle + : props.screen.hingeAngle; + return ( +
+
+ {DUO_POSES.map((pose) => ( + + ))} +
+
+ + props.onCommand({ control: "angle", value: Number(event.target.value) }) + } + /> + + {angle === undefined ? "?" : `${Math.round(angle)}°`} + + +
+ {props.state.error ? ( +

+ {props.state.error} +

+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceDuoViewport.tsx b/apps/web/src/components/device/DeviceDuoViewport.tsx new file mode 100644 index 000000000000..19f813c2b88e --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoViewport.tsx @@ -0,0 +1,150 @@ +import { useEffect, useRef, type RefObject } from "react"; +import type { DuoViewer } from "@t3tools/client-runtime/device/duo-viewer"; +import type { DeviceModelSource } from "@t3tools/client-runtime/device/model"; +import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction"; +import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; +import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream"; +import { bindPhoneTrackpad } from "./phoneTrackpad"; + +const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer"); + +/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */ +export function DeviceDuoViewport(props: { + readonly contentOffset: number; + readonly model: DeviceModelSource; + readonly source: RefObject; + readonly onResetReady: (reset: (() => void) | null) => void; + readonly onInputCancel: (cancel: (() => void) | null) => void; + readonly client: RefObject; + readonly screen: DeviceScreenSize | null; + readonly onUnavailable: () => void; + readonly onFramingAspect: (aspect: number) => void; +}) { + const hostRef = useRef(null); + const canvasRef = useRef(null); + const viewerRef = useRef(null); + const interactionRef = useRef | null>(null); + const screenRef = useRef(props.screen); + const modelRef = useRef(props.model); + const { source, client, onInputCancel, onResetReady, onUnavailable, onFramingAspect } = props; + + useEffect(() => { + screenRef.current = props.screen; + modelRef.current = props.model; + interactionRef.current?.end(); + viewerRef.current?.setScreen(props.screen); + }, [props.screen, props.model]); + + useEffect(() => { + const host = hostRef.current; + const canvas = canvasRef.current; + const decoded = source.current; + if (!host || !canvas || !decoded) return; + let disposed = false; + let trackpad: ReturnType | null = null; + const resize = () => { + const { width, height } = host.getBoundingClientRect(); + viewerRef.current?.resize(width, height, window.devicePixelRatio); + }; + const observer = new ResizeObserver(resize); + observer.observe(host); + const blur = () => { + interactionRef.current?.end(); + trackpad?.cancel(); + }; + onInputCancel(blur); + window.addEventListener("blur", blur); + void loadDuoViewer() + .then(({ createDuoViewer }) => { + if (disposed) return; + const sources = { + 1: document.createElement("canvas"), + 3: document.createElement("canvas"), + }; + const viewer = createDuoViewer({ + canvas, + sources, + onUnavailable, + onFramingAspect, + onModelError: (cause) => console.warn("Device 3D asset could not load", cause), + model: modelRef.current, + }); + viewerRef.current = viewer; + onResetReady(viewer.resetPose); + viewer.setScreen(screenRef.current); + client.current?.setDuoPanels({ + onScreen(next) { + interactionRef.current?.end(); + viewer.setScreen(next); + }, + cover: createCanvasFrameSink(sources[1], () => viewer.frameUpdated(1)), + inner: createCanvasFrameSink(sources[3], () => viewer.frameUpdated(3)), + }); + interactionRef.current = createPhoneInteraction({ + screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured), + touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y), + orbit: viewer.orbit, + zoomBy: viewer.zoomBy, + }); + trackpad = bindPhoneTrackpad(canvas, interactionRef.current); + resize(); + }) + .catch(() => { + if (!disposed) onUnavailable(); + }); + return () => { + disposed = true; + trackpad?.dispose(); + interactionRef.current?.end(); + interactionRef.current = null; + onInputCancel(null); + onResetReady(null); + client.current?.setDuoPanels(null); + observer.disconnect(); + window.removeEventListener("blur", blur); + viewerRef.current?.dispose(); + viewerRef.current = null; + }; + }, [onInputCancel, onResetReady, client, onUnavailable, source, onFramingAspect]); + + const point = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + return { + x: (event.clientX - rect.left) / rect.width, + y: (event.clientY - rect.top) / rect.height, + }; + }; + return ( +
+
+
+ { + if (event.button !== 0) return; + if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return; + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus(); + }} + onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))} + onPointerUp={(event) => { + interactionRef.current?.move(event.pointerId, point(event)); + interactionRef.current?.end(event.pointerId); + }} + onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)} + onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)} + /> +
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index f5457efd0024..1a64f4cf8166 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -8,6 +8,9 @@ import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile"; import { deviceKeyboard, deviceModel } from "./deviceModels"; import { fitDeviceFrame } from "./deviceFrameLayout"; +import { DeviceDuoViewport } from "./DeviceDuoViewport"; +import { DeviceDuoControls } from "./DeviceDuoControls"; +import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control"; import { DevicePhoneViewport } from "./DevicePhoneViewport"; import { DeviceLoadingView } from "./DeviceLoadingView"; import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi"; @@ -61,6 +64,13 @@ export function DeviceStreamView(props: { readonly onHandle?: (handle: DeviceStreamHandle | null) => void; readonly onScreen?: (screen: DeviceScreenSize | null) => void; }) { + const [duoControl, setDuoControl] = useState({ + pending: false, + requested: null, + error: null, + }); + const model = deviceModel(props.platform, props.deviceName ?? ""); + const isDuo = model?.id === "iphone-duo"; const [presentation, setPresentation] = useState<"phone" | "flat">("phone"); const [keyboardAttached, setKeyboardAttached] = useState(false); const [phoneUnavailable, setPhoneUnavailable] = useState(false); @@ -105,6 +115,8 @@ export function DeviceStreamView(props: { { platform: props.platform, deviceId: props.deviceId, access }, createCanvasFrameSink(canvas, () => frameListenerRef.current?.()), { + onDuoControl: setDuoControl, + onDuoUnavailable: onPhoneUnavailable, onStatus: (next, nextDetail) => { setStatus(next); setDetail(nextDetail); @@ -148,6 +160,7 @@ export function DeviceStreamView(props: { access, cancelPhoneInput, onHandle, + onPhoneUnavailable, onScreen, props.deviceId, props.environmentId, @@ -176,7 +189,8 @@ export function DeviceStreamView(props: { presentation === "phone" && !phoneUnavailable && !mjpegUrl && - !props.axOverlay; + !props.axOverlay && + (!isDuo || screen?.supportsHingeAngle === true); const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0; // The frame is the largest box at `aspect` that fits the container, so a @@ -275,13 +289,16 @@ export function DeviceStreamView(props: { return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; }; - const phoneUnavailableReason = phoneUnavailable - ? "3D is unavailable on this browser" - : mjpegUrl - ? "3D requires the H.264 stream" - : props.axOverlay - ? "Turn off accessibility frames to use 3D" - : null; + const phoneUnavailableReason = + isDuo && !screen?.supportsHingeAngle + ? "iPhone Duo 3D requires the pinned experimental Device Hub build" + : phoneUnavailable + ? "3D is unavailable on this browser" + : mjpegUrl + ? "3D requires the H.264 stream" + : props.axOverlay + ? "Turn off accessibility frames to use 3D" + : null; const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? ""); const resetView = useCallback(() => { @@ -413,7 +430,19 @@ export function DeviceStreamView(props: {
) : null} - {showPhone ? ( + {showPhone && isDuo && model ? ( + + ) : showPhone ? ( ) : null} + {props.allowPhoneView && isDuo && screen?.supportsHingeAngle && status === "streaming" ? ( + { + cancelPhoneInput(); + clientRef.current?.controlDuo(command); + }} + /> + ) : null} {props.allowPhoneView && !props.renderControls && status === "streaming" ? (
- ))} -
-
- - props.onCommand({ control: "angle", value: Number(event.target.value) }) - } - /> - + {([DUO_POSES.slice(0, 3), DUO_POSES.slice(3)] as const).map((poses, index) => ( +
- {angle === undefined ? "?" : `${Math.round(angle)}°`} - - -
+ {poses.map((pose) => ( + + props.onCommand({ control: "pose", value: pose.id })} + /> + } + > + + + + {pose.label} + {pose.id === "book" ? " / bookshelf" : ""} + + + ))} +
+ ))} {props.state.error ? ( -

- {props.state.error} -

+ + + ! + + } + /> + {props.state.error} + ) : null} ); diff --git a/apps/web/src/components/device/DeviceDuoGlyph.tsx b/apps/web/src/components/device/DeviceDuoGlyph.tsx new file mode 100644 index 000000000000..a179e3afe260 --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoGlyph.tsx @@ -0,0 +1,43 @@ +import type { DuoPose } from "@t3tools/client-runtime/device/duo-control"; + +// Bitrig 0.25's toolbar glyphs. The fold outlines follow the native toolbar; +// the rounded stance contours are ported from SimulatorFoldingPoseGlyph's SwiftUI paths. +const stancePaths = { + laptop: + "M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z", + tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z", +} as const; + +export function DeviceDuoGlyph({ pose }: { pose: DuoPose }) { + const stance = pose === "laptop" || pose === "tent"; + return ( + + {stance ? : null} + {pose === "closed" ? ( + <> + + + + + ) : null} + {pose === "book" ? ( + + ) : null} + {pose === "open" ? ( + <> + + + + ) : null} + + ); +} diff --git a/apps/web/src/components/device/DeviceDuoViewport.tsx b/apps/web/src/components/device/DeviceDuoViewport.tsx index e53b26f1a4f4..01556de533e5 100644 --- a/apps/web/src/components/device/DeviceDuoViewport.tsx +++ b/apps/web/src/components/device/DeviceDuoViewport.tsx @@ -4,15 +4,16 @@ import type { DeviceModelSource } from "@t3tools/client-runtime/device/model"; import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction"; import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream"; +import { createDuoPinch } from "@t3tools/client-runtime/device/duo-control"; import { bindPhoneTrackpad } from "./phoneTrackpad"; const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer"); /** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */ export function DeviceDuoViewport(props: { - readonly contentOffset: number; readonly model: DeviceModelSource; readonly source: RefObject; + readonly onFrameListener: (listener: (() => void) | null) => void; readonly onResetReady: (reset: (() => void) | null) => void; readonly onInputCancel: (cancel: (() => void) | null) => void; readonly client: RefObject; @@ -20,16 +21,17 @@ export function DeviceDuoViewport(props: { readonly hingePreview: number | null; readonly controlError: string | null; readonly onUnavailable: () => void; - readonly onFramingAspect: (aspect: number) => void; }) { const hostRef = useRef(null); const canvasRef = useRef(null); const viewerRef = useRef(null); const interactionRef = useRef | null>(null); + const pinchRef = useRef | null>(null); + const trackpadRef = useRef | null>(null); const screenRef = useRef(props.screen); const modelRef = useRef(props.model); const previewRef = useRef(props.hingePreview); - const { source, client, onInputCancel, onResetReady, onUnavailable, onFramingAspect } = props; + const { source, client, onInputCancel, onResetReady, onUnavailable, onFrameListener } = props; useEffect(() => { screenRef.current = props.screen; @@ -39,12 +41,15 @@ export function DeviceDuoViewport(props: { }, [props.screen, props.model]); useEffect(() => { - if (props.controlError) viewerRef.current?.rejectOrientation(); - }, [props.controlError]); + if (props.controlError || !props.screen) { + trackpadRef.current?.cancel(); + viewerRef.current?.rejectOrientation(); + } + }, [props.controlError, props.screen]); useEffect(() => { previewRef.current = props.hingePreview; - viewerRef.current?.setHingePreview(props.hingePreview); + if (!pinchRef.current?.active) viewerRef.current?.setHingePreview(props.hingePreview); }, [props.hingePreview]); useEffect(() => { @@ -77,7 +82,11 @@ export function DeviceDuoViewport(props: { canvas, sources, onUnavailable, - onFramingAspect, + onPanelRequested: (panel) => + client.current?.controlDuo({ + control: "physical", + value: panel === 1 ? "facedown" : "faceup", + }), onOrientationRequested: (value) => client.current?.controlDuo({ control: "orientation", value }), onModelError: (cause) => console.warn("Device 3D asset could not load", cause), @@ -86,9 +95,16 @@ export function DeviceDuoViewport(props: { viewerRef.current = viewer; onResetReady(viewer.resetPose); viewer.setScreen(screenRef.current); + const primaryFrame = () => { + const panel = screenRef.current?.screenId; + if (panel === 1 || panel === 3) viewer.frameUpdated(panel, decoded); + }; + onFrameListener(primaryFrame); + primaryFrame(); viewer.setHingePreview(previewRef.current); client.current?.setDuoPanels({ onScreen(next) { + screenRef.current = next; interactionRef.current?.end(); viewer.setScreen(next); }, @@ -99,10 +115,30 @@ export function DeviceDuoViewport(props: { screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured), touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y), orbit: viewer.orbit, - zoomBy: viewer.zoomBy, + zoomBy: () => {}, onInteractionActive: viewer.setInteractionActive, }); - trackpad = bindPhoneTrackpad(canvas, interactionRef.current); + const pinch = createDuoPinch({ + angle: () => + previewRef.current ?? + screenRef.current?.hingeAngle ?? + (screenRef.current?.screenId === 1 ? 0 : 180), + contains: (x, y) => !!screenRef.current && viewer.containsDevice(x, y), + change: (angle) => { + viewer.setHingePreview(angle ?? previewRef.current); + if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle }); + }, + }); + pinchRef.current = pinch; + trackpad = bindPhoneTrackpad(canvas, interactionRef.current, { + begin(x, y) { + interactionRef.current?.end(); + return pinch.begin(x, y); + }, + move: pinch.move, + end: pinch.end, + }); + trackpadRef.current = trackpad; resize(); }) .catch(() => { @@ -111,17 +147,20 @@ export function DeviceDuoViewport(props: { return () => { disposed = true; trackpad?.dispose(); + trackpadRef.current = null; + pinchRef.current = null; interactionRef.current?.end(); interactionRef.current = null; onInputCancel(null); onResetReady(null); + onFrameListener(null); client.current?.setDuoPanels(null); observer.disconnect(); window.removeEventListener("blur", blur); viewerRef.current?.dispose(); viewerRef.current = null; }; - }, [onInputCancel, onResetReady, client, onUnavailable, source, onFramingAspect]); + }, [onInputCancel, onResetReady, client, onUnavailable, source, onFrameListener]); const point = (event: React.PointerEvent) => { const rect = event.currentTarget.getBoundingClientRect(); @@ -132,18 +171,14 @@ export function DeviceDuoViewport(props: { }; return (
-
+
{ if (event.button !== 0) return; diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 23cf02fc2a18..a9063d43fd71 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -32,6 +32,7 @@ export interface DeviceViewControls { readonly showPhone: () => void; readonly showFlat: () => void; readonly resetView: () => void; + readonly foldingControls?: ReactNode; readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null; } @@ -326,6 +327,18 @@ export function DeviceStreamView(props: { phone: !!showPhone, streaming: status === "streaming", phoneUnavailableReason, + foldingControls: + showPhone && isDuo && screen?.supportsHingeAngle ? ( + { + cancelPhoneInput(); + clientRef.current?.controlDuo(command); + }} + /> + ) : null, keyboard: showPhone && keyboardSource ? { @@ -432,19 +445,18 @@ export function DeviceStreamView(props: {
{showPhone && isDuo && model ? ( ) : showPhone ? ( ) : null} - {props.allowPhoneView && isDuo && screen?.supportsHingeAngle && status === "streaming" ? ( - { - cancelPhoneInput(); - clientRef.current?.controlDuo(command); - }} - /> - ) : null} {props.allowPhoneView && !props.renderControls && status === "streaming" ? (