feat(web): add iPhone Duo 3D controls

apps/server/src/device/DeviceHubProxy.test.ts

@@ -139,3 +139,29 @@ it.each([
   expect(await response.text()).not.toContain("private credential diagnostic");
   expect(requests).toEqual([]);
 });
+
+it.each([1, 3])(
+  "forwards fixed Duo display %s through the authenticated read proxy",
+  async (panel) => {
+    const { handler, requests } = fixture([AuthOrchestrationReadScope]);
+    const route = `/vendor/serve-sim/helper/duo/panel/${panel}/stream.avcc`;
+    const response = await handler(
+      new Request(`http://t3.test/api/device-hub${route}?wsTicket=secret`),
+    );
+    expect(response.status).toBe(200);
+    await response.text();
+    expect(requests).toEqual([`http://hub.test${route}`]);
+  },
+);
+
+it.each(["/panel/2/stream.avcc", "/panel/1/webrtc/offer", "/panel/3/exec"])(
+  "rejects unsupported Duo route %s",
+  async (route) => {
+    const { handler, requests } = fixture([AuthOrchestrationReadScope]);
+    const response = await handler(
+      new Request(`http://t3.test/api/device-hub/vendor/serve-sim/helper/duo${route}`),
+    );
+    expect(response.status).toBe(404);
+    expect(requests).toEqual([]);
+  },
+);

apps/server/src/device/DeviceHubProxy.ts

@@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
   /^\/vendor\/serve-sim\/api\/screenshot$/,
   /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,
   /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
+  /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
   /^\/vendor\/serve-sim\/appstate$/,
   /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,
   /^\/vendor\/serve-emu\/health$/,

apps/server/src/device/DeviceToolchain.ts

@@ -26,7 +26,7 @@ import * as Semaphore from "effect/Semaphore";
 import * as ProcessRunner from "../processRunner.ts";
 
 const DEVICE_HUB_PACKAGE = "expo-device-hub";
-export const DEVICE_HUB_VERSION = "0.10.1";
+export const DEVICE_HUB_VERSION = "0.12.0";
 const AGENT_DEVICE_PACKAGE = "agent-device";
 export const AGENT_DEVICE_VERSION = "0.21.12";
 

apps/web/src/components/device/DeviceControlsRail.tsx

@@ -57,9 +57,9 @@ export function DeviceControlsRail(props: {
     <aside
       aria-label="Device controls"
       data-layout="rail"
-      className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center justify-center py-3 pr-2"
+      className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center gap-2 overflow-y-auto [justify-content:safe_center] py-3 pr-2 [scrollbar-width:none]"
     >
-      <div className="pointer-events-auto flex max-h-full shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
+      <div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
         <RailButton
           tooltipSide={popupSide}
           label="Home"
@@ -255,11 +255,12 @@ export function DeviceControlsRail(props: {
           </RailButton>
         ) : null}
         {view.phone ? (
-          <RailButton tooltipSide={popupSide} label="Reset phone view" onClick={view.resetView}>
+          <RailButton tooltipSide={popupSide} label="Restore 3D view" onClick={view.resetView}>
             <Maximize />
           </RailButton>
         ) : null}
       </div>
+      {view.foldingControls}
     </aside>
   );
 }

apps/web/src/components/device/DeviceDuoControls.tsx

@@ -0,0 +1,75 @@
+import {
+  DUO_POSES,
+  type DuoCommand,
+  type DuoControlState,
+} from "@t3tools/client-runtime/device/duo-control";
+import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream";
+import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
+import { Button } from "~/components/ui/button";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
+
+/** Physical presets live beside the device. Pinching supplies continuous hinge control. */
+export function DeviceDuoControls(props: {
+  screen: DeviceScreenSize;
+  state: DuoControlState;
+  enabled: boolean;
+  onCommand: (command: DuoCommand) => void;
+}) {
+  const angle = props.screen.hingeAngle;
+  const fold = angle == null ? null : angle === 0 ? "closed" : angle === 180 ? "open" : "book";
+  const selected = (id: (typeof DUO_POSES)[number]["id"]) =>
+    id === "laptop" || id === "tent" ? props.screen.hingePose === id : fold === id;
+  return (
+    <div aria-label="iPhone Duo stands" className="flex flex-col items-center gap-2">
+      {([DUO_POSES.slice(0, 3), DUO_POSES.slice(3)] as const).map((poses, index) => (
+        <div
+          key={poses[0]?.id}
+          role="group"
+          aria-label={index === 0 ? "Fold shape" : "Device stance"}
+          className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm"
+        >
+          {poses.map((pose) => (
+            <Tooltip key={pose.id}>
+              <TooltipTrigger
+                render={
+                  <Button
+                    size="icon"
+                    variant={selected(pose.id) ? "secondary" : "ghost"}
+                    disabled={!props.enabled}
+                    aria-label={`${pose.label} stand`}
+                    aria-pressed={selected(pose.id)}
+                    data-pressed={selected(pose.id) ? "" : undefined}
+                    onClick={() => props.onCommand({ control: "pose", value: pose.id })}
+                  />
+                }
+              >
+                <DeviceDuoGlyph pose={pose.id} />
+              </TooltipTrigger>
+              <TooltipPopup side="left">
+                {pose.label}
+                {pose.id === "book" ? " / bookshelf" : ""}
+              </TooltipPopup>
+            </Tooltip>
+          ))}
+        </div>
+      ))}
+      {props.state.error ? (
+        <Tooltip>
+          <TooltipTrigger
+            render={
+              <span
+                tabIndex={0}
+                role="alert"
+                aria-label={props.state.error}
+                className="pointer-events-auto text-xs text-destructive"
+              >
+                !
+              </span>
+            }
+          />
+          <TooltipPopup side="left">{props.state.error}</TooltipPopup>
+        </Tooltip>
+      ) : null}
+    </div>
+  );
+}

apps/web/src/components/device/DeviceDuoGlyph.tsx

@@ -0,0 +1,43 @@
+import type { DuoPose } from "@t3tools/client-runtime/device/duo-control";
+
+// Bitrig 0.25's toolbar glyphs. The fold outlines follow the native toolbar;
+// the rounded stance contours are ported from SimulatorFoldingPoseGlyph's SwiftUI paths.
+const stancePaths = {
+  laptop:
+    "M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z",
+  tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z",
+} as const;
+
+export function DeviceDuoGlyph({ pose }: { pose: DuoPose }) {
+  const stance = pose === "laptop" || pose === "tent";
+  return (
+    <svg
+      viewBox={stance ? "0 0 22 16" : "0 0 24 32"}
+      fill="none"
+      stroke="currentColor"
+      strokeWidth={stance ? 1.25 : 1.8}
+      strokeLinecap="round"
+      strokeLinejoin="round"
+      className="size-7 shrink-0"
+      aria-hidden
+    >
+      {stance ? <path d={stancePaths[pose]} /> : null}
+      {pose === "closed" ? (
+        <>
+          <rect x="2" y="9" width="20" height="13" rx="2.2" />
+          <path d="M19.3 12.5v6" />
+          <circle cx="5" cy="12" r=".5" fill="currentColor" stroke="none" />
+        </>
+      ) : null}
+      {pose === "book" ? (
+        <path d="M4.2 6.5h15.6c1 0 1.6.8 1.2 1.7L17.7 16l3.3 7.8c.4.9-.2 1.7-1.2 1.7H4.2c-1 0-1.6-.8-1.2-1.7L6.3 16 3 8.2c-.4-.9.2-1.7 1.2-1.7Z" />
+      ) : null}
+      {pose === "open" ? (
+        <>
+          <rect x="2" y="2" width="20" height="28" rx="2.4" />
+          <path d="M10 27h4" />
+        </>
+      ) : null}
+    </svg>
+  );
+}

apps/web/src/components/device/DeviceDuoViewport.tsx

@@ -0,0 +1,205 @@
+import { useEffect, useRef, type RefObject } from "react";
+import type { DuoViewer } from "@t3tools/client-runtime/device/duo-viewer";
+import type { DeviceModelSource } from "@t3tools/client-runtime/device/model";
+import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction";
+import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
+import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream";
+import { createDuoPinch } from "@t3tools/client-runtime/device/duo-control";
+import { bindPhoneTrackpad } from "./phoneTrackpad";
+
+const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer");
+
+/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */
+export function DeviceDuoViewport(props: {
+  readonly model: DeviceModelSource;
+  readonly source: RefObject<HTMLCanvasElement | null>;
+  readonly onFrameListener: (listener: (() => void) | null) => void;
+  readonly onResetReady: (reset: (() => void) | null) => void;
+  readonly onInputCancel: (cancel: (() => void) | null) => void;
+  readonly client: RefObject<DeviceStreamClient | null>;
+  readonly screen: DeviceScreenSize | null;
+  readonly hingePreview: number | null;
+  readonly controlError: string | null;
+  readonly onUnavailable: () => void;
+}) {
+  const hostRef = useRef<HTMLDivElement | null>(null);
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const viewerRef = useRef<DuoViewer | null>(null);
+  const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);
+  const pinchRef = useRef<ReturnType<typeof createDuoPinch> | null>(null);
+  const trackpadRef = useRef<ReturnType<typeof bindPhoneTrackpad> | null>(null);
+  const screenRef = useRef(props.screen);
+  const modelRef = useRef(props.model);
+  const previewRef = useRef(props.hingePreview);
+  const { source, client, onInputCancel, onResetReady, onUnavailable, onFrameListener } = props;
+
+  useEffect(() => {
+    screenRef.current = props.screen;
+    modelRef.current = props.model;
+    interactionRef.current?.end();
+    viewerRef.current?.setScreen(props.screen);
+  }, [props.screen, props.model]);
+
+  useEffect(() => {
+    if (props.controlError || !props.screen) {
+      trackpadRef.current?.cancel();
+      viewerRef.current?.rejectOrientation();
+    }
+  }, [props.controlError, props.screen]);
+
+  useEffect(() => {
+    previewRef.current = props.hingePreview;
+    if (!pinchRef.current?.active) viewerRef.current?.setHingePreview(props.hingePreview);
+  }, [props.hingePreview]);
+
+  useEffect(() => {
+    const host = hostRef.current;
+    const canvas = canvasRef.current;
+    const decoded = source.current;
+    if (!host || !canvas || !decoded) return;
+    let disposed = false;
+    let trackpad: ReturnType<typeof bindPhoneTrackpad> | null = null;
+    let stopTrackpadEnd: (() => void) | undefined;
+    const resize = () => {
+      const { width, height } = host.getBoundingClientRect();
+      viewerRef.current?.resize(width, height, window.devicePixelRatio);
+    };
+    const observer = new ResizeObserver(resize);
+    observer.observe(host);
+    const blur = () => {
+      interactionRef.current?.end();
+      trackpad?.cancel();
+      viewerRef.current?.cancelInput();
+    };
+    onInputCancel(blur);
+    window.addEventListener("blur", blur);
+    void loadDuoViewer()
+      .then(({ createDuoViewer }) => {
+        if (disposed) return;
+        const sources = {
+          1: document.createElement("canvas"),
+          3: document.createElement("canvas"),
+        };
+        const viewer = createDuoViewer({
+          canvas,
+          sources,
+          onUnavailable,
+          onPanelRequested: (panel) =>
+            client.current?.controlDuo({
+              control: "physical",
+              value: panel === 1 ? "facedown" : "faceup",
+            }),
+          onOrientationRequested: (value) =>
+            client.current?.controlDuo({ control: "orientation", value }),
+          onModelError: (cause) => console.warn("Device 3D asset could not load", cause),
+          model: modelRef.current,
+        });
+        viewerRef.current = viewer;
+        onResetReady(viewer.resetPose);
+        viewer.setScreen(screenRef.current);
+        const primaryFrame = () => {
+          const panel = screenRef.current?.screenId;
+          if (panel === 1 || panel === 3) viewer.frameUpdated(panel, decoded);
+        };
+        onFrameListener(primaryFrame);
+        primaryFrame();
+        viewer.setHingePreview(previewRef.current);
+        client.current?.setDuoPanels({
+          onScreen(next) {
+            screenRef.current = next;
+            interactionRef.current?.end();
+            viewer.setScreen(next);
+          },
+          cover: createCanvasFrameSink(sources[1], () => viewer.frameUpdated(1)),
+          inner: createCanvasFrameSink(sources[3], () => viewer.frameUpdated(3)),
+        });
+        interactionRef.current = createPhoneInteraction({
+          screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured),
+          touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y),
+          orbit: viewer.orbit,
+          zoomBy: () => {},
+          onInteractionActive: viewer.setInteractionActive,
+        });
+        const pinch = createDuoPinch({
+          angle: () =>
+            previewRef.current ??
+            screenRef.current?.hingeAngle ??
+            (screenRef.current?.screenId === 1 ? 0 : 180),
+          contains: (x, y) => !!screenRef.current && viewer.beginHinge(x, y),
+          change: (angle) => {
+            viewer.setHingePreview(angle ?? previewRef.current);
+            if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle });
+          },
+        });
+        pinchRef.current = pinch;
+        trackpad = bindPhoneTrackpad(canvas, interactionRef.current, {
+          begin(x, y) {
+            interactionRef.current?.end();
+            return pinch.begin(x, y);
+          },
+          move: pinch.move,
+          end: pinch.end,
+        });
+        trackpadRef.current = trackpad;
+        stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit());
+        resize();
+      })
+      .catch(() => {
+        if (!disposed) onUnavailable();
+      });
+    return () => {
+      disposed = true;
+      stopTrackpadEnd?.();
+      trackpad?.dispose();
+      trackpadRef.current = null;
+      pinchRef.current = null;
+      interactionRef.current?.end();
+      interactionRef.current = null;
+      onInputCancel(null);
+      onResetReady(null);
+      onFrameListener(null);
+      client.current?.setDuoPanels(null);
+      observer.disconnect();
+      window.removeEventListener("blur", blur);
+      viewerRef.current?.dispose();
+      viewerRef.current = null;
+    };
+  }, [onInputCancel, onResetReady, client, onUnavailable, source, onFrameListener]);
+
+  const point = (event: React.PointerEvent<HTMLCanvasElement>) => {
+    const rect = event.currentTarget.getBoundingClientRect();
+    return {
+      x: (event.clientX - rect.left) / rect.width,
+      y: (event.clientY - rect.top) / rect.height,
+    };
+  };
+  return (
+    <div className="absolute inset-0">
+      <div ref={hostRef} className="absolute inset-0">
+        <div
+          aria-hidden
+          className="pointer-events-none absolute bottom-[6%] left-1/2 h-5 w-2/5 -translate-x-1/2 rounded-full bg-foreground/10 blur-xl"
+        />
+        <canvas
+          ref={canvasRef}
+          aria-label="Interactive 3D iPhone Duo. Drag the screen to interact. Drag outside it or swipe with two fingers to turn. Pinch over the device to open or close its hinge."
+          className="size-full touch-none"
+          onPointerDown={(event) => {
+            if (event.button !== 0) return;
+            if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return;
+            event.preventDefault();
+            event.currentTarget.setPointerCapture(event.pointerId);
+            (event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus();
+          }}
+          onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))}
+          onPointerUp={(event) => {
+            interactionRef.current?.move(event.pointerId, point(event));
+            interactionRef.current?.end(event.pointerId);
+          }}
+          onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)}
+          onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)}
+        />
+      </div>
+    </div>
+  );
+}