Default Studio expose to software H.264 and smooth streaming (#13)

* Update SimDeck agent instructions

* Update SimDeck agent instructions

* Smooth Studio provider startup polling

* Route remote realtime controls over WebRTC

* Reduce touch latency and align Safari H264

* Send touch edges on low latency RTC input

* Restore ordered touch control path

* Accept software codec compatibility alias

Author: DjDeveloperr

README.md

@@ -77,11 +77,14 @@ simdeck studio expose "iPhone 17 Pro"
 
 The command starts or reuses the local daemon, creates an ephemeral Studio
 session, prints a unique `https://simdeck.djdev.me/simulator/...` URL, and keeps
-the outbound bridge alive until you press Ctrl-C. It uses `auto` mode by default,
-letting VideoToolbox choose the encoder. Pass `--video-codec software` when you
-need to force software encoding; Studio then defaults to the `ci-software`
-stream quality profile (`960` longest edge at `24` fps). Use
-`--stream-quality quality|balanced|smooth|economy|ci-software` to override it.
+the outbound bridge alive until you press Ctrl-C. It uses software H.264 by
+default with realtime stream settings for remote viewing, and prints the active
+codec/profile when it starts. Studio defaults to the `smooth` stream quality
+profile (`1170` longest edge, dynamic up to `60` fps). Use
+`--stream-quality quality|balanced|smooth|economy|ci-software` to override it,
+or pass `--video-codec hardware` when a dedicated hardware encoder is preferable.
+The remote viewer renders live video with the browser's native video element;
+the canvas is only used for input geometry.
 
 CLI commands automatically use the same warm daemon:
 

cli/XCWH264Encoder.m

@@ -933,8 +933,8 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height
     }
     VTSessionSetProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(expectedFrameRate));
     BOOL shortKeyframeInterval = _lowLatencyMode || _realtimeStreamMode;
-    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(shortKeyframeInterval ? expectedFrameRate : expectedFrameRate * 2));
-    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(shortKeyframeInterval ? 1.0 : 2.0));
+    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(shortKeyframeInterval ? MAX(1, expectedFrameRate / 2) : expectedFrameRate * 2));
+    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(shortKeyframeInterval ? 0.5 : 2.0));
     VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate));
     if (_realtimeStreamMode) {
         NSArray *dataRateLimits = @[

client/src/api/controls.ts

@@ -11,7 +11,13 @@ import type {
 
 export type ControlMessage =
   | ({ type: "touch" } & TouchPayload)
-  | ({ type: "key" } & KeyPayload);
+  | ({ type: "key" } & KeyPayload)
+  | { type: "dismissKeyboard" }
+  | { type: "home" }
+  | { type: "appSwitcher" }
+  | { type: "rotateLeft" }
+  | { type: "rotateRight" }
+  | { type: "toggleAppearance" };
 
 async function postSimulatorAction(
   udid: string,

client/src/api/simulators.ts

@@ -9,8 +9,10 @@ import type {
   SimulatorsResponse,
 } from "./types";
 
-export async function listSimulators(): Promise<SimulatorMetadata[]> {
-  const data = await apiRequest<SimulatorsResponse>("/api/simulators");
+export async function listSimulators(
+  options: RequestInit = {},
+): Promise<SimulatorMetadata[]> {
+  const data = await apiRequest<SimulatorsResponse>("/api/simulators", options);
   return data.simulators ?? [];
 }
 

client/src/app/AppShell.tsx

@@ -111,6 +111,15 @@ function buildAuthenticatedAssetUrl(path: string, stamp: number): string {
   return url.toString();
 }
 
+function shouldUseRemoteStreamDefault(apiRoot: string): boolean {
+  if (apiRoot) {
+    return true;
+  }
+  return (
+    new URLSearchParams(window.location.search).get("remoteStream") === "1"
+  );
+}
+
 function shouldRenderNativeChrome(simulator: SimulatorMetadata): boolean {
   const identifier = simulator.deviceTypeIdentifier ?? "";
   const name = simulator.name ?? "";
@@ -177,13 +186,15 @@ export interface AppShellProps {
   fixedSimulatorUDID?: string | null;
   hideSimulatorSelection?: boolean;
   pairingEnabled?: boolean;
+  remoteStream?: boolean;
 }
 
 export function AppShell({
   apiRoot = "",
   fixedSimulatorUDID = null,
   hideSimulatorSelection = false,
   pairingEnabled = true,
+  remoteStream = shouldUseRemoteStreamDefault(apiRoot),
 }: AppShellProps = {}) {
   configureSimDeckClient({ apiRoot });
   const [initialUiState] = useState(readPersistedUiState);
@@ -201,7 +212,7 @@ export function AppShell({
     isLoading,
     refresh,
     simulators,
-  } = useSimulatorList();
+  } = useSimulatorList({ remote: remoteStream });
   const [debugVisible, setDebugVisible] = useState(false);
   const [hierarchyVisible, setHierarchyVisible] = useState(() =>
     readStoredFlag(HIERARCHY_VISIBLE_STORAGE_KEY),
@@ -370,6 +381,7 @@ export function AppShell({
     streamCanvasKey,
   } = useLiveStream({
     canvasElement: streamCanvasElement,
+    remote: remoteStream,
     simulator: selectedSimulator,
   });
 
@@ -843,9 +855,13 @@ export function AppShell({
     pairingEnabled &&
     listError === AUTH_REQUIRED_MESSAGE &&
     !accessTokenFromLocation();
+  const visibleListError =
+    remoteStream && hasFrame && listError === "Failed to fetch"
+      ? ""
+      : listError;
   const error = pairingRequired
     ? localError || streamError
-    : localError || streamError || listError;
+    : localError || streamError || visibleListError;
   const deviceTransform = `translate(${pan.x}px, ${pan.y + autoViewportOffsetY}px) scale(${effectiveZoom})`;
   const chromeScreenRect = computeChromeScreenRect(
     viewportChromeProfile,
@@ -979,18 +995,19 @@ export function AppShell({
     return state;
   }, []);
 
-  function sendControl(udid: string, message: ControlMessage) {
+  function sendControl(udid: string, message: ControlMessage): boolean {
     setLocalError("");
     const encoded = JSON.stringify(message);
     if (sendWebRtcControlMessage(encoded)) {
-      return;
+      return true;
     }
     const state = ensureControlSocket(udid);
     if (state.socket.readyState === WebSocket.OPEN) {
       state.socket.send(encoded);
     } else {
       state.pending.push(encoded);
     }
+    return true;
   }
 
   useEffect(() => closeControlSocket, [closeControlSocket]);
@@ -1281,28 +1298,47 @@ export function AppShell({
           if (!selectedSimulator) {
             return;
           }
-          void runAction(() => dismissKeyboard(selectedSimulator.udid), false);
+          if (
+            !sendControl(selectedSimulator.udid, { type: "dismissKeyboard" })
+          ) {
+            void runAction(
+              () => dismissKeyboard(selectedSimulator.udid),
+              false,
+            );
+          }
         }}
         onHome={() => {
           if (!selectedSimulator) {
             return;
           }
           setAccessibilitySelectedId("");
           setAccessibilityHoveredId(null);
-          void runAction(() => pressHome(selectedSimulator.udid), false);
+          if (!sendControl(selectedSimulator.udid, { type: "home" })) {
+            void runAction(() => pressHome(selectedSimulator.udid), false);
+          }
         }}
         onOpenAppSwitcher={() => {
           if (!selectedSimulator) {
             return;
           }
           setAccessibilitySelectedId("");
           setAccessibilityHoveredId(null);
-          void runAction(() => openAppSwitcher(selectedSimulator.udid), false);
+          if (!sendControl(selectedSimulator.udid, { type: "appSwitcher" })) {
+            void runAction(
+              () => openAppSwitcher(selectedSimulator.udid),
+              false,
+            );
+          }
         }}
         onRotateLeft={() => {
           if (!selectedSimulator) {
             return;
           }
+          if (sendControl(selectedSimulator.udid, { type: "rotateLeft" })) {
+            setRotationQuarterTurns((current) => (current + 3) % 4);
+            setStreamStamp(Date.now());
+            return;
+          }
           void runAction(async () => {
             await rotateLeft(selectedSimulator.udid);
             setRotationQuarterTurns((current) => (current + 3) % 4);
@@ -1315,6 +1351,11 @@ export function AppShell({
           if (!selectedSimulator) {
             return;
           }
+          if (sendControl(selectedSimulator.udid, { type: "rotateRight" })) {
+            setRotationQuarterTurns((current) => (current + 1) % 4);
+            setStreamStamp(Date.now());
+            return;
+          }
           void runAction(async () => {
             await rotateRight(selectedSimulator.udid);
             setRotationQuarterTurns((current) => (current + 1) % 4);
@@ -1339,7 +1380,10 @@ export function AppShell({
           if (!selectedSimulator) {
             return;
           }
-          void runAction(() => toggleAppearance(selectedSimulator.udid));
+          const encoded = JSON.stringify({ type: "toggleAppearance" });
+          if (!sendWebRtcControlMessage(encoded)) {
+            void runAction(() => toggleAppearance(selectedSimulator.udid));
+          }
         }}
         onToggleDebug={() => setDebugVisible((current) => !current)}
         onToggleHierarchy={() => {

client/src/features/simulators/useSimulatorList.ts

@@ -1,30 +1,64 @@
-import { startTransition, useEffect, useState } from "react";
+import { startTransition, useEffect, useRef, useState } from "react";
 
 import { listSimulators } from "../../api/simulators";
 import type { SimulatorMetadata } from "../../api/types";
 
-export function useSimulatorList() {
+const LOCAL_REFRESH_MS = 5000;
+const REMOTE_REFRESH_MS = 10000;
+const REMOTE_ERROR_REFRESH_MS = 15000;
+const REMOTE_REQUEST_TIMEOUT_MS = 12000;
+
+interface UseSimulatorListOptions {
+  remote?: boolean;
+}
+
+export function useSimulatorList({
+  remote = false,
+}: UseSimulatorListOptions = {}) {
   const [simulators, setSimulators] = useState<SimulatorMetadata[]>([]);
   const [isLoading, setIsLoading] = useState(true);
   const [error, setError] = useState("");
+  const inFlightRef = useRef(false);
+  const lastLoadFailedRef = useRef(false);
 
   async function loadSimulators(cancelled = false) {
+    if (inFlightRef.current) {
+      return;
+    }
+    inFlightRef.current = true;
+    const controller =
+      remote && typeof AbortController !== "undefined"
+        ? new AbortController()
+        : null;
+    const timeoutId = controller
+      ? window.setTimeout(() => controller.abort(), REMOTE_REQUEST_TIMEOUT_MS)
+      : 0;
     try {
-      const nextSimulators = await listSimulators();
+      const nextSimulators = await listSimulators(
+        controller ? { signal: controller.signal } : {},
+      );
       if (cancelled) {
         return;
       }
       startTransition(() => setSimulators(nextSimulators));
       setError("");
+      lastLoadFailedRef.current = false;
     } catch (loadError) {
       if (!cancelled) {
         setError(
-          loadError instanceof Error
-            ? loadError.message
-            : "Failed to load simulators.",
+          loadError instanceof DOMException && loadError.name === "AbortError"
+            ? "Timed out waiting for provider."
+            : loadError instanceof Error
+              ? loadError.message
+              : "Failed to load simulators.",
         );
+        lastLoadFailedRef.current = true;
       }
     } finally {
+      if (timeoutId) {
+        window.clearTimeout(timeoutId);
+      }
+      inFlightRef.current = false;
       if (!cancelled) {
         setIsLoading(false);
       }
@@ -37,17 +71,33 @@ export function useSimulatorList() {
 
   useEffect(() => {
     let cancelled = false;
+    let timeoutId = 0;
 
-    void loadSimulators();
-    const intervalId = window.setInterval(() => {
-      void loadSimulators(cancelled);
-    }, 5000);
+    const scheduleNext = () => {
+      if (cancelled) {
+        return;
+      }
+      const delay = remote
+        ? lastLoadFailedRef.current
+          ? REMOTE_ERROR_REFRESH_MS
+          : REMOTE_REFRESH_MS
+        : LOCAL_REFRESH_MS;
+      timeoutId = window.setTimeout(run, delay);
+    };
+
+    const run = () => {
+      void loadSimulators(cancelled).finally(scheduleNext);
+    };
+
+    run();
 
     return () => {
       cancelled = true;
-      clearInterval(intervalId);
+      if (timeoutId) {
+        window.clearTimeout(timeoutId);
+      }
     };
-  }, []);
+  }, [remote]);
 
   return {
     error,

client/src/features/stream/streamTypes.ts

@@ -1,6 +1,7 @@
 import type { Size } from "../viewport/types";
 
 export interface StreamConnectTarget {
+  remote?: boolean;
   udid: string;
 }