From 7f18d769bfd6f0bb41d4697d9168105b8179729d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 5 May 2026 22:44:00 -0400 Subject: [PATCH 1/5] Fix hardware stream downscaling lag --- cli/XCWH264Encoder.m | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index bf4eb97..9aa0323 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -477,6 +477,11 @@ @interface XCWH264Encoder () - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer targetWidth:(int32_t)targetWidth targetHeight:(int32_t)targetHeight; +- (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth + sourceHeight:(int32_t)sourceHeight + targetWidth:(int32_t)targetWidth + targetHeight:(int32_t)targetHeight + pixelFormat:(OSType)pixelFormat; - (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)targetWidth height:(int32_t)targetHeight pixelFormat:(OSType)pixelFormat; @@ -715,7 +720,11 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked { } - (uint64_t)maximumHardwareFrameIntervalUsLocked { - return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWLocalStreamMaximumFrameIntervalUs(); + if (_realtimeStreamMode) { + uint64_t minimumFpsIntervalUs = (uint64_t)llround(1000000.0 / (double)XCWMinimumLocalStreamFrameRate); + return MAX(XCWRealtimeMaximumFrameIntervalUs(), minimumFpsIntervalUs); + } + return XCWLocalStreamMaximumFrameIntervalUs(); } - (int32_t)expectedFrameRateLocked { @@ -807,7 +816,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs { - if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_lowLatencyMode || latencyUs == 0) { + if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || latencyUs == 0) { return; } if (_hardwareFrameIntervalUs == 0) { @@ -1077,11 +1086,24 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix targetHeight:(int32_t)targetHeight { int32_t sourceWidth = (int32_t)CVPixelBufferGetWidth(pixelBuffer); int32_t sourceHeight = (int32_t)CVPixelBufferGetHeight(pixelBuffer); + OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); BOOL shouldCopyStableRealtimeBuffer = _realtimeStreamMode || _lowLatencyMode; if (sourceWidth == targetWidth && sourceHeight == targetHeight && !shouldCopyStableRealtimeBuffer) { CVPixelBufferRetain(pixelBuffer); return pixelBuffer; } + if ([self shouldUseSoftwareScalerForSourceWidth:sourceWidth + sourceHeight:sourceHeight + targetWidth:targetWidth + targetHeight:targetHeight + pixelFormat:sourcePixelFormat]) { + CVPixelBufferRef scaledPixelBuffer = [self copySoftwareScaledPixelBuffer:pixelBuffer + targetWidth:targetWidth + targetHeight:targetHeight]; + if (scaledPixelBuffer != NULL) { + return scaledPixelBuffer; + } + } if (_pixelTransferSession == NULL) { OSStatus sessionStatus = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_pixelTransferSession); @@ -1105,7 +1127,6 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix kVTScalingMode_Normal); } - OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); CVPixelBufferRef scaledPixelBuffer = [self copyPixelBufferFromScalingPoolWithWidth:targetWidth height:targetHeight pixelFormat:sourcePixelFormat]; @@ -1130,6 +1151,20 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix return scaledPixelBuffer; } +- (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth + sourceHeight:(int32_t)sourceHeight + targetWidth:(int32_t)targetWidth + targetHeight:(int32_t)targetHeight + pixelFormat:(OSType)pixelFormat { + if (!_realtimeStreamMode || !XCWPixelFormatSupportsSoftwareScaling(pixelFormat)) { + return NO; + } + if (sourceWidth == targetWidth && sourceHeight == targetHeight) { + return NO; + } + return _encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware; +} + - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer targetWidth:(int32_t)targetWidth targetHeight:(int32_t)targetHeight { From fa310583591b7670bb05efc31d39d3becaf0d3d2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 5 May 2026 23:57:51 -0400 Subject: [PATCH 2/5] Add encoder overload detection --- cli/XCWH264Encoder.m | 96 +++++++++++++++++++ client/src/api/types.ts | 18 ++++ client/src/app/AppShell.tsx | 7 ++ .../src/features/simulators/SimulatorMenu.tsx | 8 +- client/src/features/stream/streamTypes.ts | 4 +- client/src/features/toolbar/DebugPanel.tsx | 37 +++++++ docs/api/health.md | 29 ++++++ docs/api/rest.md | 13 ++- docs/cli/flags.md | 2 +- docs/guide/daemon.md | 2 +- docs/guide/video.md | 10 +- server/src/api/routes.rs | 25 ++++- server/src/main.rs | 18 ++++ server/src/simulators/registry.rs | 14 +++ server/src/simulators/session.rs | 4 + 15 files changed, 273 insertions(+), 14 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 9aa0323..eb59be5 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -40,6 +40,10 @@ static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; static const NSUInteger XCWMaximumRealtimeInFlightFrames = 3; static const int32_t XCWRealtimeKeyFrameIntervalSeconds = 5; +static const double XCWEncoderLatencyEWMAAlpha = 0.2; +static const double XCWEncoderStrainedLoadPercent = 85.0; +static const double XCWEncoderOverloadedLoadPercent = 105.0; +static const NSUInteger XCWEncoderConsecutiveOverBudgetFrameThreshold = 3; typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeAuto, @@ -487,6 +491,7 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)ta pixelFormat:(OSType)pixelFormat; - (void)handleCompressionOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer submittedAtUs:(uint64_t)submittedAtUs; +- (uint64_t)activeFrameIntervalUsLocked; @end @@ -520,6 +525,13 @@ @implementation XCWH264Encoder { NSUInteger _keyFrameOutputCount; NSUInteger _maxInFlightFrameCount; uint64_t _latestEncodeLatencyUs; + double _averageEncodeLatencyUs; + uint64_t _peakEncodeLatencyUs; + NSUInteger _overBudgetFrameCount; + NSUInteger _consecutiveOverBudgetFrameCount; + NSUInteger _consecutiveStrainedFrameCount; + NSUInteger _overloadEventCount; + BOOL _wasOverloaded; uint64_t _softwareFrameIntervalUs; uint64_t _lastSoftwareSubmissionUs; NSUInteger _softwarePacedFrameCount; @@ -621,6 +633,33 @@ - (NSDictionary *)statsRepresentation { __block NSDictionary *stats = nil; dispatch_sync(_queue, ^{ + uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked]; + double latestLoadPercent = encoderBudgetUs > 0 + ? ((double)self->_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + double averageLoadPercent = encoderBudgetUs > 0 + ? (self->_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent || + self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold; + BOOL strained = overloaded || + averageLoadPercent >= XCWEncoderStrainedLoadPercent || + self->_consecutiveStrainedFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold; + NSString *overloadState = overloaded + ? @"overloaded" + : strained + ? @"strained" + : @"nominal"; + NSString *overloadReason = @"within-budget"; + if (overloaded) { + overloadReason = self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold + ? @"consecutive-frames-over-budget" + : @"average-latency-over-budget"; + } else if (strained) { + overloadReason = averageLoadPercent >= XCWEncoderStrainedLoadPercent + ? @"average-latency-near-budget" + : @"consecutive-frames-near-budget"; + } stats = @{ @"inputFrames": @(inputFrameCount), @"pendingReplacements": @(pendingReplacementCount), @@ -631,6 +670,18 @@ - (NSDictionary *)statsRepresentation { @"inFlightFrames": @(self->_inFlightFrameCount), @"maxInFlightFrames": @(self->_maxInFlightFrameCount), @"latestEncodeLatencyUs": @(self->_latestEncodeLatencyUs), + @"averageEncodeLatencyUs": @(self->_averageEncodeLatencyUs), + @"peakEncodeLatencyUs": @(self->_peakEncodeLatencyUs), + @"encoderBudgetUs": @(encoderBudgetUs), + @"encoderLoadPercent": @(latestLoadPercent), + @"averageEncoderLoadPercent": @(averageLoadPercent), + @"overloadState": overloadState, + @"overloaded": @(overloaded), + @"overloadReason": overloadReason, + @"overBudgetFrames": @(self->_overBudgetFrameCount), + @"consecutiveOverBudgetFrames": @(self->_consecutiveOverBudgetFrameCount), + @"consecutiveStrainedFrames": @(self->_consecutiveStrainedFrameCount), + @"overloadEvents": @(self->_overloadEventCount), @"softwareFrameIntervalUs": @(self->_softwareFrameIntervalUs), @"softwareTargetFps": @(self->_softwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_softwareFrameIntervalUs) : 0.0), @"softwarePacedFrames": @(self->_softwarePacedFrameCount), @@ -727,6 +778,17 @@ - (uint64_t)maximumHardwareFrameIntervalUsLocked { return XCWLocalStreamMaximumFrameIntervalUs(); } +- (uint64_t)activeFrameIntervalUsLocked { + if (_encoderMode == XCWVideoEncoderModeH264Software) { + return _softwareFrameIntervalUs > 0 ? _softwareFrameIntervalUs : [self initialSoftwareFrameIntervalUsLocked]; + } + if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) { + return _hardwareFrameIntervalUs > 0 ? _hardwareFrameIntervalUs : [self initialHardwareFrameIntervalUsLocked]; + } + int32_t expectedFrameRate = MAX(1, [self expectedFrameRateLocked]); + return (uint64_t)llround(1000000.0 / (double)expectedFrameRate); +} + - (int32_t)expectedFrameRateLocked { if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { @@ -1075,6 +1137,12 @@ - (void)invalidateCompressionSessionLocked { _inFlightFrameCount = 0; _lastSoftwareSubmissionUs = 0; _lastHardwareSubmissionUs = 0; + _latestEncodeLatencyUs = 0; + _averageEncodeLatencyUs = 0; + _peakEncodeLatencyUs = 0; + _consecutiveOverBudgetFrameCount = 0; + _consecutiveStrainedFrameCount = 0; + _wasOverloaded = NO; _hardwareAccelerated = NO; _selectedEncoderID = nil; _scalingActive = NO; @@ -1303,6 +1371,34 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer if (submittedAtUs > 0) { uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); _latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0; + _peakEncodeLatencyUs = MAX(_peakEncodeLatencyUs, _latestEncodeLatencyUs); + _averageEncodeLatencyUs = _averageEncodeLatencyUs <= 0.0 + ? (double)_latestEncodeLatencyUs + : (_averageEncodeLatencyUs * (1.0 - XCWEncoderLatencyEWMAAlpha)) + ((double)_latestEncodeLatencyUs * XCWEncoderLatencyEWMAAlpha); + uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked]; + double averageLoadPercent = encoderBudgetUs > 0 + ? (_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + double latestLoadPercent = encoderBudgetUs > 0 + ? ((double)_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + if (encoderBudgetUs > 0 && _latestEncodeLatencyUs > encoderBudgetUs) { + _overBudgetFrameCount += 1; + _consecutiveOverBudgetFrameCount += 1; + } else { + _consecutiveOverBudgetFrameCount = 0; + } + if (latestLoadPercent >= XCWEncoderStrainedLoadPercent) { + _consecutiveStrainedFrameCount += 1; + } else { + _consecutiveStrainedFrameCount = 0; + } + BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent || + _consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold; + if (overloaded && !_wasOverloaded) { + _overloadEventCount += 1; + } + _wasOverloaded = overloaded; [self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs]; [self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs]; } diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 44a65b5..b775516 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -1,8 +1,26 @@ +export interface EncoderStats { + averageEncodeLatencyUs?: number; + averageEncoderLoadPercent?: number; + consecutiveOverBudgetFrames?: number; + encoderBudgetUs?: number; + encoderLoadPercent?: number; + encoderMode?: string; + hardwareAccelerated?: boolean; + latestEncodeLatencyUs?: number; + overloadEvents?: number; + overloaded?: boolean; + overloadReason?: string; + overloadState?: "nominal" | "strained" | "overloaded" | string; + peakEncodeLatencyUs?: number; + selectedEncoderId?: string | null; +} + export interface PrivateDisplayInfo { displayReady: boolean; displayStatus: string; displayWidth: number; displayHeight: number; + encoder?: EncoderStats; frameSequence: number; rotationQuarterTurns?: number; } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index b3f9580..6425be9 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -1669,6 +1669,7 @@ export function AppShell({ debugPanel={ debugVisible ? ( setDebugVisible(false)} @@ -1827,6 +1828,12 @@ function normalizeStreamQuality( if (normalized === "economy" || normalized === "ci-software") { return "economy"; } + if (normalized === "low") { + return "low"; + } + if (normalized === "tiny") { + return "tiny"; + } return fallback; } diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index f220b0d..174ccbc 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -237,9 +237,11 @@ const STREAM_QUALITY_OPTIONS: Array<{ label: string; value: StreamQualityPreset; }> = [ - { label: "Quality", value: "quality" }, - { label: "Balanced", value: "balanced" }, - { label: "Economy", value: "economy" }, + { label: "Full", value: "quality" }, + { label: "1280", value: "balanced" }, + { label: "1080", value: "economy" }, + { label: "720", value: "low" }, + { label: "540", value: "tiny" }, ]; function MenuIcon() { diff --git a/client/src/features/stream/streamTypes.ts b/client/src/features/stream/streamTypes.ts index c1b3b61..37404a3 100644 --- a/client/src/features/stream/streamTypes.ts +++ b/client/src/features/stream/streamTypes.ts @@ -14,8 +14,10 @@ export type StreamQualityPreset = | "ci-software" | "economy" | "fast" + | "low" | "quality" - | "smooth"; + | "smooth" + | "tiny"; export interface StreamConfig { encoder: StreamEncoder; diff --git a/client/src/features/toolbar/DebugPanel.tsx b/client/src/features/toolbar/DebugPanel.tsx index faeefdd..65ea520 100644 --- a/client/src/features/toolbar/DebugPanel.tsx +++ b/client/src/features/toolbar/DebugPanel.tsx @@ -1,3 +1,4 @@ +import type { EncoderStats } from "../../api/types"; import type { StreamRuntimeInfo, StreamStats, @@ -5,6 +6,7 @@ import type { } from "../stream/streamTypes"; interface DebugPanelProps { + encoder?: EncoderStats | null; fps: number; inline?: boolean; onClose?: () => void; @@ -27,6 +29,20 @@ function formatMs(value: number): string { return `${value.toFixed(1)} ms`; } +function formatUsAsMs(value: number | undefined): string { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return "—"; + } + return `${(value / 1000).toFixed(1)} ms`; +} + +function formatPercent(value: number | undefined): string { + if (typeof value !== "number" || !Number.isFinite(value)) { + return "—"; + } + return `${value.toFixed(0)}%`; +} + function formatResolution(stats: StreamStats): string { if (!stats.width || !stats.height) { return "—"; @@ -35,6 +51,7 @@ function formatResolution(stats: StreamStats): string { } export function DebugPanel({ + encoder, fps, inline = false, onClose, @@ -60,6 +77,26 @@ export function DebugPanel({ { label: "Frame Gap", value: formatMs(stats.latestFrameGapMs) }, { label: "Path", value: runtimeInfo.streamBackend }, ]; + if (encoder) { + rows.push( + { label: "Encoder", value: encoder.encoderMode ?? "—" }, + { label: "Encoder State", value: encoder.overloadState ?? "—" }, + { + label: "Encoder Load", + value: formatPercent(encoder.averageEncoderLoadPercent), + }, + { + label: "Encode Latency", + value: formatUsAsMs(encoder.averageEncodeLatencyUs), + }, + { label: "Encode Budget", value: formatUsAsMs(encoder.encoderBudgetUs) }, + { label: "Encoder Reason", value: encoder.overloadReason ?? "—" }, + { + label: "Overload Events", + value: String(encoder.overloadEvents ?? 0), + }, + ); + } return (
` element.** The canvas remains for input geometry, but it is not in the live per-frame render path and does not preserve stale frames during reconnects. - **Use `--stream-quality ci-software` for denser virtualized CI Macs.** This profile uses software H.264 at a 960-pixel longest edge, targets 24 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. - **Use `simdeck studio expose --video-codec hardware` only when a dedicated hardware encoder is preferable.** The normal Studio default stays on software H.264 so future multi-simulator provider hosts can scale across CPU cores. @@ -115,6 +116,13 @@ Useful signals: | `keyframe_requests` | Goes up every time the server forces a refresh. Frequent spikes mean rough seeks. | | `active_streams` | Number of WebRTC streams currently subscribed. | +`encoders[].encoder.overloadState` reports native encoder pressure for each +active simulator session. `strained` means encode latency is approaching the +active frame budget; `overloaded` means smoothed latency is over budget or +multiple frames in a row exceeded the budget. For hardware H.264 this usually +means the shared VideoToolbox encoder is saturated; lower resolution/FPS or +switch to software H.264. + Clients can also push their decoder/renderer stats back to the server: ```http diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 6d908f9..ae78936 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -185,9 +185,26 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ min_bitrate: 1_500_000, bits_per_pixel: 3, }, + StreamQualityProfile { + id: "low", + label: "Low", + max_edge: 720, + fps: 30, + min_bitrate: 900_000, + bits_per_pixel: 2, + }, + StreamQualityProfile { + id: "tiny", + label: "Tiny", + max_edge: 540, + fps: 24, + min_bitrate: 600_000, + bits_per_pixel: 2, + }, ]; -const VISIBLE_STREAM_QUALITY_PROFILE_IDS: &[&str] = &["quality", "balanced", "economy"]; +const VISIBLE_STREAM_QUALITY_PROFILE_IDS: &[&str] = + &["quality", "balanced", "economy", "low", "tiny"]; static STREAM_CONFIG_LOCK: OnceLock> = OnceLock::new(); @@ -626,7 +643,11 @@ fn normalize_video_codec(codec: &str) -> Option<&'static str> { } async fn metrics(State(state): State) -> Json { - json(json_value!(state.metrics.snapshot())) + let mut snapshot = json_value!(state.metrics.snapshot()); + if let Some(object) = snapshot.as_object_mut() { + object.insert("encoders".to_owned(), json_value!(state.registry.encoder_snapshots())); + } + json(snapshot) } async fn stream_quality(State(state): State) -> Json { diff --git a/server/src/main.rs b/server/src/main.rs index 7992a0c..afa406f 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -550,6 +550,8 @@ enum StreamQualityProfileArg { Fast, Smooth, Economy, + Low, + Tiny, CiSoftware, } @@ -561,6 +563,8 @@ impl StreamQualityProfileArg { Self::Fast => "fast", Self::Smooth => "smooth", Self::Economy => "economy", + Self::Low => "low", + Self::Tiny => "tiny", Self::CiSoftware => "ci-software", } } @@ -697,6 +701,20 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { + profile: "low", + max_edge: 720, + fps: 30, + min_bitrate: 900_000, + bits_per_pixel: 2, + }), + "tiny" => Ok(StreamQualityEnvironment { + profile: "tiny", + max_edge: 540, + fps: 24, + min_bitrate: 600_000, + bits_per_pixel: 2, + }), "ci-software" => Ok(StreamQualityEnvironment { profile: "ci-software", max_edge: 960, diff --git a/server/src/simulators/registry.rs b/server/src/simulators/registry.rs index d3cd8ac..f3bae59 100644 --- a/server/src/simulators/registry.rs +++ b/server/src/simulators/registry.rs @@ -132,6 +132,20 @@ impl SessionRegistry { } } + pub fn encoder_snapshots(&self) -> Vec { + self.store + .values() + .into_iter() + .map(|session| { + let snapshot = session.snapshot(); + json!({ + "udid": session.udid(), + "encoder": snapshot.get("encoder").cloned().unwrap_or_else(|| json!({})), + }) + }) + .collect() + } + pub fn enrich_simulators(&self, simulators: Vec) -> Vec { simulators .into_iter() diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs index 7ae4f31..ab33827 100644 --- a/server/src/simulators/session.rs +++ b/server/src/simulators/session.rs @@ -70,6 +70,10 @@ impl Drop for FrameSubscription { } impl SimulatorSession { + pub fn udid(&self) -> &str { + &self.inner.udid + } + pub fn new( bridge: &NativeBridge, udid: String, From 724a9f469a4177d145c7f4f801b1f015f5c4e123 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 6 May 2026 00:01:46 -0400 Subject: [PATCH 3/5] Fix CI formatting --- docs/api/health.md | 8 ++++---- docs/cli/flags.md | 20 ++++++++++---------- server/src/api/routes.rs | 5 ++++- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/docs/api/health.md b/docs/api/health.md index e139d6d..924c757 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -105,10 +105,10 @@ Returns a snapshot of every server-side counter and the rolling buffer of client `encoders` contains one entry per active simulator session. `encoder.overloadState` is derived from native VideoToolbox submit-to-output latency: -| State | Meaning | -| ------------ | ---------------------------------------------------------------------------------------- | -| `nominal` | Smoothed encode latency is comfortably below the active frame budget. | -| `strained` | Smoothed latency is near the frame budget or several frames are close to budget. | +| State | Meaning | +| ------------ | --------------------------------------------------------------------------------------- | +| `nominal` | Smoothed encode latency is comfortably below the active frame budget. | +| `strained` | Smoothed latency is near the frame budget or several frames are close to budget. | | `overloaded` | Smoothed latency exceeds the budget or several frames in a row took longer than budget. | This is an inferred pressure signal rather than a private macOS hardware queue diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 0628dfb..466c370 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -28,17 +28,17 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas `ui`, `daemon start`, and `daemon restart` accept the same server options. `ui` also accepts `--open`. -| Flag | Default | Description | -| -------------------- | --------------------- | ------------------------------------------------------------------------------------------------------ | -| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | -| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | -| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | +| Flag | Default | Description | +| -------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | +| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | +| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | +| `--client-root` | bundled `client/dist` | Override the static browser client directory. | +| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | +| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | | `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | +| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | `studio expose` defaults to software H.264. Pass `--video-codec hardware` to opt into the hardware encoder when that is preferable. diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index ae78936..185a710 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -645,7 +645,10 @@ fn normalize_video_codec(codec: &str) -> Option<&'static str> { async fn metrics(State(state): State) -> Json { let mut snapshot = json_value!(state.metrics.snapshot()); if let Some(object) = snapshot.as_object_mut() { - object.insert("encoders".to_owned(), json_value!(state.registry.encoder_snapshots())); + object.insert( + "encoders".to_owned(), + json_value!(state.registry.encoder_snapshots()), + ); } json(snapshot) } From 8ac6556b7b527593a508134d9d4d50d95d14b800 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 6 May 2026 00:22:58 -0400 Subject: [PATCH 4/5] Stabilize CLI simulator integration --- scripts/integration/cli.mjs | 86 +++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index 2fe9166..994aab5 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -156,28 +156,7 @@ async function main() { await measuredStep( "setup launch SwiftUI fixture", - async () => { - try { - await retrySimdeckJson( - cliArgs(["launch", simulatorUDID, fixtureBundleId]), - "setup launch SwiftUI fixture", - { - attempts: 3, - delayMs: 5_000, - timeoutMs: 180_000, - }, - ); - await verifyUi("setup launch SwiftUI fixture", { - expectFixture: true, - phase: phaseSetup, - waitTimeoutMs: 3_000, - }); - } catch (error) { - logStep( - `setup warm launch skipped: ${String(error?.message ?? error).split("\n")[0]}`, - ); - } - }, + () => ensureFixtureForeground("setup launch SwiftUI fixture"), { phase: phaseSetup }, ); @@ -442,6 +421,11 @@ async function runCliControls() { {}, { skip: true, phase: phaseCommandSmoke }, ); + await measuredStep( + "CLI foreground fixture before URL", + () => ensureFixtureForeground("CLI foreground fixture before URL"), + { phase: phaseTest }, + ); await cliStep( "CLI open fixture URL", ["open-url", simulatorUDID, fixtureUrl], @@ -817,6 +801,64 @@ async function cliStep(label, args, commandOptions = {}, verifyOptions = {}) { ); } +async function ensureFixtureForeground(label, options = {}) { + let launchError = null; + try { + await retrySimdeckJson( + cliArgs(["launch", simulatorUDID, fixtureBundleId]), + label, + { + attempts: options.launchAttempts ?? 1, + delayMs: options.launchDelayMs ?? 5_000, + timeoutMs: options.launchTimeoutMs ?? 180_000, + }, + ); + } catch (error) { + launchError = error; + logStep( + `${label}: launch command reported ${String(error?.message ?? error).split("\n")[0]}`, + ); + } + + try { + return await verifyUi(label, { + expectFixture: true, + attempts: options.verifyAttempts ?? 8, + delayMs: options.verifyDelayMs ?? 1_000, + }); + } catch (verifyError) { + if (launchError === null) { + throw verifyError; + } + logStep(`${label}: tapping fixture icon after launch timeout`); + } + + await retrySimdeckJson( + cliArgs([ + "tap", + simulatorUDID, + "--label", + "SimDeckFixture", + "--wait-timeout-ms", + "15000", + "--duration-ms", + "30", + ]), + `${label} fixture icon tap`, + { + attempts: 2, + delayMs: 2_000, + timeoutMs: 180_000, + }, + ); + + return verifyUi(label, { + expectFixture: true, + attempts: options.fallbackVerifyAttempts ?? 12, + delayMs: options.fallbackVerifyDelayMs ?? 1_500, + }); +} + function cliArgs(args) { return args; } From 10118819630d31a7a67b6d65010806747cf4a50f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 6 May 2026 00:38:30 -0400 Subject: [PATCH 5/5] Wait for fixture registration in CLI integration --- scripts/integration/cli.mjs | 42 +++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index 994aab5..90db63e 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -153,6 +153,9 @@ async function main() { }, { phase: phaseSetup }, ); + await measuredStep("fixture app registration", waitForFixtureRegistration, { + phase: phaseSetup, + }); await measuredStep( "setup launch SwiftUI fixture", @@ -815,16 +818,16 @@ async function ensureFixtureForeground(label, options = {}) { ); } catch (error) { launchError = error; - logStep( - `${label}: launch command reported ${String(error?.message ?? error).split("\n")[0]}`, - ); + logStep(`${label}: launch command reported ${summarizeError(error)}`); } try { return await verifyUi(label, { expectFixture: true, - attempts: options.verifyAttempts ?? 8, + attempts: options.verifyAttempts ?? (launchError === null ? 8 : 2), delayMs: options.verifyDelayMs ?? 1_000, + waitTimeoutMs: + options.waitTimeoutMs ?? (launchError === null ? 5_000 : 1_000), }); } catch (verifyError) { if (launchError === null) { @@ -859,6 +862,37 @@ async function ensureFixtureForeground(label, options = {}) { }); } +async function waitForFixtureRegistration() { + let lastError = null; + for (let attempt = 1; attempt <= 12; attempt += 1) { + try { + const appContainer = runText( + "xcrun", + ["simctl", "get_app_container", simulatorUDID, fixtureBundleId], + { timeoutMs: 60_000 }, + ).trim(); + if (appContainer.length > 0) { + logStep(`fixture registered at ${appContainer}`); + return appContainer; + } + } catch (error) { + lastError = error; + } + await sleep(1_000); + } + throw new Error( + `fixture app was not registered after install: ${summarizeError(lastError)}`, + ); +} + +function summarizeError(error) { + return String(error?.message ?? error) + .split("\n") + .filter(Boolean) + .slice(0, 3) + .join(" | "); +} + function cliArgs(args) { return args; }