diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index bf4eb97..eb59be5 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -40,6 +40,10 @@ static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; static const NSUInteger XCWMaximumRealtimeInFlightFrames = 3; static const int32_t XCWRealtimeKeyFrameIntervalSeconds = 5; +static const double XCWEncoderLatencyEWMAAlpha = 0.2; +static const double XCWEncoderStrainedLoadPercent = 85.0; +static const double XCWEncoderOverloadedLoadPercent = 105.0; +static const NSUInteger XCWEncoderConsecutiveOverBudgetFrameThreshold = 3; typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeAuto, @@ -477,11 +481,17 @@ @interface XCWH264Encoder () - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer targetWidth:(int32_t)targetWidth targetHeight:(int32_t)targetHeight; +- (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth + sourceHeight:(int32_t)sourceHeight + targetWidth:(int32_t)targetWidth + targetHeight:(int32_t)targetHeight + pixelFormat:(OSType)pixelFormat; - (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)targetWidth height:(int32_t)targetHeight pixelFormat:(OSType)pixelFormat; - (void)handleCompressionOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer submittedAtUs:(uint64_t)submittedAtUs; +- (uint64_t)activeFrameIntervalUsLocked; @end @@ -515,6 +525,13 @@ @implementation XCWH264Encoder { NSUInteger _keyFrameOutputCount; NSUInteger _maxInFlightFrameCount; uint64_t _latestEncodeLatencyUs; + double _averageEncodeLatencyUs; + uint64_t _peakEncodeLatencyUs; + NSUInteger _overBudgetFrameCount; + NSUInteger _consecutiveOverBudgetFrameCount; + NSUInteger _consecutiveStrainedFrameCount; + NSUInteger _overloadEventCount; + BOOL _wasOverloaded; uint64_t _softwareFrameIntervalUs; uint64_t _lastSoftwareSubmissionUs; NSUInteger _softwarePacedFrameCount; @@ -616,6 +633,33 @@ - (NSDictionary *)statsRepresentation { __block NSDictionary *stats = nil; dispatch_sync(_queue, ^{ + uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked]; + double latestLoadPercent = encoderBudgetUs > 0 + ? ((double)self->_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + double averageLoadPercent = encoderBudgetUs > 0 + ? (self->_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent || + self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold; + BOOL strained = overloaded || + averageLoadPercent >= XCWEncoderStrainedLoadPercent || + self->_consecutiveStrainedFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold; + NSString *overloadState = overloaded + ? @"overloaded" + : strained + ? @"strained" + : @"nominal"; + NSString *overloadReason = @"within-budget"; + if (overloaded) { + overloadReason = self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold + ? @"consecutive-frames-over-budget" + : @"average-latency-over-budget"; + } else if (strained) { + overloadReason = averageLoadPercent >= XCWEncoderStrainedLoadPercent + ? @"average-latency-near-budget" + : @"consecutive-frames-near-budget"; + } stats = @{ @"inputFrames": @(inputFrameCount), @"pendingReplacements": @(pendingReplacementCount), @@ -626,6 +670,18 @@ - (NSDictionary *)statsRepresentation { @"inFlightFrames": @(self->_inFlightFrameCount), @"maxInFlightFrames": @(self->_maxInFlightFrameCount), @"latestEncodeLatencyUs": @(self->_latestEncodeLatencyUs), + @"averageEncodeLatencyUs": @(self->_averageEncodeLatencyUs), + @"peakEncodeLatencyUs": @(self->_peakEncodeLatencyUs), + @"encoderBudgetUs": @(encoderBudgetUs), + @"encoderLoadPercent": @(latestLoadPercent), + @"averageEncoderLoadPercent": @(averageLoadPercent), + @"overloadState": overloadState, + @"overloaded": @(overloaded), + @"overloadReason": overloadReason, + @"overBudgetFrames": @(self->_overBudgetFrameCount), + @"consecutiveOverBudgetFrames": @(self->_consecutiveOverBudgetFrameCount), + @"consecutiveStrainedFrames": @(self->_consecutiveStrainedFrameCount), + @"overloadEvents": @(self->_overloadEventCount), @"softwareFrameIntervalUs": @(self->_softwareFrameIntervalUs), @"softwareTargetFps": @(self->_softwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_softwareFrameIntervalUs) : 0.0), @"softwarePacedFrames": @(self->_softwarePacedFrameCount), @@ -715,7 +771,22 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked { } - (uint64_t)maximumHardwareFrameIntervalUsLocked { - return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWLocalStreamMaximumFrameIntervalUs(); + if (_realtimeStreamMode) { + uint64_t minimumFpsIntervalUs = (uint64_t)llround(1000000.0 / (double)XCWMinimumLocalStreamFrameRate); + return MAX(XCWRealtimeMaximumFrameIntervalUs(), minimumFpsIntervalUs); + } + return XCWLocalStreamMaximumFrameIntervalUs(); +} + +- (uint64_t)activeFrameIntervalUsLocked { + if (_encoderMode == XCWVideoEncoderModeH264Software) { + return _softwareFrameIntervalUs > 0 ? _softwareFrameIntervalUs : [self initialSoftwareFrameIntervalUsLocked]; + } + if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) { + return _hardwareFrameIntervalUs > 0 ? _hardwareFrameIntervalUs : [self initialHardwareFrameIntervalUsLocked]; + } + int32_t expectedFrameRate = MAX(1, [self expectedFrameRateLocked]); + return (uint64_t)llround(1000000.0 / (double)expectedFrameRate); } - (int32_t)expectedFrameRateLocked { @@ -807,7 +878,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs { - if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_lowLatencyMode || latencyUs == 0) { + if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || latencyUs == 0) { return; } if (_hardwareFrameIntervalUs == 0) { @@ -1066,6 +1137,12 @@ - (void)invalidateCompressionSessionLocked { _inFlightFrameCount = 0; _lastSoftwareSubmissionUs = 0; _lastHardwareSubmissionUs = 0; + _latestEncodeLatencyUs = 0; + _averageEncodeLatencyUs = 0; + _peakEncodeLatencyUs = 0; + _consecutiveOverBudgetFrameCount = 0; + _consecutiveStrainedFrameCount = 0; + _wasOverloaded = NO; _hardwareAccelerated = NO; _selectedEncoderID = nil; _scalingActive = NO; @@ -1077,11 +1154,24 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix targetHeight:(int32_t)targetHeight { int32_t sourceWidth = (int32_t)CVPixelBufferGetWidth(pixelBuffer); int32_t sourceHeight = (int32_t)CVPixelBufferGetHeight(pixelBuffer); + OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); BOOL shouldCopyStableRealtimeBuffer = _realtimeStreamMode || _lowLatencyMode; if (sourceWidth == targetWidth && sourceHeight == targetHeight && !shouldCopyStableRealtimeBuffer) { CVPixelBufferRetain(pixelBuffer); return pixelBuffer; } + if ([self shouldUseSoftwareScalerForSourceWidth:sourceWidth + sourceHeight:sourceHeight + targetWidth:targetWidth + targetHeight:targetHeight + pixelFormat:sourcePixelFormat]) { + CVPixelBufferRef scaledPixelBuffer = [self copySoftwareScaledPixelBuffer:pixelBuffer + targetWidth:targetWidth + targetHeight:targetHeight]; + if (scaledPixelBuffer != NULL) { + return scaledPixelBuffer; + } + } if (_pixelTransferSession == NULL) { OSStatus sessionStatus = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_pixelTransferSession); @@ -1105,7 +1195,6 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix kVTScalingMode_Normal); } - OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); CVPixelBufferRef scaledPixelBuffer = [self copyPixelBufferFromScalingPoolWithWidth:targetWidth height:targetHeight pixelFormat:sourcePixelFormat]; @@ -1130,6 +1219,20 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix return scaledPixelBuffer; } +- (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth + sourceHeight:(int32_t)sourceHeight + targetWidth:(int32_t)targetWidth + targetHeight:(int32_t)targetHeight + pixelFormat:(OSType)pixelFormat { + if (!_realtimeStreamMode || !XCWPixelFormatSupportsSoftwareScaling(pixelFormat)) { + return NO; + } + if (sourceWidth == targetWidth && sourceHeight == targetHeight) { + return NO; + } + return _encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware; +} + - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer targetWidth:(int32_t)targetWidth targetHeight:(int32_t)targetHeight { @@ -1268,6 +1371,34 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer if (submittedAtUs > 0) { uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); _latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0; + _peakEncodeLatencyUs = MAX(_peakEncodeLatencyUs, _latestEncodeLatencyUs); + _averageEncodeLatencyUs = _averageEncodeLatencyUs <= 0.0 + ? (double)_latestEncodeLatencyUs + : (_averageEncodeLatencyUs * (1.0 - XCWEncoderLatencyEWMAAlpha)) + ((double)_latestEncodeLatencyUs * XCWEncoderLatencyEWMAAlpha); + uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked]; + double averageLoadPercent = encoderBudgetUs > 0 + ? (_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + double latestLoadPercent = encoderBudgetUs > 0 + ? ((double)_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs + : 0.0; + if (encoderBudgetUs > 0 && _latestEncodeLatencyUs > encoderBudgetUs) { + _overBudgetFrameCount += 1; + _consecutiveOverBudgetFrameCount += 1; + } else { + _consecutiveOverBudgetFrameCount = 0; + } + if (latestLoadPercent >= XCWEncoderStrainedLoadPercent) { + _consecutiveStrainedFrameCount += 1; + } else { + _consecutiveStrainedFrameCount = 0; + } + BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent || + _consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold; + if (overloaded && !_wasOverloaded) { + _overloadEventCount += 1; + } + _wasOverloaded = overloaded; [self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs]; [self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs]; } diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 44a65b5..b775516 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -1,8 +1,26 @@ +export interface EncoderStats { + averageEncodeLatencyUs?: number; + averageEncoderLoadPercent?: number; + consecutiveOverBudgetFrames?: number; + encoderBudgetUs?: number; + encoderLoadPercent?: number; + encoderMode?: string; + hardwareAccelerated?: boolean; + latestEncodeLatencyUs?: number; + overloadEvents?: number; + overloaded?: boolean; + overloadReason?: string; + overloadState?: "nominal" | "strained" | "overloaded" | string; + peakEncodeLatencyUs?: number; + selectedEncoderId?: string | null; +} + export interface PrivateDisplayInfo { displayReady: boolean; displayStatus: string; displayWidth: number; displayHeight: number; + encoder?: EncoderStats; frameSequence: number; rotationQuarterTurns?: number; } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index b3f9580..6425be9 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -1669,6 +1669,7 @@ export function AppShell({ debugPanel={ debugVisible ? ( setDebugVisible(false)} @@ -1827,6 +1828,12 @@ function normalizeStreamQuality( if (normalized === "economy" || normalized === "ci-software") { return "economy"; } + if (normalized === "low") { + return "low"; + } + if (normalized === "tiny") { + return "tiny"; + } return fallback; } diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index f220b0d..174ccbc 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -237,9 +237,11 @@ const STREAM_QUALITY_OPTIONS: Array<{ label: string; value: StreamQualityPreset; }> = [ - { label: "Quality", value: "quality" }, - { label: "Balanced", value: "balanced" }, - { label: "Economy", value: "economy" }, + { label: "Full", value: "quality" }, + { label: "1280", value: "balanced" }, + { label: "1080", value: "economy" }, + { label: "720", value: "low" }, + { label: "540", value: "tiny" }, ]; function MenuIcon() { diff --git a/client/src/features/stream/streamTypes.ts b/client/src/features/stream/streamTypes.ts index c1b3b61..37404a3 100644 --- a/client/src/features/stream/streamTypes.ts +++ b/client/src/features/stream/streamTypes.ts @@ -14,8 +14,10 @@ export type StreamQualityPreset = | "ci-software" | "economy" | "fast" + | "low" | "quality" - | "smooth"; + | "smooth" + | "tiny"; export interface StreamConfig { encoder: StreamEncoder; diff --git a/client/src/features/toolbar/DebugPanel.tsx b/client/src/features/toolbar/DebugPanel.tsx index faeefdd..65ea520 100644 --- a/client/src/features/toolbar/DebugPanel.tsx +++ b/client/src/features/toolbar/DebugPanel.tsx @@ -1,3 +1,4 @@ +import type { EncoderStats } from "../../api/types"; import type { StreamRuntimeInfo, StreamStats, @@ -5,6 +6,7 @@ import type { } from "../stream/streamTypes"; interface DebugPanelProps { + encoder?: EncoderStats | null; fps: number; inline?: boolean; onClose?: () => void; @@ -27,6 +29,20 @@ function formatMs(value: number): string { return `${value.toFixed(1)} ms`; } +function formatUsAsMs(value: number | undefined): string { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return "—"; + } + return `${(value / 1000).toFixed(1)} ms`; +} + +function formatPercent(value: number | undefined): string { + if (typeof value !== "number" || !Number.isFinite(value)) { + return "—"; + } + return `${value.toFixed(0)}%`; +} + function formatResolution(stats: StreamStats): string { if (!stats.width || !stats.height) { return "—"; @@ -35,6 +51,7 @@ function formatResolution(stats: StreamStats): string { } export function DebugPanel({ + encoder, fps, inline = false, onClose, @@ -60,6 +77,26 @@ export function DebugPanel({ { label: "Frame Gap", value: formatMs(stats.latestFrameGapMs) }, { label: "Path", value: runtimeInfo.streamBackend }, ]; + if (encoder) { + rows.push( + { label: "Encoder", value: encoder.encoderMode ?? "—" }, + { label: "Encoder State", value: encoder.overloadState ?? "—" }, + { + label: "Encoder Load", + value: formatPercent(encoder.averageEncoderLoadPercent), + }, + { + label: "Encode Latency", + value: formatUsAsMs(encoder.averageEncodeLatencyUs), + }, + { label: "Encode Budget", value: formatUsAsMs(encoder.encoderBudgetUs) }, + { label: "Encoder Reason", value: encoder.overloadReason ?? "—" }, + { + label: "Overload Events", + value: String(encoder.overloadEvents ?? 0), + }, + ); + } return (
` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | -| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | -| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | -| `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, or `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +| Flag | Default | Description | +| -------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | +| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | +| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | +| `--client-root` | bundled `client/dist` | Override the static browser client directory. | +| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | +| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | +| `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | +| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | +| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | `studio expose` defaults to software H.264. Pass `--video-codec hardware` to opt into the hardware encoder when that is preferable. diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index 5443305..c7553e7 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -62,7 +62,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and | `--client-root` | bundled `client/dist` | Override the static browser client directory. | | `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | -| `--stream-quality` | `smooth` | Realtime stream quality profile, including `ci-software` for CI providers. | +| `--stream-quality` | `smooth` | Realtime stream quality profile, including `low`, `tiny`, and `ci-software`. | | `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/video.md b/docs/guide/video.md index e59c851..cb5700a 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -32,7 +32,7 @@ It is CLI-only because it is meant for less capable machines where freshness matters more than maximum smoothness. The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`. -The browser UI exposes stream controls for encoder, FPS, and three quality choices: `quality`, `balanced`, and `economy`. Local browser sessions default to hardware H.264, 120 fps, and `quality`/full resolution with FPS choices of 30, 60, and 120. Remote browser sessions default to software H.264, 30 fps, and `balanced` with FPS choices of 15, 30, and 60. +The browser UI exposes stream controls for encoder, FPS, and five quality choices: `quality` (4096 px), `balanced` (1280 px), `economy` (1080 px), `low` (720 px), and `tiny` (540 px). Local browser sessions default to hardware H.264, 120 fps, and `quality`/full resolution with FPS choices of 30, 60, and 120. Remote browser sessions default to software H.264, 30 fps, and `balanced` with FPS choices of 15, 30, and 60. ## Remote WebRTC ICE @@ -82,6 +82,7 @@ A few practical guidelines: - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. - **Studio providers default to software H.264 plus `--stream-quality smooth`.** This profile uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. +- **Use `low` or `tiny` when resolution is the bottleneck.** `low` caps the longest edge at 720 pixels and targets 30 fps; `tiny` caps the longest edge at 540 pixels and targets 24 fps. - **The remote browser renders the live stream as a native `