Skip to content

Commit b641805

Browse files
DjDeveloperrDjDeveloperr
authored
Fix hardware realtime downscaling lag (#20)
* Fix hardware stream downscaling lag * Add encoder overload detection * Fix CI formatting * Stabilize CLI simulator integration * Wait for fixture registration in CLI integration
1 parent f06df49 commit b641805

16 files changed

Lines changed: 422 additions & 49 deletions

File tree

‎cli/XCWH264Encoder.m‎

Lines changed: 134 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6;
4141
static const NSUInteger XCWMaximumRealtimeInFlightFrames = 3;
4242
static const int32_t XCWRealtimeKeyFrameIntervalSeconds = 5;
43+
static const double XCWEncoderLatencyEWMAAlpha = 0.2;
44+
static const double XCWEncoderStrainedLoadPercent = 85.0;
45+
static const double XCWEncoderOverloadedLoadPercent = 105.0;
46+
static const NSUInteger XCWEncoderConsecutiveOverBudgetFrameThreshold = 3;
4347

4448
typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) {
4549
XCWVideoEncoderModeAuto,
@@ -477,11 +481,17 @@ @interface XCWH264Encoder ()
477481
- (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer
478482
targetWidth:(int32_t)targetWidth
479483
targetHeight:(int32_t)targetHeight;
484+
- (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth
485+
sourceHeight:(int32_t)sourceHeight
486+
targetWidth:(int32_t)targetWidth
487+
targetHeight:(int32_t)targetHeight
488+
pixelFormat:(OSType)pixelFormat;
480489
- (nullable CVPixelBufferRef)copyPixelBufferFromScalingPoolWithWidth:(int32_t)targetWidth
481490
height:(int32_t)targetHeight
482491
pixelFormat:(OSType)pixelFormat;
483492
- (void)handleCompressionOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
484493
submittedAtUs:(uint64_t)submittedAtUs;
494+
- (uint64_t)activeFrameIntervalUsLocked;
485495

486496
@end
487497

@@ -515,6 +525,13 @@ @implementation XCWH264Encoder {
515525
NSUInteger _keyFrameOutputCount;
516526
NSUInteger _maxInFlightFrameCount;
517527
uint64_t _latestEncodeLatencyUs;
528+
double _averageEncodeLatencyUs;
529+
uint64_t _peakEncodeLatencyUs;
530+
NSUInteger _overBudgetFrameCount;
531+
NSUInteger _consecutiveOverBudgetFrameCount;
532+
NSUInteger _consecutiveStrainedFrameCount;
533+
NSUInteger _overloadEventCount;
534+
BOOL _wasOverloaded;
518535
uint64_t _softwareFrameIntervalUs;
519536
uint64_t _lastSoftwareSubmissionUs;
520537
NSUInteger _softwarePacedFrameCount;
@@ -616,6 +633,33 @@ - (NSDictionary *)statsRepresentation {
616633

617634
__block NSDictionary *stats = nil;
618635
dispatch_sync(_queue, ^{
636+
uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked];
637+
double latestLoadPercent = encoderBudgetUs > 0
638+
? ((double)self->_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
639+
: 0.0;
640+
double averageLoadPercent = encoderBudgetUs > 0
641+
? (self->_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
642+
: 0.0;
643+
BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent ||
644+
self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold;
645+
BOOL strained = overloaded ||
646+
averageLoadPercent >= XCWEncoderStrainedLoadPercent ||
647+
self->_consecutiveStrainedFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold;
648+
NSString *overloadState = overloaded
649+
? @"overloaded"
650+
: strained
651+
? @"strained"
652+
: @"nominal";
653+
NSString *overloadReason = @"within-budget";
654+
if (overloaded) {
655+
overloadReason = self->_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold
656+
? @"consecutive-frames-over-budget"
657+
: @"average-latency-over-budget";
658+
} else if (strained) {
659+
overloadReason = averageLoadPercent >= XCWEncoderStrainedLoadPercent
660+
? @"average-latency-near-budget"
661+
: @"consecutive-frames-near-budget";
662+
}
619663
stats = @{
620664
@"inputFrames": @(inputFrameCount),
621665
@"pendingReplacements": @(pendingReplacementCount),
@@ -626,6 +670,18 @@ - (NSDictionary *)statsRepresentation {
626670
@"inFlightFrames": @(self->_inFlightFrameCount),
627671
@"maxInFlightFrames": @(self->_maxInFlightFrameCount),
628672
@"latestEncodeLatencyUs": @(self->_latestEncodeLatencyUs),
673+
@"averageEncodeLatencyUs": @(self->_averageEncodeLatencyUs),
674+
@"peakEncodeLatencyUs": @(self->_peakEncodeLatencyUs),
675+
@"encoderBudgetUs": @(encoderBudgetUs),
676+
@"encoderLoadPercent": @(latestLoadPercent),
677+
@"averageEncoderLoadPercent": @(averageLoadPercent),
678+
@"overloadState": overloadState,
679+
@"overloaded": @(overloaded),
680+
@"overloadReason": overloadReason,
681+
@"overBudgetFrames": @(self->_overBudgetFrameCount),
682+
@"consecutiveOverBudgetFrames": @(self->_consecutiveOverBudgetFrameCount),
683+
@"consecutiveStrainedFrames": @(self->_consecutiveStrainedFrameCount),
684+
@"overloadEvents": @(self->_overloadEventCount),
629685
@"softwareFrameIntervalUs": @(self->_softwareFrameIntervalUs),
630686
@"softwareTargetFps": @(self->_softwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_softwareFrameIntervalUs) : 0.0),
631687
@"softwarePacedFrames": @(self->_softwarePacedFrameCount),
@@ -715,7 +771,22 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked {
715771
}
716772

717773
- (uint64_t)maximumHardwareFrameIntervalUsLocked {
718-
return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWLocalStreamMaximumFrameIntervalUs();
774+
if (_realtimeStreamMode) {
775+
uint64_t minimumFpsIntervalUs = (uint64_t)llround(1000000.0 / (double)XCWMinimumLocalStreamFrameRate);
776+
return MAX(XCWRealtimeMaximumFrameIntervalUs(), minimumFpsIntervalUs);
777+
}
778+
return XCWLocalStreamMaximumFrameIntervalUs();
779+
}
780+
781+
- (uint64_t)activeFrameIntervalUsLocked {
782+
if (_encoderMode == XCWVideoEncoderModeH264Software) {
783+
return _softwareFrameIntervalUs > 0 ? _softwareFrameIntervalUs : [self initialSoftwareFrameIntervalUsLocked];
784+
}
785+
if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) {
786+
return _hardwareFrameIntervalUs > 0 ? _hardwareFrameIntervalUs : [self initialHardwareFrameIntervalUsLocked];
787+
}
788+
int32_t expectedFrameRate = MAX(1, [self expectedFrameRateLocked]);
789+
return (uint64_t)llround(1000000.0 / (double)expectedFrameRate);
719790
}
720791

721792
- (int32_t)expectedFrameRateLocked {
@@ -807,7 +878,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs {
807878
}
808879

809880
- (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs {
810-
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_lowLatencyMode || latencyUs == 0) {
881+
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || latencyUs == 0) {
811882
return;
812883
}
813884
if (_hardwareFrameIntervalUs == 0) {
@@ -1066,6 +1137,12 @@ - (void)invalidateCompressionSessionLocked {
10661137
_inFlightFrameCount = 0;
10671138
_lastSoftwareSubmissionUs = 0;
10681139
_lastHardwareSubmissionUs = 0;
1140+
_latestEncodeLatencyUs = 0;
1141+
_averageEncodeLatencyUs = 0;
1142+
_peakEncodeLatencyUs = 0;
1143+
_consecutiveOverBudgetFrameCount = 0;
1144+
_consecutiveStrainedFrameCount = 0;
1145+
_wasOverloaded = NO;
10691146
_hardwareAccelerated = NO;
10701147
_selectedEncoderID = nil;
10711148
_scalingActive = NO;
@@ -1077,11 +1154,24 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix
10771154
targetHeight:(int32_t)targetHeight {
10781155
int32_t sourceWidth = (int32_t)CVPixelBufferGetWidth(pixelBuffer);
10791156
int32_t sourceHeight = (int32_t)CVPixelBufferGetHeight(pixelBuffer);
1157+
OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
10801158
BOOL shouldCopyStableRealtimeBuffer = _realtimeStreamMode || _lowLatencyMode;
10811159
if (sourceWidth == targetWidth && sourceHeight == targetHeight && !shouldCopyStableRealtimeBuffer) {
10821160
CVPixelBufferRetain(pixelBuffer);
10831161
return pixelBuffer;
10841162
}
1163+
if ([self shouldUseSoftwareScalerForSourceWidth:sourceWidth
1164+
sourceHeight:sourceHeight
1165+
targetWidth:targetWidth
1166+
targetHeight:targetHeight
1167+
pixelFormat:sourcePixelFormat]) {
1168+
CVPixelBufferRef scaledPixelBuffer = [self copySoftwareScaledPixelBuffer:pixelBuffer
1169+
targetWidth:targetWidth
1170+
targetHeight:targetHeight];
1171+
if (scaledPixelBuffer != NULL) {
1172+
return scaledPixelBuffer;
1173+
}
1174+
}
10851175

10861176
if (_pixelTransferSession == NULL) {
10871177
OSStatus sessionStatus = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_pixelTransferSession);
@@ -1105,7 +1195,6 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix
11051195
kVTScalingMode_Normal);
11061196
}
11071197

1108-
OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
11091198
CVPixelBufferRef scaledPixelBuffer = [self copyPixelBufferFromScalingPoolWithWidth:targetWidth
11101199
height:targetHeight
11111200
pixelFormat:sourcePixelFormat];
@@ -1130,6 +1219,20 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix
11301219
return scaledPixelBuffer;
11311220
}
11321221

1222+
- (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth
1223+
sourceHeight:(int32_t)sourceHeight
1224+
targetWidth:(int32_t)targetWidth
1225+
targetHeight:(int32_t)targetHeight
1226+
pixelFormat:(OSType)pixelFormat {
1227+
if (!_realtimeStreamMode || !XCWPixelFormatSupportsSoftwareScaling(pixelFormat)) {
1228+
return NO;
1229+
}
1230+
if (sourceWidth == targetWidth && sourceHeight == targetHeight) {
1231+
return NO;
1232+
}
1233+
return _encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware;
1234+
}
1235+
11331236
- (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer
11341237
targetWidth:(int32_t)targetWidth
11351238
targetHeight:(int32_t)targetHeight {
@@ -1268,6 +1371,34 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer
12681371
if (submittedAtUs > 0) {
12691372
uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0);
12701373
_latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0;
1374+
_peakEncodeLatencyUs = MAX(_peakEncodeLatencyUs, _latestEncodeLatencyUs);
1375+
_averageEncodeLatencyUs = _averageEncodeLatencyUs <= 0.0
1376+
? (double)_latestEncodeLatencyUs
1377+
: (_averageEncodeLatencyUs * (1.0 - XCWEncoderLatencyEWMAAlpha)) + ((double)_latestEncodeLatencyUs * XCWEncoderLatencyEWMAAlpha);
1378+
uint64_t encoderBudgetUs = [self activeFrameIntervalUsLocked];
1379+
double averageLoadPercent = encoderBudgetUs > 0
1380+
? (_averageEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
1381+
: 0.0;
1382+
double latestLoadPercent = encoderBudgetUs > 0
1383+
? ((double)_latestEncodeLatencyUs * 100.0) / (double)encoderBudgetUs
1384+
: 0.0;
1385+
if (encoderBudgetUs > 0 && _latestEncodeLatencyUs > encoderBudgetUs) {
1386+
_overBudgetFrameCount += 1;
1387+
_consecutiveOverBudgetFrameCount += 1;
1388+
} else {
1389+
_consecutiveOverBudgetFrameCount = 0;
1390+
}
1391+
if (latestLoadPercent >= XCWEncoderStrainedLoadPercent) {
1392+
_consecutiveStrainedFrameCount += 1;
1393+
} else {
1394+
_consecutiveStrainedFrameCount = 0;
1395+
}
1396+
BOOL overloaded = averageLoadPercent >= XCWEncoderOverloadedLoadPercent ||
1397+
_consecutiveOverBudgetFrameCount >= XCWEncoderConsecutiveOverBudgetFrameThreshold;
1398+
if (overloaded && !_wasOverloaded) {
1399+
_overloadEventCount += 1;
1400+
}
1401+
_wasOverloaded = overloaded;
12711402
[self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs];
12721403
[self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs];
12731404
}

‎client/src/api/types.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,26 @@
1+
export interface EncoderStats {
2+
averageEncodeLatencyUs?: number;
3+
averageEncoderLoadPercent?: number;
4+
consecutiveOverBudgetFrames?: number;
5+
encoderBudgetUs?: number;
6+
encoderLoadPercent?: number;
7+
encoderMode?: string;
8+
hardwareAccelerated?: boolean;
9+
latestEncodeLatencyUs?: number;
10+
overloadEvents?: number;
11+
overloaded?: boolean;
12+
overloadReason?: string;
13+
overloadState?: "nominal" | "strained" | "overloaded" | string;
14+
peakEncodeLatencyUs?: number;
15+
selectedEncoderId?: string | null;
16+
}
17+
118
export interface PrivateDisplayInfo {
219
displayReady: boolean;
320
displayStatus: string;
421
displayWidth: number;
522
displayHeight: number;
23+
encoder?: EncoderStats;
624
frameSequence: number;
725
rotationQuarterTurns?: number;
826
}

‎client/src/app/AppShell.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1669,6 +1669,7 @@ export function AppShell({
16691669
debugPanel={
16701670
debugVisible ? (
16711671
<DebugPanel
1672+
encoder={selectedSimulator.privateDisplay?.encoder}
16721673
fps={fps}
16731674
inline
16741675
onClose={() => setDebugVisible(false)}
@@ -1827,6 +1828,12 @@ function normalizeStreamQuality(
18271828
if (normalized === "economy" || normalized === "ci-software") {
18281829
return "economy";
18291830
}
1831+
if (normalized === "low") {
1832+
return "low";
1833+
}
1834+
if (normalized === "tiny") {
1835+
return "tiny";
1836+
}
18301837
return fallback;
18311838
}
18321839

‎client/src/features/simulators/SimulatorMenu.tsx‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -237,9 +237,11 @@ const STREAM_QUALITY_OPTIONS: Array<{
237237
label: string;
238238
value: StreamQualityPreset;
239239
}> = [
240-
{ label: "Quality", value: "quality" },
241-
{ label: "Balanced", value: "balanced" },
242-
{ label: "Economy", value: "economy" },
240+
{ label: "Full", value: "quality" },
241+
{ label: "1280", value: "balanced" },
242+
{ label: "1080", value: "economy" },
243+
{ label: "720", value: "low" },
244+
{ label: "540", value: "tiny" },
243245
];
244246

245247
function MenuIcon() {

‎client/src/features/stream/streamTypes.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@ export type StreamQualityPreset =
1414
| "ci-software"
1515
| "economy"
1616
| "fast"
17+
| "low"
1718
| "quality"
18-
| "smooth";
19+
| "smooth"
20+
| "tiny";
1921

2022
export interface StreamConfig {
2123
encoder: StreamEncoder;

‎client/src/features/toolbar/DebugPanel.tsx‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1+
import type { EncoderStats } from "../../api/types";
12
import type {
23
StreamRuntimeInfo,
34
StreamStats,
45
StreamStatus,
56
} from "../stream/streamTypes";
67

78
interface DebugPanelProps {
9+
encoder?: EncoderStats | null;
810
fps: number;
911
inline?: boolean;
1012
onClose?: () => void;
@@ -27,6 +29,20 @@ function formatMs(value: number): string {
2729
return `${value.toFixed(1)} ms`;
2830
}
2931

32+
function formatUsAsMs(value: number | undefined): string {
33+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
34+
return "—";
35+
}
36+
return `${(value / 1000).toFixed(1)} ms`;
37+
}
38+
39+
function formatPercent(value: number | undefined): string {
40+
if (typeof value !== "number" || !Number.isFinite(value)) {
41+
return "—";
42+
}
43+
return `${value.toFixed(0)}%`;
44+
}
45+
3046
function formatResolution(stats: StreamStats): string {
3147
if (!stats.width || !stats.height) {
3248
return "—";
@@ -35,6 +51,7 @@ function formatResolution(stats: StreamStats): string {
3551
}
3652

3753
export function DebugPanel({
54+
encoder,
3855
fps,
3956
inline = false,
4057
onClose,
@@ -60,6 +77,26 @@ export function DebugPanel({
6077
{ label: "Frame Gap", value: formatMs(stats.latestFrameGapMs) },
6178
{ label: "Path", value: runtimeInfo.streamBackend },
6279
];
80+
if (encoder) {
81+
rows.push(
82+
{ label: "Encoder", value: encoder.encoderMode ?? "—" },
83+
{ label: "Encoder State", value: encoder.overloadState ?? "—" },
84+
{
85+
label: "Encoder Load",
86+
value: formatPercent(encoder.averageEncoderLoadPercent),
87+
},
88+
{
89+
label: "Encode Latency",
90+
value: formatUsAsMs(encoder.averageEncodeLatencyUs),
91+
},
92+
{ label: "Encode Budget", value: formatUsAsMs(encoder.encoderBudgetUs) },
93+
{ label: "Encoder Reason", value: encoder.overloadReason ?? "—" },
94+
{
95+
label: "Overload Events",
96+
value: String(encoder.overloadEvents ?? 0),
97+
},
98+
);
99+
}
63100

64101
return (
65102
<section

0 commit comments

Comments
 (0)