Stabilize codec switching and add low latency mode

Author: DjDeveloperr

README.md

@@ -83,6 +83,13 @@ simdeck daemon stop
 
 `simdeck daemon` manages the normal per-project warm process.
 
+Use software H.264's low-latency profile on slower runners where freshness is
+more important than full-resolution smoothness:
+
+```sh
+simdeck daemon start --video-codec h264-software --low-latency
+```
+
 Restart the CoreSimulator service layer when `simctl` reports a stale service
 version or the live display gets stuck before the first frame:
 

cli/XCWH264Encoder.h

@@ -19,6 +19,7 @@ typedef void (^XCWH264EncoderOutputHandler)(NSData *sampleData,
 
 - (void)encodePixelBuffer:(CVPixelBufferRef)pixelBuffer;
 - (void)requestKeyFrame;
+- (void)reconfigureWithCurrentEnvironment;
 - (NSDictionary *)statsRepresentation;
 - (void)invalidate;
 

cli/XCWH264Encoder.m

@@ -8,18 +8,27 @@
 
 static const int32_t XCWMaximumEncodedDimension = 1920;
 static const int32_t XCWMaximumSoftwareEncodedDimension = 1600;
+static const int32_t XCWMaximumLowLatencySoftwareEncodedDimension = 1472;
 static const int32_t XCWTargetRealTimeFrameRate = 60;
 static const int32_t XCWTargetSoftwareFrameRate = 60;
+static const int32_t XCWTargetLowLatencySoftwareFrameRate = 45;
 static const NSUInteger XCWMaximumInFlightFrames = 2;
 static const int32_t XCWMinimumAverageBitRate = 18000000;
 static const int32_t XCWMinimumSoftwareAverageBitRate = 3000000;
+static const int32_t XCWMinimumLowLatencySoftwareAverageBitRate = 2500000;
 static const int64_t XCWBitsPerPixelBudget = 10;
 static const int64_t XCWSoftwareBitsPerPixelBudget = 6;
+static const int64_t XCWLowLatencySoftwareBitsPerPixelBudget = 4;
 static const uint64_t XCWSoftwareMinimumFrameIntervalUs = 16667;
 static const uint64_t XCWSoftwareInitialFrameIntervalUs = 16667;
 static const uint64_t XCWSoftwareMaximumFrameIntervalUs = 50000;
 static const uint64_t XCWSoftwareFrameIntervalStepUs = 5556;
 static const NSUInteger XCWSoftwareHealthyFrameWindow = 4;
+static const uint64_t XCWLowLatencySoftwareMinimumFrameIntervalUs = 22222;
+static const uint64_t XCWLowLatencySoftwareInitialFrameIntervalUs = 22222;
+static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 100000;
+static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111;
+static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8;
 
 typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) {
     XCWVideoEncoderModeHEVCHardware,
@@ -39,6 +48,15 @@ static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) {
     return XCWVideoEncoderModeHEVCHardware;
 }
 
+static BOOL XCWLowLatencyModeFromEnvironment(void) {
+    const char *rawValue = getenv("SIMDECK_LOW_LATENCY");
+    NSString *value = rawValue != NULL ? [[[NSString alloc] initWithUTF8String:rawValue] lowercaseString] : @"";
+    return [value isEqualToString:@"1"] ||
+        [value isEqualToString:@"true"] ||
+        [value isEqualToString:@"yes"] ||
+        [value isEqualToString:@"on"];
+}
+
 static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) {
     switch (mode) {
         case XCWVideoEncoderModeH264Hardware:
@@ -242,14 +260,17 @@ static int32_t XCWRoundToEvenDimension(double value) {
     return rounded;
 }
 
-static CGSize XCWScaledDimensionsForSourceSize(int32_t width, int32_t height, XCWVideoEncoderMode mode) {
+static CGSize XCWScaledDimensionsForSourceSize(int32_t width, int32_t height, XCWVideoEncoderMode mode, BOOL lowLatencyMode) {
     if (width <= 0 || height <= 0) {
         return CGSizeZero;
     }
 
-    int32_t maximumDimension = mode == XCWVideoEncoderModeH264Software
-        ? XCWMaximumSoftwareEncodedDimension
-        : XCWMaximumEncodedDimension;
+    int32_t maximumDimension = XCWMaximumEncodedDimension;
+    if (mode == XCWVideoEncoderModeH264Software) {
+        maximumDimension = lowLatencyMode
+            ? XCWMaximumLowLatencySoftwareEncodedDimension
+            : XCWMaximumSoftwareEncodedDimension;
+    }
     int32_t longestEdge = MAX(width, height);
     if (longestEdge <= maximumDimension) {
         return CGSizeMake(width, height);
@@ -260,13 +281,17 @@ static CGSize XCWScaledDimensionsForSourceSize(int32_t width, int32_t height, XC
                       XCWRoundToEvenDimension(height * scale));
 }
 
-static int32_t XCWAverageBitRateForDimensions(int32_t width, int32_t height, XCWVideoEncoderMode mode) {
-    int64_t bitsPerPixelBudget = mode == XCWVideoEncoderModeH264Software
-        ? XCWSoftwareBitsPerPixelBudget
-        : XCWBitsPerPixelBudget;
-    int64_t minimumAverageBitRate = mode == XCWVideoEncoderModeH264Software
-        ? XCWMinimumSoftwareAverageBitRate
-        : XCWMinimumAverageBitRate;
+static int32_t XCWAverageBitRateForDimensions(int32_t width, int32_t height, XCWVideoEncoderMode mode, BOOL lowLatencyMode) {
+    int64_t bitsPerPixelBudget = XCWBitsPerPixelBudget;
+    int64_t minimumAverageBitRate = XCWMinimumAverageBitRate;
+    if (mode == XCWVideoEncoderModeH264Software) {
+        bitsPerPixelBudget = lowLatencyMode
+            ? XCWLowLatencySoftwareBitsPerPixelBudget
+            : XCWSoftwareBitsPerPixelBudget;
+        minimumAverageBitRate = lowLatencyMode
+            ? XCWMinimumLowLatencySoftwareAverageBitRate
+            : XCWMinimumSoftwareAverageBitRate;
+    }
     int64_t computedBitRate = (int64_t)width * (int64_t)height * bitsPerPixelBudget;
     if (computedBitRate < minimumAverageBitRate) {
         computedBitRate = minimumAverageBitRate;
@@ -364,6 +389,7 @@ @implementation XCWH264Encoder {
     CVPixelBufferRef _scaledPixelBuffer;
     OSType _scaledPixelFormat;
     XCWVideoEncoderMode _encoderMode;
+    BOOL _lowLatencyMode;
     CMVideoCodecType _codecType;
     BOOL _hardwareAccelerated;
     NSUInteger _inputFrameCount;
@@ -397,8 +423,9 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler
     _pendingLock = OS_UNFAIR_LOCK_INIT;
     _needsKeyFrame = YES;
     _encoderMode = XCWVideoEncoderModeFromEnvironment();
+    _lowLatencyMode = XCWLowLatencyModeFromEnvironment();
     _codecType = XCWVideoCodecTypeForMode(_encoderMode);
-    _softwareFrameIntervalUs = XCWSoftwareInitialFrameIntervalUs;
+    _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
     return self;
 }
 
@@ -441,6 +468,28 @@ - (void)requestKeyFrame {
     });
 }
 
+- (void)reconfigureWithCurrentEnvironment {
+    os_unfair_lock_lock(&_pendingLock);
+    if (_pendingPixelBuffer != NULL) {
+        CVPixelBufferRelease(_pendingPixelBuffer);
+        _pendingPixelBuffer = NULL;
+    }
+    _drainScheduled = NO;
+    os_unfair_lock_unlock(&_pendingLock);
+
+    dispatch_sync(_queue, ^{
+        [self invalidateCompressionSessionLocked];
+        self->_encoderMode = XCWVideoEncoderModeFromEnvironment();
+        self->_lowLatencyMode = XCWLowLatencyModeFromEnvironment();
+        self->_codecType = XCWVideoCodecTypeForMode(self->_encoderMode);
+        self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
+        self->_lastSoftwareSubmissionUs = 0;
+        self->_softwarePacedFrameCount = 0;
+        self->_softwareHealthyFrameCount = 0;
+        self->_needsKeyFrame = YES;
+    });
+}
+
 - (NSDictionary *)statsRepresentation {
     __block NSUInteger inputFrameCount = 0;
     __block NSUInteger pendingReplacementCount = 0;
@@ -466,6 +515,7 @@ - (NSDictionary *)statsRepresentation {
             @"softwarePacedFrames": @(self->_softwarePacedFrameCount),
             @"transportCodec": XCWCodecName(self->_codecType),
             @"encoderMode": XCWVideoEncoderModeName(self->_encoderMode),
+            @"lowLatencyMode": @(self->_lowLatencyMode),
             @"encoderId": XCWVideoEncoderIDForMode(self->_encoderMode) ?: @"automatic",
             @"hardwareAccelerated": @(self->_hardwareAccelerated),
             @"lastSessionStatus": @(self->_lastSessionStatus),
@@ -492,12 +542,43 @@ - (void)invalidate {
     os_unfair_lock_unlock(&_pendingLock);
 }
 
+- (NSUInteger)maximumInFlightFrameCountLocked {
+    return (_encoderMode == XCWVideoEncoderModeH264Software && _lowLatencyMode) ? 1 : XCWMaximumInFlightFrames;
+}
+
+- (uint64_t)minimumSoftwareFrameIntervalUsLocked {
+    return _lowLatencyMode ? XCWLowLatencySoftwareMinimumFrameIntervalUs : XCWSoftwareMinimumFrameIntervalUs;
+}
+
+- (uint64_t)initialSoftwareFrameIntervalUsLocked {
+    return _lowLatencyMode ? XCWLowLatencySoftwareInitialFrameIntervalUs : XCWSoftwareInitialFrameIntervalUs;
+}
+
+- (uint64_t)maximumSoftwareFrameIntervalUsLocked {
+    return _lowLatencyMode ? XCWLowLatencySoftwareMaximumFrameIntervalUs : XCWSoftwareMaximumFrameIntervalUs;
+}
+
+- (uint64_t)softwareFrameIntervalStepUsLocked {
+    return _lowLatencyMode ? XCWLowLatencySoftwareFrameIntervalStepUs : XCWSoftwareFrameIntervalStepUs;
+}
+
+- (NSUInteger)softwareHealthyFrameWindowLocked {
+    return _lowLatencyMode ? XCWLowLatencySoftwareHealthyFrameWindow : XCWSoftwareHealthyFrameWindow;
+}
+
+- (int32_t)expectedFrameRateLocked {
+    if (_encoderMode == XCWVideoEncoderModeH264Software) {
+        return _lowLatencyMode ? XCWTargetLowLatencySoftwareFrameRate : XCWTargetSoftwareFrameRate;
+    }
+    return XCWTargetRealTimeFrameRate;
+}
+
 - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs {
     if (_encoderMode != XCWVideoEncoderModeH264Software || _needsKeyFrame) {
         return NO;
     }
     if (_softwareFrameIntervalUs == 0) {
-        _softwareFrameIntervalUs = XCWSoftwareInitialFrameIntervalUs;
+        _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
     }
     if (_lastSoftwareSubmissionUs == 0) {
         return NO;
@@ -515,29 +596,32 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs {
         return;
     }
     if (_softwareFrameIntervalUs == 0) {
-        _softwareFrameIntervalUs = XCWSoftwareInitialFrameIntervalUs;
+        _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
     }
 
-    uint64_t highLatencyBudgetUs = (_softwareFrameIntervalUs * 3) / 2;
+    uint64_t highLatencyBudgetUs = _lowLatencyMode ? _softwareFrameIntervalUs : ((_softwareFrameIntervalUs * 3) / 2);
     if (latencyUs > highLatencyBudgetUs) {
-        uint64_t nextIntervalUs = _softwareFrameIntervalUs + XCWSoftwareFrameIntervalStepUs;
-        uint64_t latencyBoundIntervalUs = latencyUs + XCWSoftwareFrameIntervalStepUs;
+        uint64_t stepUs = [self softwareFrameIntervalStepUsLocked];
+        uint64_t nextIntervalUs = _softwareFrameIntervalUs + stepUs;
+        uint64_t latencyBoundIntervalUs = latencyUs + stepUs;
         if (nextIntervalUs < latencyBoundIntervalUs) {
             nextIntervalUs = latencyBoundIntervalUs;
         }
-        _softwareFrameIntervalUs = MIN(nextIntervalUs, XCWSoftwareMaximumFrameIntervalUs);
+        _softwareFrameIntervalUs = MIN(nextIntervalUs, [self maximumSoftwareFrameIntervalUsLocked]);
         _softwareHealthyFrameCount = 0;
         return;
     }
 
     if (latencyUs < _softwareFrameIntervalUs &&
-        _softwareFrameIntervalUs > XCWSoftwareMinimumFrameIntervalUs) {
+        _softwareFrameIntervalUs > [self minimumSoftwareFrameIntervalUsLocked]) {
         _softwareHealthyFrameCount += 1;
-        if (_softwareHealthyFrameCount >= XCWSoftwareHealthyFrameWindow) {
-            uint64_t nextIntervalUs = _softwareFrameIntervalUs > XCWSoftwareFrameIntervalStepUs
-                ? _softwareFrameIntervalUs - XCWSoftwareFrameIntervalStepUs
-                : XCWSoftwareMinimumFrameIntervalUs;
-            _softwareFrameIntervalUs = MAX(nextIntervalUs, XCWSoftwareMinimumFrameIntervalUs);
+        if (_softwareHealthyFrameCount >= [self softwareHealthyFrameWindowLocked]) {
+            uint64_t stepUs = [self softwareFrameIntervalStepUsLocked];
+            uint64_t minimumIntervalUs = [self minimumSoftwareFrameIntervalUsLocked];
+            uint64_t nextIntervalUs = _softwareFrameIntervalUs > stepUs
+                ? _softwareFrameIntervalUs - stepUs
+                : minimumIntervalUs;
+            _softwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs);
             _softwareHealthyFrameCount = 0;
         }
         return;
@@ -548,7 +632,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs {
 
 - (void)drainPendingFramesLocked {
     while (YES) {
-        if (_inFlightFrameCount >= XCWMaximumInFlightFrames) {
+        if (_inFlightFrameCount >= [self maximumInFlightFrameCountLocked]) {
             _drainScheduled = NO;
             return;
         }
@@ -576,7 +660,7 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer {
         return NO;
     }
 
-    CGSize targetSize = XCWScaledDimensionsForSourceSize(sourceWidth, sourceHeight, _encoderMode);
+    CGSize targetSize = XCWScaledDimensionsForSourceSize(sourceWidth, sourceHeight, _encoderMode, _lowLatencyMode);
     int32_t targetWidth = (int32_t)targetSize.width;
     int32_t targetHeight = (int32_t)targetSize.height;
     if (targetWidth <= 0 || targetHeight <= 0) {
@@ -683,10 +767,8 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height
     _timestampOriginUs = 0;
     _needsKeyFrame = YES;
 
-    int expectedFrameRate = _encoderMode == XCWVideoEncoderModeH264Software
-        ? XCWTargetSoftwareFrameRate
-        : XCWTargetRealTimeFrameRate;
-    int averageBitRate = XCWAverageBitRateForDimensions(width, height, _encoderMode);
+    int expectedFrameRate = [self expectedFrameRateLocked];
+    int averageBitRate = XCWAverageBitRateForDimensions(width, height, _encoderMode, _lowLatencyMode);
 
     VTSessionSetProperty(session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
     if (@available(macOS 10.14, *)) {
@@ -713,8 +795,8 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height
         VTSessionSetProperty(session, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel);
     }
     VTSessionSetProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(expectedFrameRate));
-    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(expectedFrameRate * 2));
-    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@2.0);
+    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(_lowLatencyMode ? expectedFrameRate : expectedFrameRate * 2));
+    VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(_lowLatencyMode ? 1.0 : 2.0));
     VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate));
     if (@available(macOS 11.0, *)) {
         VTSessionSetProperty(session,

cli/XCWPrivateSimulatorSession.h

@@ -27,6 +27,7 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData,
 
 - (BOOL)waitUntilReadyWithTimeout:(NSTimeInterval)timeout;
 - (BOOL)waitForFirstEncodedFrameWithTimeout:(NSTimeInterval)timeout;
+- (void)reconfigureVideoEncoder;
 - (void)requestKeyFrameRefresh;
 - (void)requestFrameRefresh;
 - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler;

cli/XCWPrivateSimulatorSession.m

@@ -167,6 +167,19 @@ - (void)requestKeyFrameRefresh {
     [_videoEncoder requestKeyFrame];
 }
 
+- (void)reconfigureVideoEncoder {
+    dispatch_sync(_stateQueue, ^{
+        self->_latestKeyFrameData = nil;
+        self->_latestKeyFrameTimestampUs = 0;
+        self->_latestKeyFrameCodec = nil;
+        self->_latestKeyFrameDecoderConfig = nil;
+        self->_latestKeyFrameDimensions = CGSizeZero;
+        self->_latestKeyFrameSequenceValue = 0;
+    });
+    [_videoEncoder reconfigureWithCurrentEnvironment];
+    [self requestKeyFrameRefresh];
+}
+
 - (void)requestFrameRefresh {
     [self refreshCurrentFrame];
 }

cli/native/XCWNativeBridge.h

@@ -69,6 +69,7 @@ bool xcw_native_input_send_key_event(void * _Nonnull handle, uint16_t key_code,
 void * _Nullable xcw_native_session_create(const char * _Nonnull udid, char * _Nullable * _Nullable error_message);
 void xcw_native_session_destroy(void * _Nullable handle);
 bool xcw_native_session_start(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
+bool xcw_native_session_reconfigure_video_encoder(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
 void xcw_native_session_request_refresh(void * _Nonnull handle);
 void xcw_native_session_request_keyframe(void * _Nonnull handle);
 bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message);

cli/native/XCWNativeBridge.m

@@ -662,6 +662,17 @@ bool xcw_native_session_start(void *handle, char **error_message) {
     }
 }
 
+bool xcw_native_session_reconfigure_video_encoder(void *handle, char **error_message) {
+    @autoreleasepool {
+        NSError *error = nil;
+        BOOL ok = [XCWNativeSessionFromHandle(handle) reconfigureVideoEncoder:&error];
+        if (!ok) {
+            XCWSetErrorMessage(error_message, error);
+        }
+        return ok;
+    }
+}
+
 void xcw_native_session_request_refresh(void *handle) {
     @autoreleasepool {
         [XCWNativeSessionFromHandle(handle) requestRefresh];

cli/native/XCWNativeSession.h

@@ -12,6 +12,7 @@ NS_ASSUME_NONNULL_BEGIN
                                 error:(NSError * _Nullable * _Nullable)error NS_DESIGNATED_INITIALIZER;
 
 - (BOOL)start:(NSError * _Nullable * _Nullable)error;
+- (BOOL)reconfigureVideoEncoder:(NSError * _Nullable * _Nullable)error;
 - (void)requestRefresh;
 - (void)requestKeyFrame;
 - (BOOL)sendTouchAtX:(double)x

cli/native/XCWNativeSession.m

@@ -94,6 +94,21 @@ - (void)requestRefresh {
     [self.session requestFrameRefresh];
 }
 
+- (BOOL)reconfigureVideoEncoder:(NSError * _Nullable __autoreleasing *)error {
+    [self.session reconfigureVideoEncoder];
+    if (![self.session waitForFirstEncodedFrameWithTimeout:2.0]) {
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:@"SimDeck.NativeSession"
+                                         code:3
+                                     userInfo:@{
+                NSLocalizedDescriptionKey: @"Timed out waiting for the first encoded simulator frame after codec reconfiguration.",
+            }];
+        }
+        return NO;
+    }
+    return YES;
+}
+
 - (void)requestKeyFrame {
     [self.session requestKeyFrameRefresh];
 }

client/src/app/AppShell.tsx

@@ -1249,8 +1249,9 @@ export function AppShell() {
     }
     const udid = selectedSimulator.udid;
     setVideoCodec(codec);
-    if (codec === "hevc" && streamTransportMode === "webrtc") {
-      setStreamTransportMode("auto");
+    if (codec === "hevc" && streamTransportMode !== "webtransport") {
+      setStreamTransportMode("webtransport");
+      setStreamSettingsRevision((current) => current + 1);
     }
     void (async () => {
       codecSwitchInFlightRef.current = true;