Improve Studio stream controls and rotation sync

Author: DjDeveloperr

README.md

@@ -36,11 +36,12 @@ view inside the editor.
 ## Features
 
 - Local simulator video stream over browser-native WebRTC H.264
-- Full simulator control & inspection using private accessibility APIs
+- Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI
+- Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents
 - CoreSimulator chrome asset rendering for device bezels
 - NativeScript, React Native, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live
 - `simdeck/test` for fast JS/TS app tests that can query accessibility state and drive simulator controls.
-- SimDeck Studio for automatic PR deployments to on-demand simulators
+- SimDeck Studio for sharing Simulator streams & automatic PR deployments to on-demand simulators
 
 ## Documentation
 
@@ -59,33 +60,26 @@ To focus a specific simulator by name or UDID, pass it as the only argument:
 simdeck "iPhone 17 Pro Max"
 ```
 
-Use `simdeck ui --open` or `simdeck daemon start` when you want a reusable background daemon instead.
-The no-subcommand lifecycle shortcuts are `simdeck -d` for detached start, `simdeck -k` to kill the background daemon, and `simdeck -r` to restart it.
-The served loopback browser UI receives the generated API access token automatically. LAN browsers pair with the printed code before receiving the API cookie.
+`simdeck -d` for detached start, `simdeck -k` to kill the background daemon, and `simdeck -r` to restart it.
+
+The served loopback browser UI receives the generated API access token automatically.
+LAN clients should pair with the printed code before receiving the API cookie.
 
 SimDeck Studio providers run the daemon on loopback and use
 `scripts/studio-provider-bridge.mjs` for outbound control-plane communication
 with Studio. Studio hosts the browser UI and proxies SimDeck REST requests over
 that bridge while WebRTC media still negotiates directly between the browser and
 runner through ICE.
 
-Expose a local simulator through Studio with one command:
+Expose a local simulator through SimDeck Studio with one command:
 
 ```sh
 simdeck studio expose "iPhone 17 Pro"
 ```
 
 The command starts or reuses the local daemon, creates an ephemeral Studio
 session, prints a unique `https://simdeck.djdev.me/simulator/...` URL, and keeps
-the outbound bridge alive until you press Ctrl-C. It uses software H.264 by
-default with realtime stream settings for remote viewing, and prints the active
-codec/profile when it starts. Studio defaults to the `smooth` stream quality
-profile (`1170` longest edge, dynamic up to `60` fps). Use
-`--stream-quality quality|balanced|fast|smooth|economy|ci-software` to override it,
-or pass `--video-codec hardware` when a dedicated hardware encoder is preferable.
-The remote viewer renders live video with the browser's native video element;
-the canvas is only used for input geometry. Remote viewers can choose 15, 30,
-or 60 fps in the browser stream menu.
+the outbound bridge alive until you press Ctrl-C.
 
 CLI commands automatically use the same warm daemon:
 
@@ -117,14 +111,6 @@ more important than full-resolution smoothness:
 simdeck daemon start --video-codec software --low-latency
 ```
 
-Local browser streams default to realtime WebRTC delivery with the `quality`
-profile on VideoToolbox H.264: full resolution, 120 fps, and a high bitrate floor. On
-high-refresh local displays, raise the local stream target explicitly:
-
-```sh
-simdeck daemon restart --local-stream-fps 240
-```
-
 Restart the CoreSimulator service layer when `simctl` reports a stale service
 version or the live display gets stuck before the first frame:
 
@@ -255,7 +241,7 @@ React Fiber commits.
 
 ## VS Code
 
-Install the `nativescript.simdeck` extension from the VS Code Marketplace, then
+Install the `nativescript.simdeck-vscode` extension from the VS Code Marketplace, then
 run `SimDeck: Open Simulator View` from the Command Palette. The extension
 opens the simulator inside a VS Code panel and auto-starts the local daemon
 when it is not already reachable.

cli/DFPrivateSimulatorDisplayBridge.h

@@ -38,6 +38,7 @@ NS_SWIFT_NAME(PrivateSimulatorDisplayBridge)
 @property (nonatomic, readonly, getter=isDisplayReady) BOOL displayReady;
 @property (nonatomic, readonly) NSString *displayStatus;
 @property (nonatomic, readonly) CGSize displaySize;
+@property (nonatomic, readonly) NSInteger rotationQuarterTurns;
 
 - (nullable CVPixelBufferRef)copyPixelBuffer CF_RETURNS_RETAINED;
 

cli/DFPrivateSimulatorDisplayBridge.m

@@ -1455,6 +1455,32 @@ static double DFNormalizedDegrees(double value) {
     return normalized;
 }
 
+static NSInteger DFRotationQuarterTurnsForDegrees(double degrees) {
+    NSInteger turns = (NSInteger)llround(DFNormalizedDegrees(degrees) / 90.0) % 4;
+    if (turns < 0) {
+        turns += 4;
+    }
+    return turns;
+}
+
+static void DFReconcileRotationWithDisplaySize(double *rotationDegrees, CGSize displaySize) {
+    if (rotationDegrees == NULL || displaySize.width <= 0.0 || displaySize.height <= 0.0) {
+        return;
+    }
+
+    double aspectDelta = fabs(displaySize.width - displaySize.height);
+    if (aspectDelta < 1.0) {
+        return;
+    }
+
+    NSInteger currentTurns = DFRotationQuarterTurnsForDegrees(*rotationDegrees);
+    BOOL displayIsLandscape = displaySize.width > displaySize.height;
+    BOOL rotationIsLandscape = (currentTurns % 2) != 0;
+    if (displayIsLandscape != rotationIsLandscape) {
+        *rotationDegrees = displayIsLandscape ? 90.0 : 0.0;
+    }
+}
+
 static NSArray<NSString *> * DFInterestingSelectorsForObject(id object) {
     if (object == nil) {
         return @[];
@@ -2758,6 +2784,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid
             size_t width = CVPixelBufferGetWidth(pixelBuffer);
             size_t height = CVPixelBufferGetHeight(pixelBuffer);
             strongSelf->_displayPixelSize = CGSizeMake((CGFloat)width, (CGFloat)height);
+            DFReconcileRotationWithDisplaySize(&strongSelf->_deviceRotationDegrees, strongSelf->_displayPixelSize);
             [strongSelf notifyDelegateOfFrame:pixelBuffer];
             DFRunOnMainAsync(^{
                 if (strongSelf->_headlessHostWindow != nil) {
@@ -3435,6 +3462,21 @@ - (CGSize)displaySize {
     return size;
 }
 
+- (NSInteger)rotationQuarterTurns {
+    __block NSInteger turns = 0;
+    dispatch_block_t work = ^{
+        turns = DFRotationQuarterTurnsForDegrees(self->_deviceRotationDegrees);
+    };
+
+    if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) {
+        work();
+    } else {
+        dispatch_sync(_callbackQueue, work);
+    }
+
+    return turns;
+}
+
 - (BOOL)sendTouchAtNormalizedX:(double)normalizedX
                    normalizedY:(double)normalizedY
                          phase:(DFPrivateSimulatorTouchPhase)phase

cli/XCWPrivateSimulatorSession.h

@@ -23,6 +23,7 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData,
 @property (nonatomic, readonly, getter=isDisplayReady) BOOL displayReady;
 @property (nonatomic, copy, readonly) NSString *displayStatus;
 @property (nonatomic, readonly) CGSize displaySize;
+@property (nonatomic, readonly) NSInteger rotationQuarterTurns;
 @property (nonatomic, readonly) NSUInteger frameSequence;
 
 - (BOOL)waitUntilReadyWithTimeout:(NSTimeInterval)timeout;

cli/XCWPrivateSimulatorSession.m

@@ -229,6 +229,10 @@ - (CGSize)displaySize {
     return size;
 }
 
+- (NSInteger)rotationQuarterTurns {
+    return _displayBridge.rotationQuarterTurns;
+}
+
 - (NSUInteger)frameSequence {
     __block NSUInteger sequence = 0;
     dispatch_sync(_stateQueue, ^{

cli/native/XCWNativeBridge.h

@@ -74,6 +74,7 @@ void xcw_native_session_request_refresh(void * _Nonnull handle);
 void xcw_native_session_request_keyframe(void * _Nonnull handle);
 void xcw_native_session_reconfigure_video_encoder(void * _Nonnull handle);
 char * _Nullable xcw_native_session_video_encoder_stats(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
+int32_t xcw_native_session_rotation_quarter_turns(void * _Nonnull handle);
 bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message);
 bool xcw_native_session_send_multitouch(void * _Nonnull handle, double x1, double y1, double x2, double y2, const char * _Nonnull phase, char * _Nullable * _Nullable error_message);
 bool xcw_native_session_send_key(void * _Nonnull handle, uint16_t key_code, uint32_t modifiers, char * _Nullable * _Nullable error_message);

cli/native/XCWNativeBridge.m

@@ -702,6 +702,14 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) {
     }
 }
 
+int32_t xcw_native_session_rotation_quarter_turns(void *handle) {
+    @autoreleasepool {
+        NSInteger turns = [XCWNativeSessionFromHandle(handle) rotationQuarterTurns];
+        NSInteger normalized = ((turns % 4) + 4) % 4;
+        return (int32_t)normalized;
+    }
+}
+
 bool xcw_native_session_send_touch(void *handle, double x, double y, const char *phase, char **error_message) {
     @autoreleasepool {
         NSError *error = nil;

cli/native/XCWNativeSession.h

@@ -16,6 +16,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (void)requestKeyFrame;
 - (void)reconfigureVideoEncoder;
 - (NSDictionary *)videoEncoderStats;
+- (NSInteger)rotationQuarterTurns;
 - (BOOL)sendTouchAtX:(double)x
                    y:(double)y
                phase:(NSString *)phase

cli/native/XCWNativeSession.m

@@ -106,6 +106,10 @@ - (NSDictionary *)videoEncoderStats {
     return [self.session videoEncoderStats];
 }
 
+- (NSInteger)rotationQuarterTurns {
+    return self.session.rotationQuarterTurns;
+}
+
 - (BOOL)sendTouchAtX:(double)x
                    y:(double)y
                phase:(NSString *)phase

client/src/api/types.ts

@@ -4,6 +4,7 @@ export interface PrivateDisplayInfo {
   displayWidth: number;
   displayHeight: number;
   frameSequence: number;
+  rotationQuarterTurns?: number;
 }
 
 export interface SimulatorMetadata {