Skip to content

Commit 8dc07ac

Browse files
DjDeveloperrDjDeveloperr
authored
Add stream control APIs and harden simulator streaming (#14)
* Add farm view, stream controls, and native HID fallback * Remove farm view and related docs * Format WebRTC transport * Fix WebRTC data channel clippy lint
1 parent 1924b01 commit 8dc07ac

9 files changed

Lines changed: 397 additions & 17 deletions

File tree

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ simdeck toggle-appearance <udid>
152152
simdeck pasteboard set <udid> "hello"
153153
simdeck pasteboard get <udid>
154154
simdeck screenshot <udid> --output screen.png
155+
simdeck stream <udid> --frames 120 > stream.h264
155156
simdeck describe <udid>
156157
simdeck describe <udid> --format agent --max-depth 4
157158
simdeck describe <udid> --point 120,240
@@ -179,6 +180,13 @@ simdeck chrome-profile <udid>
179180
simdeck logs <udid> --seconds 30 --limit 200
180181
```
181182

183+
`boot` prefers SimDeck's private CoreSimulator boot path so it can start devices
184+
without launching Simulator.app, then falls back to `xcrun simctl` when private
185+
booting is unavailable.
186+
187+
`stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or
188+
external tools such as `ffplay`.
189+
182190
`describe` uses the project daemon to prefer React Native, NativeScript, or
183191
UIKit in-app inspectors, then falls back to the built-in private CoreSimulator
184192
accessibility bridge. Use `--format agent` or `--format compact-json` for

‎cli/DFPrivateSimulatorDisplayBridge.m‎

Lines changed: 185 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@
3232
typedef uint32_t IndigoHIDEdge;
3333

3434
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, NSEventType type, NSSize displaySize, IndigoHIDEdge edge);
35+
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEvent9Fn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, uint32_t direction, double unused1, double unused2, double widthPoints, double heightPoints);
3536
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardArbitraryFn)(int keyCode, int op);
3637
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event);
3738
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target);
3839
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForHIDArbitraryFn)(uint32_t target, uint32_t page, uint32_t usage, uint32_t operation);
40+
typedef IndigoHIDMessage *(*DFIndigoHIDServiceMessageFn)(void);
3941

4042
#pragma pack(push, 4)
4143
typedef struct {
@@ -91,6 +93,12 @@
9193
static const uint32_t DFIndigoTouchTarget = 0x32;
9294
static const uint8_t DFIndigoEventTypeTouch = 0x02;
9395
static const uint32_t DFIndigoTouchEventKind = 0x0b;
96+
static const uint32_t DFIndigoMouseEventDown = 1;
97+
static const uint32_t DFIndigoMouseEventUp = 2;
98+
static const uint32_t DFIndigoMouseEventDragged = 6;
99+
static const uint32_t DFIndigoMouseDirectionDown = 1;
100+
static const uint32_t DFIndigoMouseDirectionMove = 0;
101+
static const uint32_t DFIndigoMouseDirectionUp = 2;
94102
static const int DFKeyboardDirectionDown = 1;
95103
static const int DFKeyboardDirectionUp = 2;
96104
static const uint32_t DFButtonDirectionDown = 1;
@@ -128,6 +136,39 @@
128136
return @"/Applications/Xcode.app/Contents/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit";
129137
}
130138

139+
static NSInteger DFXcodeMajorVersion(void) {
140+
NSString *developerPath = nil;
141+
const char *developerDir = getenv("DEVELOPER_DIR");
142+
if (developerDir != NULL && developerDir[0] != '\0') {
143+
developerPath = [NSString stringWithUTF8String:developerDir];
144+
} else {
145+
FILE *pipe = popen("/usr/bin/xcode-select -p 2>/dev/null", "r");
146+
if (pipe != NULL) {
147+
char buffer[PATH_MAX] = {0};
148+
if (fgets(buffer, sizeof(buffer), pipe) != NULL) {
149+
developerPath = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]];
150+
}
151+
pclose(pipe);
152+
}
153+
}
154+
if (developerPath.length == 0) {
155+
return 0;
156+
}
157+
158+
NSString *contentsPath = developerPath;
159+
if ([contentsPath.lastPathComponent isEqualToString:@"Developer"]) {
160+
contentsPath = contentsPath.stringByDeletingLastPathComponent;
161+
}
162+
163+
NSDictionary *versionInfo = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"version.plist"]];
164+
NSString *version = versionInfo[@"CFBundleShortVersionString"];
165+
if (version.length == 0) {
166+
NSDictionary *info = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"Info.plist"]];
167+
version = info[@"CFBundleShortVersionString"];
168+
}
169+
return [[version componentsSeparatedByString:@"."].firstObject integerValue];
170+
}
171+
131172
typedef struct {
132173
__unsafe_unretained id unit;
133174
double value;
@@ -187,6 +228,23 @@ static BOOL DFVerboseTouchLoggingEnabled(void) {
187228
return enabled;
188229
}
189230

231+
static BOOL DFShouldUseIndigoMouse9Path(void) {
232+
static BOOL enabled = NO;
233+
static dispatch_once_t onceToken;
234+
dispatch_once(&onceToken, ^{
235+
NSString *override = NSProcessInfo.processInfo.environment[@"SIMDECK_INDIGO_MOUSE_9ARG"];
236+
if (override.length > 0) {
237+
enabled = [override isEqualToString:@"1"] ||
238+
[override caseInsensitiveCompare:@"true"] == NSOrderedSame ||
239+
[override caseInsensitiveCompare:@"yes"] == NSOrderedSame;
240+
return;
241+
}
242+
243+
enabled = DFXcodeMajorVersion() >= 26;
244+
});
245+
return enabled;
246+
}
247+
190248
#pragma mark - SimulatorKit Swift symbol resolver
191249
//
192250
// We call into SimulatorKit's private Swift API by dlsym'ing mangled symbol
@@ -1002,6 +1060,109 @@ static BOOL DFSendHIDMessage(id hidClient, IndigoHIDMessage *message, BOOL freeW
10021060
return YES;
10031061
}
10041062

1063+
static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phase) {
1064+
switch (phase) {
1065+
case DFPrivateSimulatorTouchPhaseBegan:
1066+
return DFIndigoMouseEventDown;
1067+
case DFPrivateSimulatorTouchPhaseMoved:
1068+
return DFIndigoMouseEventDragged;
1069+
case DFPrivateSimulatorTouchPhaseEnded:
1070+
case DFPrivateSimulatorTouchPhaseCancelled:
1071+
return DFIndigoMouseEventUp;
1072+
}
1073+
}
1074+
1075+
static uint32_t DFIndigoMouseDirectionForPhase(DFPrivateSimulatorTouchPhase phase) {
1076+
switch (phase) {
1077+
case DFPrivateSimulatorTouchPhaseBegan:
1078+
return DFIndigoMouseDirectionDown;
1079+
case DFPrivateSimulatorTouchPhaseMoved:
1080+
return DFIndigoMouseDirectionMove;
1081+
case DFPrivateSimulatorTouchPhaseEnded:
1082+
case DFPrivateSimulatorTouchPhaseCancelled:
1083+
return DFIndigoMouseDirectionUp;
1084+
}
1085+
}
1086+
1087+
static IndigoHIDMessage *DFCreateIndigoTouchMessage9(CGPoint normalizedPoint, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) {
1088+
if (!DFShouldUseIndigoMouse9Path()) {
1089+
return NULL;
1090+
}
1091+
DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
1092+
if (mouseMessage == NULL) {
1093+
return NULL;
1094+
}
1095+
1096+
CGPoint ratioPoint = CGPointMake(
1097+
fmax(0.0, fmin(1.0, normalizedPoint.x)),
1098+
fmax(0.0, fmin(1.0, normalizedPoint.y))
1099+
);
1100+
return mouseMessage(&ratioPoint,
1101+
NULL,
1102+
DFIndigoTouchTarget,
1103+
DFIndigoMouseEventTypeForPhase(phase),
1104+
DFIndigoMouseDirectionForPhase(phase),
1105+
1.0,
1106+
1.0,
1107+
displaySize.width,
1108+
displaySize.height);
1109+
}
1110+
1111+
static IndigoHIDMessage *DFCreateIndigoMultiTouchMessage9(CGPoint normalizedPoint1, CGPoint normalizedPoint2, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) {
1112+
if (!DFShouldUseIndigoMouse9Path()) {
1113+
return NULL;
1114+
}
1115+
DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
1116+
if (mouseMessage == NULL) {
1117+
return NULL;
1118+
}
1119+
1120+
CGPoint ratioPoint = CGPointMake(
1121+
fmax(0.0, fmin(1.0, normalizedPoint1.x)),
1122+
fmax(0.0, fmin(1.0, normalizedPoint1.y))
1123+
);
1124+
CGPoint secondRatioPoint = CGPointMake(
1125+
fmax(0.0, fmin(1.0, normalizedPoint2.x)),
1126+
fmax(0.0, fmin(1.0, normalizedPoint2.y))
1127+
);
1128+
return mouseMessage(&ratioPoint,
1129+
&secondRatioPoint,
1130+
DFIndigoTouchTarget,
1131+
DFIndigoMouseEventTypeForPhase(phase),
1132+
DFIndigoMouseDirectionForPhase(phase),
1133+
1.0,
1134+
1.0,
1135+
displaySize.width,
1136+
displaySize.height);
1137+
}
1138+
1139+
static void DFWarmIndigoHIDServices(id hidClient) {
1140+
if (hidClient == nil) {
1141+
return;
1142+
}
1143+
1144+
DFIndigoHIDServiceMessageFn createPointer = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreatePointerService");
1145+
DFIndigoHIDServiceMessageFn createMouse = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreateMouseService");
1146+
NSError *error = nil;
1147+
if (createPointer != NULL) {
1148+
IndigoHIDMessage *message = createPointer();
1149+
if (message != NULL) {
1150+
(void)DFSendHIDMessage(hidClient, message, YES, &error);
1151+
usleep(20 * 1000);
1152+
}
1153+
}
1154+
if (createMouse != NULL) {
1155+
IndigoHIDMessage *message = createMouse();
1156+
if (message != NULL) {
1157+
(void)DFSendHIDMessage(hidClient, message, YES, &error);
1158+
usleep(20 * 1000);
1159+
}
1160+
}
1161+
if (error != nil) {
1162+
DFLog(@"Indigo HID service warm-up reported: %@", error.localizedDescription ?: @"unknown error");
1163+
}
1164+
}
1165+
10051166
static DFIndigoMessage *DFCreateIndigoTouchMessage(CGPoint normalizedPoint, NSSize displaySize, BOOL touchDown, NSError **error) {
10061167
DFIndigoHIDMessageForMouseNSEventFn mouseMessage = (DFIndigoHIDMessageForMouseNSEventFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
10071168
if (mouseMessage == NULL) {
@@ -2378,6 +2539,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid
23782539

23792540
if (_hidClient != nil) {
23802541
DFLog(@"Created private SimulatorKit HID client for %@", udid);
2542+
DFWarmIndigoHIDServices(_hidClient);
23812543
} else {
23822544
DFLog(@"Failed to create private SimulatorKit HID client for %@: %@", udid, hidClientError.localizedDescription ?: @"unknown error");
23832545
}
@@ -3311,14 +3473,18 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX
33113473
phaseLabel = phase == DFPrivateSimulatorTouchPhaseEnded ? @"ended" : @"cancelled";
33123474
break;
33133475
}
3314-
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3315-
DFIndigoMessage *message = DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError);
3476+
IndigoHIDMessage *message = DFCreateIndigoTouchMessage9(CGPointMake(clampedX, clampedY), displaySize, phase);
3477+
BOOL freeWhenDone = YES;
33163478
if (message == NULL) {
3317-
return;
3479+
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3480+
message = (IndigoHIDMessage *)DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError);
3481+
if (message == NULL) {
3482+
return;
3483+
}
33183484
}
33193485

33203486
NSError *messageError = nil;
3321-
if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) {
3487+
if (!DFSendHIDMessage(self->_hidClient, message, freeWhenDone, &messageError)) {
33223488
dispatchError = messageError ?: DFMakeError(
33233489
DFPrivateSimulatorErrorCodeTouchDispatchFailed,
33243490
@"SimulatorKit rejected the Indigo HID touch packet."
@@ -3394,14 +3560,25 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1
33943560
break;
33953561
}
33963562

3397-
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3398-
DFIndigoMessage *message = DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError);
3563+
IndigoHIDMessage *message = NULL;
3564+
const NSUInteger maxAttempts = phase == DFPrivateSimulatorTouchPhaseMoved ? 12 : 3;
3565+
for (NSUInteger attempt = 0; attempt < maxAttempts; attempt++) {
3566+
message = DFCreateIndigoMultiTouchMessage9(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, phase);
3567+
if (message != NULL) {
3568+
break;
3569+
}
3570+
usleep(5 * 1000);
3571+
}
33993572
if (message == NULL) {
3400-
return;
3573+
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3574+
message = (IndigoHIDMessage *)DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError);
3575+
if (message == NULL) {
3576+
return;
3577+
}
34013578
}
34023579

34033580
NSError *messageError = nil;
3404-
if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) {
3581+
if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) {
34053582
dispatchError = messageError ?: DFMakeError(
34063583
DFPrivateSimulatorErrorCodeTouchDispatchFailed,
34073584
@"SimulatorKit rejected the Indigo HID multi-touch packet."

‎client/src/features/stream/streamWorkerClient.ts‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,18 @@ export function sendWebRtcClientStats(stats: unknown): boolean {
3535
);
3636
}
3737

38+
export function sendWebRtcStreamControl(options: {
39+
forceKeyframe?: boolean;
40+
fps?: number;
41+
profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail";
42+
snapshot?: boolean;
43+
}): boolean {
44+
return sendDataChannelMessage(
45+
activeWebRtcControlChannel,
46+
JSON.stringify({ ...options, type: "streamControl" }),
47+
);
48+
}
49+
3850
function sendDataChannelMessage(
3951
channel: RTCDataChannel | null,
4052
encoded: string,
@@ -63,6 +75,7 @@ interface StreamClientBackend {
6375
connect(target: StreamConnectTarget): void | Promise<void>;
6476
destroy(): void;
6577
disconnect(): void;
78+
sendControl?(payload: unknown): boolean;
6679
}
6780

6881
class WebRtcStreamClient implements StreamClientBackend {
@@ -318,6 +331,10 @@ class WebRtcStreamClient implements StreamClientBackend {
318331
this.onMessage({ type: "status", status: { state: "idle" } });
319332
}
320333

334+
sendControl(payload: unknown): boolean {
335+
return sendDataChannelMessage(this.controlChannel, JSON.stringify(payload));
336+
}
337+
321338
destroy() {
322339
this.disconnect();
323340
}
@@ -989,9 +1006,6 @@ export class StreamWorkerClient {
9891006

9901007
constructor(onMessage: (message: WorkerToMainMessage) => void) {
9911008
this.onMessage = onMessage;
992-
if (activeStreamClient && activeStreamClient !== this) {
993-
activeStreamClient.destroy();
994-
}
9951009
activeStreamClient = this;
9961010
}
9971011

@@ -1038,6 +1052,17 @@ export class StreamWorkerClient {
10381052
this.backend?.clear();
10391053
}
10401054

1055+
sendStreamControl(options: {
1056+
forceKeyframe?: boolean;
1057+
fps?: number;
1058+
profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail";
1059+
snapshot?: boolean;
1060+
}) {
1061+
return Boolean(
1062+
this.backend?.sendControl?.({ ...options, type: "streamControl" }),
1063+
);
1064+
}
1065+
10411066
destroy() {
10421067
if (this.disposed) {
10431068
return;

‎client/src/features/stream/useLiveStream.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ interface UseLiveStreamOptions {
2828
paused?: boolean;
2929
remote?: boolean;
3030
simulator: SimulatorMetadata | null;
31+
streamProfile?: "focus" | "full" | "paused" | "thumb" | "thumbnail";
3132
}
3233

3334
interface UseLiveStreamResult {
@@ -72,6 +73,7 @@ export function useLiveStream({
7273
paused = false,
7374
remote = false,
7475
simulator,
76+
streamProfile = "focus",
7577
}: UseLiveStreamOptions): UseLiveStreamResult {
7678
const clientTelemetryIdRef = useRef("");
7779
const workerClientRef = useRef<StreamWorkerClient | null>(null);
@@ -253,6 +255,31 @@ export function useLiveStream({
253255
};
254256
}, [canvasElement, simulator?.isBooted, simulator?.udid, paused, remote]);
255257

258+
useEffect(() => {
259+
if (paused || !simulator?.isBooted) {
260+
return;
261+
}
262+
let attempts = 0;
263+
const send = () => {
264+
attempts += 1;
265+
return Boolean(
266+
workerClientRef.current?.sendStreamControl({
267+
forceKeyframe: streamProfile === "focus" || streamProfile === "full",
268+
profile: streamProfile,
269+
}),
270+
);
271+
};
272+
if (send()) {
273+
return;
274+
}
275+
const interval = window.setInterval(() => {
276+
if (send() || attempts >= 8) {
277+
window.clearInterval(interval);
278+
}
279+
}, 250);
280+
return () => window.clearInterval(interval);
281+
}, [paused, simulator?.isBooted, simulator?.udid, streamProfile]);
282+
256283
useEffect(() => {
257284
if (!simulator?.udid) {
258285
return;

0 commit comments

Comments
 (0)