Skip to content

Commit e66460e

Browse files
DjDeveloperrDjDeveloperr
committed
Add farm view, stream controls, and native HID fallback
1 parent aea6218 commit e66460e

14 files changed

Lines changed: 1029 additions & 15 deletions

File tree

‎README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ view inside the editor.
3636
## Features
3737

3838
- Local simulator video stream over browser-native WebRTC H.264
39+
- Multi-simulator farm view at `/farm` with low-rate thumbnails and a focused full-rate stream
3940
- Full simulator control & inspection using private accessibility APIs
4041
- CoreSimulator chrome asset rendering for device bezels
4142
- NativeScript, React Native, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live
@@ -62,6 +63,9 @@ simdeck "iPhone 17 Pro Max"
6263
Use `simdeck ui --open` or `simdeck daemon start` when you want a reusable background daemon instead.
6364
The no-subcommand lifecycle shortcuts are `simdeck -d` for detached start, `simdeck -k` to kill the background daemon, and `simdeck -r` to restart it.
6465
The served loopback browser UI receives the generated API access token automatically. LAN browsers pair with the printed code before receiving the API cookie.
66+
Open `http://127.0.0.1:4310/farm` to monitor every simulator in one dashboard.
67+
The farm uses thumbnail stream profiles for the grid and promotes the focused
68+
simulator to a full-rate WebRTC stream.
6569

6670
SimDeck Studio providers run the daemon on loopback and use
6771
`scripts/studio-provider-bridge.mjs` for outbound control-plane communication
@@ -145,6 +149,7 @@ simdeck toggle-appearance <udid>
145149
simdeck pasteboard set <udid> "hello"
146150
simdeck pasteboard get <udid>
147151
simdeck screenshot <udid> --output screen.png
152+
simdeck stream <udid> --frames 120 > stream.h264
148153
simdeck describe <udid>
149154
simdeck describe <udid> --format agent --max-depth 4
150155
simdeck describe <udid> --point 120,240
@@ -172,6 +177,13 @@ simdeck chrome-profile <udid>
172177
simdeck logs <udid> --seconds 30 --limit 200
173178
```
174179

180+
`boot` prefers SimDeck's private CoreSimulator boot path so it can start devices
181+
without launching Simulator.app, then falls back to `xcrun simctl` when private
182+
booting is unavailable.
183+
184+
`stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or
185+
external tools such as `ffplay`.
186+
175187
`describe` uses the project daemon to prefer React Native, NativeScript, or
176188
UIKit in-app inspectors, then falls back to the built-in private CoreSimulator
177189
accessibility bridge. Use `--format agent` or `--format compact-json` for

‎cli/DFPrivateSimulatorDisplayBridge.m‎

Lines changed: 165 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@
3232
typedef uint32_t IndigoHIDEdge;
3333

3434
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, NSEventType type, NSSize displaySize, IndigoHIDEdge edge);
35+
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEvent9Fn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, uint32_t direction, double unused1, double unused2, double widthPoints, double heightPoints);
3536
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardArbitraryFn)(int keyCode, int op);
3637
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event);
3738
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target);
3839
typedef IndigoHIDMessage *(*DFIndigoHIDMessageForHIDArbitraryFn)(uint32_t target, uint32_t page, uint32_t usage, uint32_t operation);
40+
typedef IndigoHIDMessage *(*DFIndigoHIDServiceMessageFn)(void);
3941

4042
#pragma pack(push, 4)
4143
typedef struct {
@@ -91,6 +93,12 @@
9193
static const uint32_t DFIndigoTouchTarget = 0x32;
9294
static const uint8_t DFIndigoEventTypeTouch = 0x02;
9395
static const uint32_t DFIndigoTouchEventKind = 0x0b;
96+
static const uint32_t DFIndigoMouseEventDown = 1;
97+
static const uint32_t DFIndigoMouseEventUp = 2;
98+
static const uint32_t DFIndigoMouseEventDragged = 6;
99+
static const uint32_t DFIndigoMouseDirectionDown = 1;
100+
static const uint32_t DFIndigoMouseDirectionMove = 0;
101+
static const uint32_t DFIndigoMouseDirectionUp = 2;
94102
static const int DFKeyboardDirectionDown = 1;
95103
static const int DFKeyboardDirectionUp = 2;
96104
static const uint32_t DFButtonDirectionDown = 1;
@@ -187,6 +195,36 @@ static BOOL DFVerboseTouchLoggingEnabled(void) {
187195
return enabled;
188196
}
189197

198+
static BOOL DFShouldUseIndigoMouse9Path(void) {
199+
static BOOL enabled = NO;
200+
static dispatch_once_t onceToken;
201+
dispatch_once(&onceToken, ^{
202+
NSString *override = NSProcessInfo.processInfo.environment[@"SIMDECK_INDIGO_MOUSE_9ARG"];
203+
if (override.length > 0) {
204+
enabled = [override isEqualToString:@"1"] ||
205+
[override caseInsensitiveCompare:@"true"] == NSOrderedSame ||
206+
[override caseInsensitiveCompare:@"yes"] == NSOrderedSame;
207+
return;
208+
}
209+
210+
FILE *pipe = popen("/usr/bin/xcodebuild -version 2>/dev/null", "r");
211+
if (pipe == NULL) {
212+
return;
213+
}
214+
char buffer[256] = {0};
215+
if (fgets(buffer, sizeof(buffer), pipe) != NULL) {
216+
NSString *line = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]];
217+
NSArray<NSString *> *parts = [line componentsSeparatedByString:@" "];
218+
if (parts.count >= 2 && [parts[0] isEqualToString:@"Xcode"]) {
219+
NSInteger major = [[parts[1] componentsSeparatedByString:@"."].firstObject integerValue];
220+
enabled = major >= 26;
221+
}
222+
}
223+
pclose(pipe);
224+
});
225+
return enabled;
226+
}
227+
190228
#pragma mark - SimulatorKit Swift symbol resolver
191229
//
192230
// We call into SimulatorKit's private Swift API by dlsym'ing mangled symbol
@@ -1002,6 +1040,109 @@ static BOOL DFSendHIDMessage(id hidClient, IndigoHIDMessage *message, BOOL freeW
10021040
return YES;
10031041
}
10041042

1043+
static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phase) {
1044+
switch (phase) {
1045+
case DFPrivateSimulatorTouchPhaseBegan:
1046+
return DFIndigoMouseEventDown;
1047+
case DFPrivateSimulatorTouchPhaseMoved:
1048+
return DFIndigoMouseEventDragged;
1049+
case DFPrivateSimulatorTouchPhaseEnded:
1050+
case DFPrivateSimulatorTouchPhaseCancelled:
1051+
return DFIndigoMouseEventUp;
1052+
}
1053+
}
1054+
1055+
static uint32_t DFIndigoMouseDirectionForPhase(DFPrivateSimulatorTouchPhase phase) {
1056+
switch (phase) {
1057+
case DFPrivateSimulatorTouchPhaseBegan:
1058+
return DFIndigoMouseDirectionDown;
1059+
case DFPrivateSimulatorTouchPhaseMoved:
1060+
return DFIndigoMouseDirectionMove;
1061+
case DFPrivateSimulatorTouchPhaseEnded:
1062+
case DFPrivateSimulatorTouchPhaseCancelled:
1063+
return DFIndigoMouseDirectionUp;
1064+
}
1065+
}
1066+
1067+
static IndigoHIDMessage *DFCreateIndigoTouchMessage9(CGPoint normalizedPoint, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) {
1068+
if (!DFShouldUseIndigoMouse9Path()) {
1069+
return NULL;
1070+
}
1071+
DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
1072+
if (mouseMessage == NULL) {
1073+
return NULL;
1074+
}
1075+
1076+
CGPoint ratioPoint = CGPointMake(
1077+
fmax(0.0, fmin(1.0, normalizedPoint.x)),
1078+
fmax(0.0, fmin(1.0, normalizedPoint.y))
1079+
);
1080+
return mouseMessage(&ratioPoint,
1081+
NULL,
1082+
DFIndigoTouchTarget,
1083+
DFIndigoMouseEventTypeForPhase(phase),
1084+
DFIndigoMouseDirectionForPhase(phase),
1085+
1.0,
1086+
1.0,
1087+
displaySize.width,
1088+
displaySize.height);
1089+
}
1090+
1091+
static IndigoHIDMessage *DFCreateIndigoMultiTouchMessage9(CGPoint normalizedPoint1, CGPoint normalizedPoint2, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) {
1092+
if (!DFShouldUseIndigoMouse9Path()) {
1093+
return NULL;
1094+
}
1095+
DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
1096+
if (mouseMessage == NULL) {
1097+
return NULL;
1098+
}
1099+
1100+
CGPoint ratioPoint = CGPointMake(
1101+
fmax(0.0, fmin(1.0, normalizedPoint1.x)),
1102+
fmax(0.0, fmin(1.0, normalizedPoint1.y))
1103+
);
1104+
CGPoint secondRatioPoint = CGPointMake(
1105+
fmax(0.0, fmin(1.0, normalizedPoint2.x)),
1106+
fmax(0.0, fmin(1.0, normalizedPoint2.y))
1107+
);
1108+
return mouseMessage(&ratioPoint,
1109+
&secondRatioPoint,
1110+
DFIndigoTouchTarget,
1111+
DFIndigoMouseEventTypeForPhase(phase),
1112+
DFIndigoMouseDirectionForPhase(phase),
1113+
1.0,
1114+
1.0,
1115+
displaySize.width,
1116+
displaySize.height);
1117+
}
1118+
1119+
static void DFWarmIndigoHIDServices(id hidClient) {
1120+
if (hidClient == nil) {
1121+
return;
1122+
}
1123+
1124+
DFIndigoHIDServiceMessageFn createPointer = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreatePointerService");
1125+
DFIndigoHIDServiceMessageFn createMouse = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreateMouseService");
1126+
NSError *error = nil;
1127+
if (createPointer != NULL) {
1128+
IndigoHIDMessage *message = createPointer();
1129+
if (message != NULL) {
1130+
(void)DFSendHIDMessage(hidClient, message, YES, &error);
1131+
usleep(20 * 1000);
1132+
}
1133+
}
1134+
if (createMouse != NULL) {
1135+
IndigoHIDMessage *message = createMouse();
1136+
if (message != NULL) {
1137+
(void)DFSendHIDMessage(hidClient, message, YES, &error);
1138+
usleep(20 * 1000);
1139+
}
1140+
}
1141+
if (error != nil) {
1142+
DFLog(@"Indigo HID service warm-up reported: %@", error.localizedDescription ?: @"unknown error");
1143+
}
1144+
}
1145+
10051146
static DFIndigoMessage *DFCreateIndigoTouchMessage(CGPoint normalizedPoint, NSSize displaySize, BOOL touchDown, NSError **error) {
10061147
DFIndigoHIDMessageForMouseNSEventFn mouseMessage = (DFIndigoHIDMessageForMouseNSEventFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
10071148
if (mouseMessage == NULL) {
@@ -2378,6 +2519,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid
23782519

23792520
if (_hidClient != nil) {
23802521
DFLog(@"Created private SimulatorKit HID client for %@", udid);
2522+
DFWarmIndigoHIDServices(_hidClient);
23812523
} else {
23822524
DFLog(@"Failed to create private SimulatorKit HID client for %@: %@", udid, hidClientError.localizedDescription ?: @"unknown error");
23832525
}
@@ -3311,14 +3453,18 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX
33113453
phaseLabel = phase == DFPrivateSimulatorTouchPhaseEnded ? @"ended" : @"cancelled";
33123454
break;
33133455
}
3314-
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3315-
DFIndigoMessage *message = DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError);
3456+
IndigoHIDMessage *message = DFCreateIndigoTouchMessage9(CGPointMake(clampedX, clampedY), displaySize, phase);
3457+
BOOL freeWhenDone = YES;
33163458
if (message == NULL) {
3317-
return;
3459+
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3460+
message = (IndigoHIDMessage *)DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError);
3461+
if (message == NULL) {
3462+
return;
3463+
}
33183464
}
33193465

33203466
NSError *messageError = nil;
3321-
if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) {
3467+
if (!DFSendHIDMessage(self->_hidClient, message, freeWhenDone, &messageError)) {
33223468
dispatchError = messageError ?: DFMakeError(
33233469
DFPrivateSimulatorErrorCodeTouchDispatchFailed,
33243470
@"SimulatorKit rejected the Indigo HID touch packet."
@@ -3394,14 +3540,25 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1
33943540
break;
33953541
}
33963542

3397-
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3398-
DFIndigoMessage *message = DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError);
3543+
IndigoHIDMessage *message = NULL;
3544+
const NSUInteger maxAttempts = phase == DFPrivateSimulatorTouchPhaseMoved ? 12 : 3;
3545+
for (NSUInteger attempt = 0; attempt < maxAttempts; attempt++) {
3546+
message = DFCreateIndigoMultiTouchMessage9(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, phase);
3547+
if (message != NULL) {
3548+
break;
3549+
}
3550+
usleep(5 * 1000);
3551+
}
33993552
if (message == NULL) {
3400-
return;
3553+
BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
3554+
message = (IndigoHIDMessage *)DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError);
3555+
if (message == NULL) {
3556+
return;
3557+
}
34013558
}
34023559

34033560
NSError *messageError = nil;
3404-
if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) {
3561+
if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) {
34053562
dispatchError = messageError ?: DFMakeError(
34063563
DFPrivateSimulatorErrorCodeTouchDispatchFailed,
34073564
@"SimulatorKit rejected the Indigo HID multi-touch packet."

‎client/src/app/App.tsx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import { AppShell } from "./AppShell";
2+
import { FarmView } from "../features/farm/FarmView";
23

34
export default function App() {
5+
if (window.location.pathname === "/farm") {
6+
return <FarmView />;
7+
}
48
return <AppShell />;
59
}

0 commit comments

Comments
 (0)