From eebd8a10ee4f7e942d911539e60db81f8d15975b Mon Sep 17 00:00:00 2001 From: Bryan English Date: Tue, 5 May 2026 23:45:26 -0400 Subject: [PATCH 1/4] ffi: add V8 fast-call path Add a parallel dispatch path that uses V8 fast API calls instead of libffi for eligible native calls. At DynamicLibrary.getFunction time, generate a per-function JIT'd trampoline that strips V8's receiver argument and tail-calls the target. Signatures with callbacks, unsupported argument types, or more register-passed args than the platform ABI permits transparently fall back to libffi. Stub emitters cover Linux/macOS/FreeBSD on x86_64 and AArch64, Windows on x86_64 and AArch64, and Linux on AArch32. JIT memory is allocated per isolate via direct mmap with MAP_JIT on macOS and W^X enforcement elsewhere. The JS wrapper validates each argument per declared type, mirroring the libffi slow callback so the contract is identical across both paths and across V8 optimization tiers. The path is gated behind --experimental-ffi and can be disabled at build time with --without-ffi-fastcall. The previous shared-buffer JS fast path is removed, replaced by this fast-call path. Signed-off-by: Bryan English --- configure.py | 25 + doc/contributing/ffi-fastcall-internals.md | 186 ++++ lib/ffi.js | 2 +- lib/internal/errors.js | 1 + lib/internal/ffi-fastcall.js | 572 +++++++++++++ lib/internal/ffi-shared-buffer.js | 635 -------------- node.gyp | 56 ++ src/env_properties.h | 8 +- src/ffi/fastcall/cfunction_info.cc | 135 +++ src/ffi/fastcall/cfunction_info.h | 39 + src/ffi/fastcall/jit_memory.cc | 292 +++++++ src/ffi/fastcall/jit_memory.h | 87 ++ src/ffi/fastcall/stub_emitter.h | 49 ++ src/ffi/fastcall/stub_emitter_aarch64.cc | 95 +++ src/ffi/fastcall/stub_emitter_arm.cc | 71 ++ src/ffi/fastcall/stub_emitter_x64_sysv.cc | 87 ++ src/ffi/fastcall/stub_emitter_x64_win.cc | 80 ++ src/ffi/types.cc | 222 ++--- src/ffi/types.h | 35 +- src/node_builtins.cc | 2 +- src/node_ffi.cc | 401 +++++---- src/node_ffi.h | 73 +- test/cctest/test_ffi_fastcall_cfunction.cc | 116 +++ test/cctest/test_ffi_fastcall_eligibility.cc | 233 +++++ test/cctest/test_ffi_fastcall_emitter.cc | 233 +++++ test/cctest/test_ffi_fastcall_jit.cc | 159 ++++ test/ffi/fixture_library/ffi_test_library.c | 18 + test/ffi/test-ffi-dynamic-library.js | 4 +- test/ffi/test-ffi-fastcall-arity.js | 90 ++ test/ffi/test-ffi-fastcall-close.js | 56 ++ test/ffi/test-ffi-fastcall-eligibility.js | 50 ++ test/ffi/test-ffi-fastcall-optimized.js | 160 ++++ .../ffi/test-ffi-fastcall-pointer-fallback.js | 105 +++ test/ffi/test-ffi-fastcall-types.js | 83 ++ test/ffi/test-ffi-shared-buffer.js | 806 ------------------ 35 files changed, 3537 insertions(+), 1729 deletions(-) create mode 100644 doc/contributing/ffi-fastcall-internals.md create mode 100644 lib/internal/ffi-fastcall.js delete mode 100644 lib/internal/ffi-shared-buffer.js create mode 100644 src/ffi/fastcall/cfunction_info.cc create mode 100644 src/ffi/fastcall/cfunction_info.h create mode 100644 src/ffi/fastcall/jit_memory.cc create mode 100644 src/ffi/fastcall/jit_memory.h create mode 100644 src/ffi/fastcall/stub_emitter.h create mode 100644 src/ffi/fastcall/stub_emitter_aarch64.cc create mode 100644 src/ffi/fastcall/stub_emitter_arm.cc create mode 100644 src/ffi/fastcall/stub_emitter_x64_sysv.cc create mode 100644 src/ffi/fastcall/stub_emitter_x64_win.cc create mode 100644 test/cctest/test_ffi_fastcall_cfunction.cc create mode 100644 test/cctest/test_ffi_fastcall_eligibility.cc create mode 100644 test/cctest/test_ffi_fastcall_emitter.cc create mode 100644 test/cctest/test_ffi_fastcall_jit.cc create mode 100644 test/ffi/test-ffi-fastcall-arity.js create mode 100644 test/ffi/test-ffi-fastcall-close.js create mode 100644 test/ffi/test-ffi-fastcall-eligibility.js create mode 100644 test/ffi/test-ffi-fastcall-optimized.js create mode 100644 test/ffi/test-ffi-fastcall-pointer-fallback.js create mode 100644 test/ffi/test-ffi-fastcall-types.js delete mode 100644 test/ffi/test-ffi-shared-buffer.js diff --git a/configure.py b/configure.py index c167a0a0b27358..106a0acdee50f4 100755 --- a/configure.py +++ b/configure.py @@ -1059,6 +1059,12 @@ default=None, help='build without FFI (Foreign Function Interface) support') +parser.add_argument('--without-ffi-fastcall', + action='store_true', + dest='without_ffi_fastcall', + default=False, + help='disable the FFI V8-fast-call path; libffi-only') + parser.add_argument('--experimental-quic', action='store_true', dest='experimental_quic', @@ -2324,6 +2330,19 @@ def bundled_ffi_supported(os_name, target_arch): return target_arch in supported.get(os_name, set()) +def fastcall_supported(os_name, target_arch): + supported = { + 'freebsd': {'arm', 'arm64', 'x64'}, + 'linux': {'arm', 'arm64', 'x64'}, + 'mac': {'arm64', 'x64'}, + 'win': {'arm64', 'x64'}, + } + + if target_arch == 'x86': + target_arch = 'ia32' + + return target_arch in supported.get(os_name, set()) + def configure_ffi(o): use_ffi = not options.without_ffi @@ -2337,6 +2356,7 @@ def configure_ffi(o): use_ffi = False o['variables']['node_use_ffi'] = b(use_ffi) + o['variables']['node_use_ffi_fastcall'] = b(False) if options.without_ffi: if options.shared_ffi: @@ -2348,6 +2368,11 @@ def configure_ffi(o): configure_library('ffi', o, pkgname='libffi') + use_fastcall = use_ffi and not options.without_ffi_fastcall + if use_fastcall and not fastcall_supported(flavor, o['variables']['target_arch']): + use_fastcall = False + o['variables']['node_use_ffi_fastcall'] = b(use_fastcall) + def configure_quic(o): o['variables']['node_use_quic'] = b(options.experimental_quic and not options.without_ssl) diff --git a/doc/contributing/ffi-fastcall-internals.md b/doc/contributing/ffi-fastcall-internals.md new file mode 100644 index 00000000000000..31e9070a1914e1 --- /dev/null +++ b/doc/contributing/ffi-fastcall-internals.md @@ -0,0 +1,186 @@ +# FFI Fast-Call Internals + +This document is for contributors who maintain or extend the FFI +fast-call path (the V8 Fast API Calls implementation in `node:ffi`). +For end-user behavior, see [doc/api/ffi.md](../api/ffi.md). + +## Overview + +For each registered FFI function whose signature is fast-call eligible +(`src/ffi/types.cc:IsFastCallEligible`), Node generates a tiny native +trampoline that strips the `Local` receiver V8 fast calls +require and tail-calls the user's target function. The trampoline +address is handed to `v8::CFunction`. A JS wrapper +(`lib/internal/ffi-fastcall.js`) validates args, routes object-typed +pointer args to a libffi slow path, and checks a per-library "alive" +sentinel before each call. + +The libffi path remains for callbacks (`ffi_prep_closure_loc`), +ineligible signatures (signatures containing the FFI `function` type), +and unsupported platforms. + +## Eligibility (`src/ffi/types.cc:IsFastCallEligible`) + +A signature is fast-call eligible iff all of: + +1. The platform is supported (see Platform Support below). +2. Return type is one of: void, i8/u8/i16/u16, i32/u32, i64/u64, + f32/f64, pointer. +3. Every arg type is in that set. +4. No arg or return is the FFI `function` type. +5. Per-ABI argument caps: + - AArch64: ≤ 7 GP, ≤ 8 FP + - x86_64 SysV: ≤ 6 GP, ≤ 8 FP + - x86_64 Win64: GP + FP combined ≤ 3 (positional register slots — 4 minus the receiver) + - AArch32 hardfp: ≤ 3 GP, ≤ 8 FP; i64/u64 args and return type rejected + +`IsFastCallEligible(fn, &reason)` returns false with a static reason +string on miss. + +## Platform support + +| ABI | Emitter file | Status | +|---|---|---| +| AArch64 (Linux/macOS/FreeBSD/Windows) | `stub_emitter_aarch64.cc` | Implemented, runtime-verified | +| x86_64 SysV (Linux/macOS/FreeBSD) | `stub_emitter_x64_sysv.cc` | Implemented, CI-verified | +| x86_64 Win64 | `stub_emitter_x64_win.cc` | Implemented, CI-verified | +| AArch32 hardfp (Linux/FreeBSD) | `stub_emitter_arm.cc` | Implemented, CI-verified | + +On platforms without an emitter, all registrations fall back to libffi. + +Adding a new ABI: implement `EmitForwarder` for the new platform in a +new `stub_emitter_.cc`, gate it via `node.gyp` conditions on +`target_arch` and `OS`, and add the `(os, arch)` pair to +`fastcall_supported` in `configure.py`. + +## Stub generation (`src/ffi/fastcall/stub_emitter_*.cc`) + +Each stub does, at most, three things: + +1. Shift GP regs down by one slot (drop the receiver). +2. (Win64 only) shift FP regs down by one slot — Win64's FP/GP register + slots are positional, so stripping a GP arg also reindexes FP slots. +3. Tail-call the target via an indirect jump. + +For SysV ≥ 6 GP args, the stub uses a call+ret pattern with stack +rewrite (because the 7th GP slot lives on the stack). Other ABIs cap +below their stack overflow point in v1 to keep emitters simple. + +## JIT memory (`src/ffi/fastcall/jit_memory.cc`) + +A process-global singleton on top of platform `mmap`/`VirtualAlloc`. +Allocates 64-byte slot-aligned chunks within page-aligned allocations. +After writing the stub, the page is transitioned to RX via `mprotect` / +`VirtualProtect`; once a page goes RX, no further allocation happens +in it (the bump cursor is locked). + +The original spec called for `v8::PageAllocator`, but neither +`Isolate::GetArrayBufferAllocator()->GetPageAllocator()` nor +`Platform::GetPageAllocator()` returns a usable allocator in Node's +embedded configuration — both default to `nullptr`. The implementation +uses direct system calls (with `MAP_JIT` on Apple Silicon) instead. + +`Free` decrements the live-byte counter but does not return memory. +Pages stay alive for the process lifetime. + +Concurrent emit from multiple isolates is safe via +`JitMemory::EmitStub(code, size)`, which holds the singleton mutex across +allocate + memcpy + RX-transition. The lower-level `Allocate` / +`MakeExecutable` / `Free` methods remain public for the self-test only +(which writes platform-specific instruction bytes after Allocate but +before MakeExecutable, and needs that explicit step ordering). + +## Self-test + +`JitMemory::SelfTest` allocates a tiny stub, writes a `ret`-style +native sequence, transitions to RX, and calls it. Cached in a +process-wide atomic via `std::call_once`. Run once per process at +first FFI registration. On failure, every subsequent registration +falls back to libffi-only and a process warning is emitted via +`ProcessEmitWarning`. + +This catches: +- macOS `MAP_JIT` entitlement missing (e.g., signed binary without + `com.apple.security.cs.allow-jit`). +- Hardened-runtime restrictions. +- SELinux execmem denial. + +## JS wrapper (`lib/internal/ffi-fastcall.js`) + +For each fast-call-eligible inner v8::Function returned from C++, +`buildWrapper` creates a JS wrapper that: + +1. Reads the per-library "alive" `Uint8Array` and throws + `ERR_FFI_LIBRARY_CLOSED` if `[0] !== 0`. +2. Per-arg validation, mirroring `ToFFIArgument` in + `src/ffi/types.cc:ToFFIArgument`. Same `ERR_INVALID_ARG_VALUE` + codes, same messages, same range bounds. +3. Pointer args: + - BigInt or null/undefined: pass through as primitive. + - String / Buffer / ArrayBuffer / ArrayBufferView: `ReflectApply` + the `kFastcallInvokeSlow` libffi-backed v8::Function with the + original args. +4. Calls the inner v8::Function with positional primitives. V8's fast + call engages when TurboFan inlines the wrapper. + +The wrapper body is **arity-specialized**: arities 0..6 are unrolled into +distinct closures with named parameters (`function(a0, a1, ...)`), so V8 +inlines them and the per-arg type info / pointer flag are read from +closure locals instead of arrays. Arities 7+ use a rest-args fallback. This +matters: an earlier draft used a single generic `function(...args)` plus +`ReflectApply`, which dropped FFI throughput by 30–50% vs. the libffi+SB +path. The arity specialization gets the throughput back to 5–13× the +libffi+SB baseline (see commit `81d908e48da` for the fix and benchmarks). + +The wrapper is patched onto `DynamicLibrary.prototype.getFunction`, +`getFunctions`, and the `functions` accessor. + +## Internal symbols + +The JS wrapper looks for these per-isolate Symbols on the inner +`v8::Function`. They are defined in `src/env_properties.h` and +attached by `DynamicLibrary::CreateFunction` for fast-call-eligible +signatures only: + +| Symbol | Value | Purpose | +|---|---|---| +| `kFastcallAlive` | `Uint8Array(1)` shared with `DynamicLibrary` | close sentinel | +| `kFastcallInvokeSlow` | `v8::Function` over `InvokeFunction` | object-arg fallback | +| `kFastcallParams` | `string[]` of parameter type names | wrapper introspection | +| `kFastcallResult` | result type name string | wrapper introspection | + +## Lifecycle + +**Registration:** `CreateFunction` in `src/node_ffi.cc` builds a +`fastcall::CFunctionInfoBundle` (which owns the heap-allocated +`v8::CFunctionInfo` + `v8::CTypeInfo[]`), allocates and emits the stub via +`JitMemory::EmitStub`, then constructs the inner `v8::Function` via a +`FunctionTemplate` with the `CFunction` attached. Per-function fast-call +state is stored on `FFIFunctionInfo::fast` (a `unique_ptr`, +null when fast-call is unavailable for that signature). + +**Per-call:** wrapper validates → calls inner. V8 picks fast or slow +callback. Slow = `InvokeFunction` (libffi); fast = our generated stub → +target. + +**`lib.close()`:** flips the alive sentinel (`alive[0] = 1`). The wrapper +throws `ERR_FFI_LIBRARY_CLOSED` on subsequent calls. Slow-path +`InvokeFunction` independently checks `fn->closed` for the same effect on +ineligible signatures. Stubs are NOT freed at close. + +**Weak callback (function GC'd):** `CleanupFunctionInfo` resets +`info->fast`, whose `~FastCallState` destructor calls `JitMemory::Free` +on the stub. + +## Testing + +- `test/cctest/test_ffi_fastcall_*.cc`: unit tests for emitters, JIT + memory, eligibility, CFunctionInfo builder. +- `test/ffi/test-ffi-*.js`: JS-level integration tests covering + types, arity, callbacks, permissions, etc. (existing FFI suite — + reused as the integration baseline). + +When debugging unexpected fast-call behavior, log the eligibility miss +reason via the second arg to `IsFastCallEligible`. Set the +`--without-ffi-fastcall` configure flag to A/B test against the +libffi-only path. diff --git a/lib/ffi.js b/lib/ffi.js index 98af095e0cb01c..57975d97a5bd7c 100644 --- a/lib/ffi.js +++ b/lib/ffi.js @@ -54,7 +54,7 @@ const { toArrayBuffer, } = internalBinding('ffi'); -require('internal/ffi-shared-buffer'); +require('internal/ffi-fastcall'); DynamicLibrary.prototype[SymbolDispose] = function() { this.close(); diff --git a/lib/internal/errors.js b/lib/internal/errors.js index c40eed86bca834..e59a24d939c1dc 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1231,6 +1231,7 @@ E('ERR_FEATURE_UNAVAILABLE_ON_PLATFORM', 'The feature %s is unavailable on the current platform' + ', which is being used to run Node.js', TypeError); +E('ERR_FFI_LIBRARY_CLOSED', 'Library is closed', Error); E('ERR_FS_CP_DIR_TO_NON_DIR', 'Cannot overwrite non-directory with directory', SystemError); E('ERR_FS_CP_EEXIST', 'Target already exists', SystemError); diff --git a/lib/internal/ffi-fastcall.js b/lib/internal/ffi-fastcall.js new file mode 100644 index 00000000000000..3c0f435f0855d9 --- /dev/null +++ b/lib/internal/ffi-fastcall.js @@ -0,0 +1,572 @@ +'use strict'; + +const { + ArrayPrototypePush, + Error, + FunctionPrototypeCall, + NumberIsInteger, + ObjectDefineProperty, + ObjectGetOwnPropertyDescriptor, + ObjectKeys, + ReflectApply, + TypeError, + Uint8Array, +} = primordials; + +const { + codes: { + ERR_FFI_LIBRARY_CLOSED, + ERR_INTERNAL_ASSERTION, + }, +} = require('internal/errors'); + +const ffiBinding = internalBinding('ffi'); +const { + DynamicLibrary, + charIsSigned, + uintptrMax, +} = ffiBinding; + +const kFastcallAlive = ffiBinding.kFastcallAlive; +const kFastcallInvokeSlow = ffiBinding.kFastcallInvokeSlow; +const kFastcallParams = ffiBinding.kFastcallParams; +const kFastcallResult = ffiBinding.kFastcallResult; + +const fastcallEnabled = kFastcallAlive !== undefined; + +const U64_MAX = 0xFFFFFFFFFFFFFFFFn; +const I64_MAX = 0x7FFFFFFFFFFFFFFFn; +const I64_MIN = -0x8000000000000000n; + +// Note: TYPES (the outer object) has __proto__: null because user-input +// type names are looked up here (e.g., TYPES[parameters[i]]), so the +// outer guard prevents prototype-chain probing. The inner entries +// deliberately do NOT have __proto__: null. The hot path reads +// info.kind / info.min / info.max / info.label from these objects, and +// V8's inline caches dispatch ~50% faster on Object-prototype-inheriting +// shapes than on null-proto ones across this dispatch pattern. The +// inner objects are never user-keyed, so the safety concern doesn't +// apply. See commit 53bf0595a1f for the regression analysis. +const TYPES = { + __proto__: null, + i8: { kind: 'int', min: -128, max: 127, label: 'an int8' }, + int8: { kind: 'int', min: -128, max: 127, label: 'an int8' }, + char: charIsSigned + ? { kind: 'int', min: -128, max: 127, label: 'an int8' } + : { kind: 'int', min: 0, max: 255, label: 'a uint8' }, + u8: { kind: 'int', min: 0, max: 255, label: 'a uint8' }, + uint8: { kind: 'int', min: 0, max: 255, label: 'a uint8' }, + bool: { kind: 'int', min: 0, max: 255, label: 'a uint8' }, + i16: { kind: 'int', min: -32768, max: 32767, label: 'an int16' }, + int16: { kind: 'int', min: -32768, max: 32767, label: 'an int16' }, + u16: { kind: 'int', min: 0, max: 65535, label: 'a uint16' }, + uint16: { kind: 'int', min: 0, max: 65535, label: 'a uint16' }, + i32: { kind: 'int32', label: 'an int32' }, + int32: { kind: 'int32', label: 'an int32' }, + u32: { kind: 'uint32', label: 'a uint32' }, + uint32: { kind: 'uint32', label: 'a uint32' }, + i64: { kind: 'i64', label: 'an int64' }, + int64: { kind: 'i64', label: 'an int64' }, + u64: { kind: 'u64', label: 'a uint64' }, + uint64: { kind: 'u64', label: 'a uint64' }, + f32: { kind: 'float', label: 'a float' }, + float: { kind: 'float', label: 'a float' }, + float32: { kind: 'float', label: 'a float' }, + f64: { kind: 'double', label: 'a double' }, + double: { kind: 'double', label: 'a double' }, + float64: { kind: 'double', label: 'a double' }, + pointer: { kind: 'pointer' }, + ptr: { kind: 'pointer' }, + buffer: { kind: 'pointer' }, + arraybuffer: { kind: 'pointer' }, + string: { kind: 'pointer' }, + str: { kind: 'pointer' }, +}; + +function throwArg(msg) { + // eslint-disable-next-line no-restricted-syntax + const err = new TypeError(msg); + err.code = 'ERR_INVALID_ARG_VALUE'; + throw err; +} + + +function validateAndCoerce(info, arg, idx) { + const k = info.kind; + if (k === 'int') { + if (typeof arg !== 'number' || !NumberIsInteger(arg) || + arg < info.min || arg > info.max) { + throwArg(`Argument ${idx} must be ${info.label}`); + } + return arg; + } + if (k === 'int32') { + if (typeof arg !== 'number' || !NumberIsInteger(arg) || + arg < -2147483648 || arg > 2147483647) { + throwArg(`Argument ${idx} must be ${info.label}`); + } + return arg; + } + if (k === 'uint32') { + if (typeof arg !== 'number' || !NumberIsInteger(arg) || + arg < 0 || arg > 4294967295) { + throwArg(`Argument ${idx} must be ${info.label}`); + } + return arg; + } + if (k === 'i64') { + if (typeof arg !== 'bigint' || arg < I64_MIN || arg > I64_MAX) { + throwArg(`Argument ${idx} must be ${info.label}`); + } + return arg; + } + if (k === 'u64') { + if (typeof arg !== 'bigint' || arg < 0n || arg > U64_MAX) { + throwArg(`Argument ${idx} must be ${info.label}`); + } + return arg; + } + if (k === 'float' || k === 'double') { + if (typeof arg !== 'number') { + throwArg(`Argument ${idx} must be ${info.label}`); + } + return arg; + } + throw new ERR_INTERNAL_ASSERTION( + `FFI: unexpected kind="${k}" in validateAndCoerce`); +} + +const SLOW_PATH = Symbol('slow-path'); +function coercePointer(arg, idx) { + if (typeof arg === 'bigint') { + if (arg < 0n || arg > uintptrMax) { + throwArg(`Argument ${idx} must be a non-negative pointer bigint`); + } + return arg; + } + if (arg === null || arg === undefined) return 0n; + return SLOW_PATH; +} + +// Specialized wrapper for strict-numeric signatures (no pointers). Validates +// each arg the same way the full wrapper does — V8's fast-call coercion +// silently truncates non-integers and out-of-range values when the call site +// is fully optimized (e.g. CheckedNumberAsWord32 wraps mod 2^32; for i64/u64, +// CheckedBigIntTruncatingWord64 wraps to low 64 bits). Skipping the JS check +// would make the same FFI binding throw cold and silently corrupt hot — so +// the per-arg validation stays. The win over the full wrapper is the dropped +// per-arg `if (isPointer)` branch and the smaller closure (no slowInvoke, +// no isPointer table). +function buildStrictNumericWrapper(rawFn, alive, infos, nargs) { + switch (nargs) { + case 0: + return function() { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 0) { + throwArg(`Invalid argument count: expected 0, got ${arguments.length}`); + } + return rawFn(); + }; + case 1: { + const i0 = infos[0]; + return function(a0) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 1) { + throwArg(`Invalid argument count: expected 1, got ${arguments.length}`); + } + return rawFn(validateAndCoerce(i0, a0, 0)); + }; + } + case 2: { + const i0 = infos[0]; const i1 = infos[1]; + return function(a0, a1) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 2) { + throwArg(`Invalid argument count: expected 2, got ${arguments.length}`); + } + const v0 = validateAndCoerce(i0, a0, 0); + const v1 = validateAndCoerce(i1, a1, 1); + return rawFn(v0, v1); + }; + } + case 3: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; + return function(a0, a1, a2) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 3) { + throwArg(`Invalid argument count: expected 3, got ${arguments.length}`); + } + const v0 = validateAndCoerce(i0, a0, 0); + const v1 = validateAndCoerce(i1, a1, 1); + const v2 = validateAndCoerce(i2, a2, 2); + return rawFn(v0, v1, v2); + }; + } + case 4: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; + return function(a0, a1, a2, a3) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 4) { + throwArg(`Invalid argument count: expected 4, got ${arguments.length}`); + } + const v0 = validateAndCoerce(i0, a0, 0); + const v1 = validateAndCoerce(i1, a1, 1); + const v2 = validateAndCoerce(i2, a2, 2); + const v3 = validateAndCoerce(i3, a3, 3); + return rawFn(v0, v1, v2, v3); + }; + } + case 5: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; const i4 = infos[4]; + return function(a0, a1, a2, a3, a4) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 5) { + throwArg(`Invalid argument count: expected 5, got ${arguments.length}`); + } + const v0 = validateAndCoerce(i0, a0, 0); + const v1 = validateAndCoerce(i1, a1, 1); + const v2 = validateAndCoerce(i2, a2, 2); + const v3 = validateAndCoerce(i3, a3, 3); + const v4 = validateAndCoerce(i4, a4, 4); + return rawFn(v0, v1, v2, v3, v4); + }; + } + case 6: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; const i4 = infos[4]; const i5 = infos[5]; + return function(a0, a1, a2, a3, a4, a5) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 6) { + throwArg(`Invalid argument count: expected 6, got ${arguments.length}`); + } + const v0 = validateAndCoerce(i0, a0, 0); + const v1 = validateAndCoerce(i1, a1, 1); + const v2 = validateAndCoerce(i2, a2, 2); + const v3 = validateAndCoerce(i3, a3, 3); + const v4 = validateAndCoerce(i4, a4, 4); + const v5 = validateAndCoerce(i5, a5, 5); + return rawFn(v0, v1, v2, v3, v4, v5); + }; + } + default: + return function(...args) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (args.length !== nargs) { + throwArg(`Invalid argument count: expected ${nargs}, got ${args.length}`); + } + const out = []; + for (let i = 0; i < nargs; i++) { + ArrayPrototypePush(out, validateAndCoerce(infos[i], args[i], i)); + } + return ReflectApply(rawFn, undefined, out); + }; + } +} + +function inheritMetadata(wrapper, rawFn, nargs) { + ObjectDefineProperty(wrapper, 'name', { + __proto__: null, value: rawFn.name, configurable: true, + }); + ObjectDefineProperty(wrapper, 'length', { + __proto__: null, value: nargs, configurable: true, + }); + ObjectDefineProperty(wrapper, 'pointer', { + __proto__: null, value: rawFn.pointer, + writable: true, configurable: true, enumerable: true, + }); + return wrapper; +} + +// Types eligible for the strict-numeric wrapper: per-arg validation runs +// the same way as the full wrapper, but the dispatch is simpler (no pointer +// branches, smaller closure). We don't include narrower ints (i8/u8/i16/u16/ +// bool/char) because their range check is per-type rather than per-kind, +// and they're a minority of fast-call signatures — the full wrapper already +// handles them. Pointer types are excluded because they need null/undefined +// → 0n coercion plus object-arg slow-path fallback. +const STRICT_NUMERIC_TYPES = new Set([ + 'i32', 'int32', 'u32', 'uint32', + 'i64', 'int64', 'u64', 'uint64', + 'f32', 'float', 'float32', + 'f64', 'double', 'float64', +]); + +function isStrictNumericSignature(parameters, resultType) { + if (resultType !== 'void' && !STRICT_NUMERIC_TYPES.has(resultType)) { + return false; + } + for (let i = 0; i < parameters.length; i++) { + if (!STRICT_NUMERIC_TYPES.has(parameters[i])) return false; + } + return true; +} + +function buildWrapper(rawFn, parameters, resultType) { + if (!fastcallEnabled) return rawFn; + if (rawFn === undefined || rawFn === null) return rawFn; + const aliveAB = rawFn[kFastcallAlive]; + if (aliveAB === undefined) return rawFn; + + const slowInvoke = rawFn[kFastcallInvokeSlow]; + if (parameters === undefined) parameters = rawFn[kFastcallParams]; + if (resultType === undefined) resultType = rawFn[kFastcallResult]; + if (parameters === undefined || resultType === undefined || + slowInvoke === undefined) { + throw new ERR_INTERNAL_ASSERTION( + 'FFI: fast-call raw function is missing required metadata'); + } + + const alive = new Uint8Array(aliveAB); + const nargs = parameters.length; + + const infos = []; + const isPointer = []; + for (let i = 0; i < nargs; i++) { + const info = TYPES[parameters[i]]; + if (info === undefined) { + throw new ERR_INTERNAL_ASSERTION( + `FFI: fast-call type table missing entry for "${parameters[i]}"`); + } + infos.push(info); + isPointer.push(info.kind === 'pointer'); + } + + // Fast path: pure-numeric signatures get a wrapper without per-arg pointer + // branches or slow-path closure capture. Same per-arg validation as the + // full wrapper. + if (isStrictNumericSignature(parameters, resultType)) { + return inheritMetadata( + buildStrictNumericWrapper(rawFn, alive, infos, nargs), rawFn, nargs); + } + + let wrapper; + switch (nargs) { + case 0: + wrapper = function() { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 0) { + throwArg(`Invalid argument count: expected 0, got ${arguments.length}`); + } + return rawFn(); + }; + break; + case 1: { + const i0 = infos[0]; + const p0 = isPointer[0]; + wrapper = function(a0) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 1) { + throwArg(`Invalid argument count: expected 1, got ${arguments.length}`); + } + let v0; + if (p0) { + v0 = coercePointer(a0, 0); + if (v0 === SLOW_PATH) return slowInvoke(a0); + } else { + v0 = validateAndCoerce(i0, a0, 0); + } + return rawFn(v0); + }; + break; + } + case 2: { + const i0 = infos[0]; const i1 = infos[1]; + const p0 = isPointer[0]; const p1 = isPointer[1]; + wrapper = function(a0, a1) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 2) { + throwArg(`Invalid argument count: expected 2, got ${arguments.length}`); + } + let v0, v1; + if (p0) { + v0 = coercePointer(a0, 0); + if (v0 === SLOW_PATH) return slowInvoke(a0, a1); + } else { + v0 = validateAndCoerce(i0, a0, 0); + } + if (p1) { + v1 = coercePointer(a1, 1); + if (v1 === SLOW_PATH) return slowInvoke(a0, a1); + } else { + v1 = validateAndCoerce(i1, a1, 1); + } + return rawFn(v0, v1); + }; + break; + } + case 3: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; + const p0 = isPointer[0]; const p1 = isPointer[1]; const p2 = isPointer[2]; + wrapper = function(a0, a1, a2) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 3) { + throwArg(`Invalid argument count: expected 3, got ${arguments.length}`); + } + let v0, v1, v2; + if (p0) { v0 = coercePointer(a0, 0); if (v0 === SLOW_PATH) return slowInvoke(a0, a1, a2); } + else { v0 = validateAndCoerce(i0, a0, 0); } + if (p1) { v1 = coercePointer(a1, 1); if (v1 === SLOW_PATH) return slowInvoke(a0, a1, a2); } + else { v1 = validateAndCoerce(i1, a1, 1); } + if (p2) { v2 = coercePointer(a2, 2); if (v2 === SLOW_PATH) return slowInvoke(a0, a1, a2); } + else { v2 = validateAndCoerce(i2, a2, 2); } + return rawFn(v0, v1, v2); + }; + break; + } + case 4: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; + const p0 = isPointer[0]; const p1 = isPointer[1]; const p2 = isPointer[2]; const p3 = isPointer[3]; + wrapper = function(a0, a1, a2, a3) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 4) { + throwArg(`Invalid argument count: expected 4, got ${arguments.length}`); + } + let v0, v1, v2, v3; + if (p0) { v0 = coercePointer(a0, 0); if (v0 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3); } + else { v0 = validateAndCoerce(i0, a0, 0); } + if (p1) { v1 = coercePointer(a1, 1); if (v1 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3); } + else { v1 = validateAndCoerce(i1, a1, 1); } + if (p2) { v2 = coercePointer(a2, 2); if (v2 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3); } + else { v2 = validateAndCoerce(i2, a2, 2); } + if (p3) { v3 = coercePointer(a3, 3); if (v3 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3); } + else { v3 = validateAndCoerce(i3, a3, 3); } + return rawFn(v0, v1, v2, v3); + }; + break; + } + case 5: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; const i4 = infos[4]; + const p0 = isPointer[0]; const p1 = isPointer[1]; const p2 = isPointer[2]; const p3 = isPointer[3]; const p4 = isPointer[4]; + wrapper = function(a0, a1, a2, a3, a4) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 5) { + throwArg(`Invalid argument count: expected 5, got ${arguments.length}`); + } + let v0, v1, v2, v3, v4; + if (p0) { v0 = coercePointer(a0, 0); if (v0 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4); } + else { v0 = validateAndCoerce(i0, a0, 0); } + if (p1) { v1 = coercePointer(a1, 1); if (v1 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4); } + else { v1 = validateAndCoerce(i1, a1, 1); } + if (p2) { v2 = coercePointer(a2, 2); if (v2 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4); } + else { v2 = validateAndCoerce(i2, a2, 2); } + if (p3) { v3 = coercePointer(a3, 3); if (v3 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4); } + else { v3 = validateAndCoerce(i3, a3, 3); } + if (p4) { v4 = coercePointer(a4, 4); if (v4 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4); } + else { v4 = validateAndCoerce(i4, a4, 4); } + return rawFn(v0, v1, v2, v3, v4); + }; + break; + } + case 6: { + const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; const i4 = infos[4]; const i5 = infos[5]; + const p0 = isPointer[0]; const p1 = isPointer[1]; const p2 = isPointer[2]; const p3 = isPointer[3]; const p4 = isPointer[4]; const p5 = isPointer[5]; + wrapper = function(a0, a1, a2, a3, a4, a5) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (arguments.length !== 6) { + throwArg(`Invalid argument count: expected 6, got ${arguments.length}`); + } + let v0, v1, v2, v3, v4, v5; + if (p0) { v0 = coercePointer(a0, 0); if (v0 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4, a5); } + else { v0 = validateAndCoerce(i0, a0, 0); } + if (p1) { v1 = coercePointer(a1, 1); if (v1 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4, a5); } + else { v1 = validateAndCoerce(i1, a1, 1); } + if (p2) { v2 = coercePointer(a2, 2); if (v2 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4, a5); } + else { v2 = validateAndCoerce(i2, a2, 2); } + if (p3) { v3 = coercePointer(a3, 3); if (v3 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4, a5); } + else { v3 = validateAndCoerce(i3, a3, 3); } + if (p4) { v4 = coercePointer(a4, 4); if (v4 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4, a5); } + else { v4 = validateAndCoerce(i4, a4, 4); } + if (p5) { v5 = coercePointer(a5, 5); if (v5 === SLOW_PATH) return slowInvoke(a0, a1, a2, a3, a4, a5); } + else { v5 = validateAndCoerce(i5, a5, 5); } + return rawFn(v0, v1, v2, v3, v4, v5); + }; + break; + } + default: + // 7+ args: per-call array allocation is unavoidable in the rest-args + // path. Specializing further is diminishing returns; large-arity FFI + // calls are rare. + wrapper = function(...args) { + if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); + if (args.length !== nargs) { + throwArg(`Invalid argument count: expected ${nargs}, got ${args.length}`); + } + const out = []; + for (let i = 0; i < nargs; i++) { + if (isPointer[i]) { + const v = coercePointer(args[i], i); + if (v === SLOW_PATH) { + return ReflectApply(slowInvoke, undefined, args); + } + ArrayPrototypePush(out, v); + } else { + ArrayPrototypePush(out, validateAndCoerce(infos[i], args[i], i)); + } + } + return ReflectApply(rawFn, undefined, out); + }; + break; + } + return inheritMetadata(wrapper, rawFn, nargs); +} + +function sigParams(sig) { + return sig.parameters ?? sig.arguments ?? []; +} +function sigResult(sig) { + return sig.result ?? sig.return ?? sig.returns ?? 'void'; +} + +const rawGetFunction = DynamicLibrary.prototype.getFunction; +const rawGetFunctions = DynamicLibrary.prototype.getFunctions; + +DynamicLibrary.prototype.getFunction = function getFunction(name, sig) { + const raw = FunctionPrototypeCall(rawGetFunction, this, name, sig); + return buildWrapper(raw, sigParams(sig), sigResult(sig)); +}; + +DynamicLibrary.prototype.getFunctions = function getFunctions(definitions) { + const raw = definitions === undefined ? + FunctionPrototypeCall(rawGetFunctions, this) : + FunctionPrototypeCall(rawGetFunctions, this, definitions); + if (raw === undefined || raw === null) return raw; + const keys = ObjectKeys(raw); + const out = { __proto__: null }; + for (let i = 0; i < keys.length; i++) { + const name = keys[i]; + if (definitions === undefined) { + out[name] = buildWrapper(raw[name]); + } else { + const sig = definitions[name]; + out[name] = buildWrapper(raw[name], sigParams(sig), sigResult(sig)); + } + } + return out; +}; + +{ + const desc = ObjectGetOwnPropertyDescriptor( + DynamicLibrary.prototype, 'functions'); + if (desc === undefined || !desc.get) { + throw new ERR_INTERNAL_ASSERTION( + 'FFI: DynamicLibrary.prototype.functions accessor not found'); + } + const origGetter = desc.get; + ObjectDefineProperty(DynamicLibrary.prototype, 'functions', { + __proto__: null, + configurable: true, + enumerable: desc.enumerable, + get() { + const raw = FunctionPrototypeCall(origGetter, this); + if (raw === undefined || raw === null) return raw; + const wrapped = { __proto__: null }; + const keys = ObjectKeys(raw); + for (let i = 0; i < keys.length; i++) { + const name = keys[i]; + wrapped[name] = buildWrapper(raw[name]); + } + return wrapped; + }, + }); +} + +module.exports = { buildWrapper }; diff --git a/lib/internal/ffi-shared-buffer.js b/lib/internal/ffi-shared-buffer.js deleted file mode 100644 index bce51fd79959dd..00000000000000 --- a/lib/internal/ffi-shared-buffer.js +++ /dev/null @@ -1,635 +0,0 @@ -'use strict'; - -const { - DataView, - DataViewPrototypeGetBigInt64, - DataViewPrototypeGetBigUint64, - DataViewPrototypeGetFloat32, - DataViewPrototypeGetFloat64, - DataViewPrototypeGetInt16, - DataViewPrototypeGetInt32, - DataViewPrototypeGetInt8, - DataViewPrototypeGetUint16, - DataViewPrototypeGetUint32, - DataViewPrototypeGetUint8, - DataViewPrototypeSetBigInt64, - DataViewPrototypeSetBigUint64, - DataViewPrototypeSetFloat32, - DataViewPrototypeSetFloat64, - DataViewPrototypeSetInt16, - DataViewPrototypeSetInt32, - DataViewPrototypeSetInt8, - DataViewPrototypeSetUint16, - DataViewPrototypeSetUint32, - DataViewPrototypeSetUint8, - FunctionPrototypeCall, - NumberIsInteger, - ObjectDefineProperty, - ObjectGetOwnPropertyDescriptor, - ObjectKeys, - ReflectApply, - TypeError, -} = primordials; - -const { - codes: { - ERR_INTERNAL_ASSERTION, - }, -} = require('internal/errors'); - -const { - DynamicLibrary, - charIsSigned, - kSbInvokeSlow, - kSbParams, - kSbResult, - kSbSharedBuffer, - uintptrMax, -} = internalBinding('ffi'); - -// Validator fields (`min`, `max`, `label`) must mirror `ToFFIArgument` in -// `src/ffi/types.cc` so the fast and slow paths produce identical errors. -const sI8 = DataViewPrototypeSetInt8; -const gI8 = DataViewPrototypeGetInt8; -const sU8 = DataViewPrototypeSetUint8; -const gU8 = DataViewPrototypeGetUint8; -const sI16 = DataViewPrototypeSetInt16; -const gI16 = DataViewPrototypeGetInt16; -const sU16 = DataViewPrototypeSetUint16; -const gU16 = DataViewPrototypeGetUint16; -const sI32 = DataViewPrototypeSetInt32; -const gI32 = DataViewPrototypeGetInt32; -const sU32 = DataViewPrototypeSetUint32; -const gU32 = DataViewPrototypeGetUint32; -const sI64 = DataViewPrototypeSetBigInt64; -const gI64 = DataViewPrototypeGetBigInt64; -const sU64 = DataViewPrototypeSetBigUint64; -const gU64 = DataViewPrototypeGetBigUint64; -const sF32 = DataViewPrototypeSetFloat32; -const gF32 = DataViewPrototypeGetFloat32; -const sF64 = DataViewPrototypeSetFloat64; -const gF64 = DataViewPrototypeGetFloat64; - -const sbTypeInfo = { - __proto__: null, - i8: { set: sI8, get: gI8, kind: 'int', min: -128, max: 127, label: 'an int8' }, - int8: { set: sI8, get: gI8, kind: 'int', min: -128, max: 127, label: 'an int8' }, - char: charIsSigned ? - { set: sI8, get: gI8, kind: 'int', min: -128, max: 127, label: 'an int8' } : - { set: sU8, get: gU8, kind: 'int', min: 0, max: 255, label: 'a uint8' }, - u8: { set: sU8, get: gU8, kind: 'int', min: 0, max: 255, label: 'a uint8' }, - uint8: { set: sU8, get: gU8, kind: 'int', min: 0, max: 255, label: 'a uint8' }, - bool: { set: sU8, get: gU8, kind: 'int', min: 0, max: 255, label: 'a uint8' }, - i16: { set: sI16, get: gI16, kind: 'int', min: -32768, max: 32767, label: 'an int16' }, - int16: { set: sI16, get: gI16, kind: 'int', min: -32768, max: 32767, label: 'an int16' }, - u16: { set: sU16, get: gU16, kind: 'int', min: 0, max: 65535, label: 'a uint16' }, - uint16: { set: sU16, get: gU16, kind: 'int', min: 0, max: 65535, label: 'a uint16' }, - i32: { set: sI32, get: gI32, kind: 'int', min: -2147483648, max: 2147483647, label: 'an int32' }, - int32: { set: sI32, get: gI32, kind: 'int', min: -2147483648, max: 2147483647, label: 'an int32' }, - u32: { set: sU32, get: gU32, kind: 'int', min: 0, max: 4294967295, label: 'a uint32' }, - uint32: { set: sU32, get: gU32, kind: 'int', min: 0, max: 4294967295, label: 'a uint32' }, - i64: { set: sI64, get: gI64, kind: 'i64', label: 'an int64' }, - int64: { set: sI64, get: gI64, kind: 'i64', label: 'an int64' }, - u64: { set: sU64, get: gU64, kind: 'u64', label: 'a uint64' }, - uint64: { set: sU64, get: gU64, kind: 'u64', label: 'a uint64' }, - f32: { set: sF32, get: gF32, kind: 'float', label: 'a float' }, - float: { set: sF32, get: gF32, kind: 'float', label: 'a float' }, - float32: { set: sF32, get: gF32, kind: 'float', label: 'a float' }, - f64: { set: sF64, get: gF64, kind: 'float', label: 'a double' }, - double: { set: sF64, get: gF64, kind: 'float', label: 'a double' }, - float64: { set: sF64, get: gF64, kind: 'float', label: 'a double' }, - pointer: { set: sU64, get: gU64, kind: 'pointer' }, - ptr: { set: sU64, get: gU64, kind: 'pointer' }, - function: { set: sU64, get: gU64, kind: 'pointer' }, - buffer: { set: sU64, get: gU64, kind: 'pointer' }, - arraybuffer: { set: sU64, get: gU64, kind: 'pointer' }, - string: { set: sU64, get: gU64, kind: 'pointer' }, - str: { set: sU64, get: gU64, kind: 'pointer' }, -}; - -const U64_MAX = 0xFFFFFFFFFFFFFFFFn; -const I64_MAX = 0x7FFFFFFFFFFFFFFFn; -const I64_MIN = -0x8000000000000000n; - -// Builds the exact error shape the non-SB implementation (the native FFI -// invoker) produces. -function throwFFIArgError(msg) { - // eslint-disable-next-line no-restricted-syntax - const err = new TypeError(msg); - err.code = 'ERR_INVALID_ARG_VALUE'; - throw err; -} - -function throwFFIArgCountError(expected, actual) { - throwFFIArgError( - `Invalid argument count: expected ${expected}, got ${actual}`); -} - -// Validation and error messages must mirror `ToFFIArgument` in -// `src/ffi/types.cc`. -function writeNumericArg(view, info, offset, arg, index) { - const kind = info.kind; - if (kind === 'int') { - if (typeof arg !== 'number' || !NumberIsInteger(arg) || - arg < info.min || arg > info.max) { - throwFFIArgError(`Argument ${index} must be ${info.label}`); - } - info.set(view, offset, arg, true); - return; - } - if (kind === 'i64') { - if (typeof arg !== 'bigint' || arg < I64_MIN || arg > I64_MAX) { - throwFFIArgError(`Argument ${index} must be ${info.label}`); - } - sI64(view, offset, arg, true); - return; - } - if (kind === 'u64') { - if (typeof arg !== 'bigint' || arg < 0n || arg > U64_MAX) { - throwFFIArgError(`Argument ${index} must be ${info.label}`); - } - sU64(view, offset, arg, true); - return; - } - if (kind === 'float') { - if (typeof arg !== 'number') { - throwFFIArgError(`Argument ${index} must be ${info.label}`); - } - info.set(view, offset, arg, true); - return; - } - - /* c8 ignore start */ - // Unreachable: caller filters out non-numeric kinds. - throw new ERR_INTERNAL_ASSERTION( - `FFI: writeNumericArg reached with unexpected kind="${kind}"`); - /* c8 ignore stop */ -} - -// Returns true on fast-path success, false when the caller must fall back -// to the slow path (strings, Buffers, ArrayBuffers, ArrayBufferViews). -function writePointerArg(view, offset, arg, index) { - if (typeof arg === 'bigint') { - // Bound by `uintptrMax` (not `U64_MAX`) to mirror the slow path: on - // 32-bit platforms a BigInt that doesn't fit in `uintptr_t` would be - // silently truncated by `ReadFFIArgFromBuffer`'s - // `memcpy(..., type->size, ...)`. - if (arg < 0n || arg > uintptrMax) { - throwFFIArgError( - `Argument ${index} must be a non-negative pointer bigint`); - } - sU64(view, offset, arg, true); - return true; - } - if (arg === null || arg === undefined) { - sU64(view, offset, 0n, true); - return true; - } - return false; -} - -// The `pointer` descriptor mirrors the raw function's so user code that -// reassigns `.pointer` keeps working through the wrapper. -function inheritMetadata(wrapper, rawFn, nargs) { - ObjectDefineProperty(wrapper, 'name', { - __proto__: null, value: rawFn.name, configurable: true, - }); - ObjectDefineProperty(wrapper, 'length', { - __proto__: null, value: nargs, configurable: true, - }); - ObjectDefineProperty(wrapper, 'pointer', { - __proto__: null, value: rawFn.pointer, - writable: true, configurable: true, enumerable: true, - }); - return wrapper; -} - -// Reentrancy: the ArrayBuffer is per-function, but `InvokeFunctionSB` copies -// arguments out of it into invocation-local storage before `ffi_call` and -// reads the return value back only after, so nested/reentrant calls into -// the same function are safe. -function wrapWithSharedBuffer(rawFn, parameters, resultType) { - if (rawFn === undefined || rawFn === null) return rawFn; - const buffer = rawFn[kSbSharedBuffer]; - if (buffer === undefined) return rawFn; - - // Callers without explicit signature info (the `functions` accessor - // patch below) rely on the `kSbParams` / `kSbResult` metadata attached - // by the native `CreateFunction`. - if (parameters === undefined) parameters = rawFn[kSbParams]; - if (resultType === undefined) resultType = rawFn[kSbResult]; - // `CreateFunction` always attaches these for SB-eligible functions. - // Missing here means the native side and this wrapper are out of sync. - /* c8 ignore start */ - if (parameters === undefined || resultType === undefined) { - throw new ERR_INTERNAL_ASSERTION( - 'FFI: shared-buffer raw function is missing kSbParams or kSbResult'); - } - /* c8 ignore stop */ - - const slowInvoke = rawFn[kSbInvokeSlow]; - const view = new DataView(buffer); - let retGetter = null; - if (resultType !== 'void') { - const retInfo = sbTypeInfo[resultType]; - /* c8 ignore start */ - if (retInfo === undefined) { - throw new ERR_INTERNAL_ASSERTION( - `FFI: shared-buffer type table missing entry for result type "${resultType}"`); - } - /* c8 ignore stop */ - retGetter = retInfo.get; - } - - const nargs = parameters.length; - const argInfos = []; - const argOffsets = []; - let anyPointer = false; - for (let i = 0; i < nargs; i++) { - const info = sbTypeInfo[parameters[i]]; - /* c8 ignore start */ - if (info === undefined) { - throw new ERR_INTERNAL_ASSERTION( - `FFI: shared-buffer type table missing entry for parameter type "${parameters[i]}"`); - } - /* c8 ignore stop */ - // Push the `sbTypeInfo` entry directly (entries with the same `kind` - // share a shape, keeping `writeNumericArg`'s call sites - // low-polymorphism) and store offsets in a parallel array to avoid - // per-arg object cloning. - argInfos.push(info); - argOffsets.push(8 * (i + 1)); - if (info.kind === 'pointer') anyPointer = true; - } - - let wrapper; - if (anyPointer) { - // Pointer signatures need a per-arg runtime type check and fall back - // to the native slow-path invoker for non-BigInt pointer arguments, - // so arity specialization wouldn't buy much here. - /* c8 ignore start */ - if (slowInvoke === undefined) { - throw new ERR_INTERNAL_ASSERTION( - 'FFI: shared-buffer raw function with pointer arguments is ' + - 'missing kSbInvokeSlow'); - } - /* c8 ignore stop */ - wrapper = function(...args) { - if (args.length !== nargs) { - throwFFIArgCountError(nargs, args.length); - } - for (let i = 0; i < nargs; i++) { - const info = argInfos[i]; - const offset = argOffsets[i]; - if (info.kind === 'pointer') { - if (!writePointerArg(view, offset, args[i], i)) { - return ReflectApply(slowInvoke, undefined, args); - } - } else { - writeNumericArg(view, info, offset, args[i], i); - } - } - rawFn(); - return retGetter === null ? undefined : retGetter(view, 0, true); - }; - } else { - // Arity specialization avoids the per-call `Array` allocation of - // `...args`; the void/non-void split removes a per-call branch on - // `retGetter`. - wrapper = buildNumericWrapper( - rawFn, view, argInfos, argOffsets, nargs, retGetter); - } - return inheritMetadata(wrapper, rawFn, nargs); -} - -// Specialized for nargs 0..6 with a generic rest-params fallback for 7+. -// Per-arg type info and offsets are captured into closure locals so the -// hot path reads a single variable per arg. This is admittedly pretty weird -// but it's the result of lots of perf-hunting. -function buildNumericWrapper( - rawFn, view, argInfos, argOffsets, nargs, retGetter) { - // `IsSBEligibleSignature` on the native side rejects 0-arg signatures, - // so this branch is unreachable today. It's kept as defense-in-depth - // for when that filter changes or for programmatic callers that hand a - // 0-arg signature through `wrapWithSharedBuffer` directly. - /* c8 ignore start */ - if (nargs === 0) { - if (retGetter === null) { - return function() { - if (arguments.length !== 0) { - throwFFIArgCountError(0, arguments.length); - } - rawFn(); - }; - } - return function() { - if (arguments.length !== 0) { - throwFFIArgCountError(0, arguments.length); - } - rawFn(); - return retGetter(view, 0, true); - }; - } - /* c8 ignore stop */ - if (nargs === 1) { - const i0 = argInfos[0]; - const o0 = argOffsets[0]; - if (retGetter === null) { - return function(a0) { - if (arguments.length !== 1) { - throwFFIArgCountError(1, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - rawFn(); - }; - } - return function(a0) { - if (arguments.length !== 1) { - throwFFIArgCountError(1, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - rawFn(); - return retGetter(view, 0, true); - }; - } - if (nargs === 2) { - const i0 = argInfos[0]; - const i1 = argInfos[1]; - const o0 = argOffsets[0]; - const o1 = argOffsets[1]; - if (retGetter === null) { - return function(a0, a1) { - if (arguments.length !== 2) { - throwFFIArgCountError(2, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - rawFn(); - }; - } - return function(a0, a1) { - if (arguments.length !== 2) { - throwFFIArgCountError(2, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - rawFn(); - return retGetter(view, 0, true); - }; - } - if (nargs === 3) { - const i0 = argInfos[0]; - const i1 = argInfos[1]; - const i2 = argInfos[2]; - const o0 = argOffsets[0]; - const o1 = argOffsets[1]; - const o2 = argOffsets[2]; - if (retGetter === null) { - return function(a0, a1, a2) { - if (arguments.length !== 3) { - throwFFIArgCountError(3, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - rawFn(); - }; - } - return function(a0, a1, a2) { - if (arguments.length !== 3) { - throwFFIArgCountError(3, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - rawFn(); - return retGetter(view, 0, true); - }; - } - if (nargs === 4) { - const i0 = argInfos[0]; - const i1 = argInfos[1]; - const i2 = argInfos[2]; - const i3 = argInfos[3]; - const o0 = argOffsets[0]; - const o1 = argOffsets[1]; - const o2 = argOffsets[2]; - const o3 = argOffsets[3]; - if (retGetter === null) { - return function(a0, a1, a2, a3) { - if (arguments.length !== 4) { - throwFFIArgCountError(4, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - writeNumericArg(view, i3, o3, a3, 3); - rawFn(); - }; - } - return function(a0, a1, a2, a3) { - if (arguments.length !== 4) { - throwFFIArgCountError(4, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - writeNumericArg(view, i3, o3, a3, 3); - rawFn(); - return retGetter(view, 0, true); - }; - } - if (nargs === 5) { - const i0 = argInfos[0]; - const i1 = argInfos[1]; - const i2 = argInfos[2]; - const i3 = argInfos[3]; - const i4 = argInfos[4]; - const o0 = argOffsets[0]; - const o1 = argOffsets[1]; - const o2 = argOffsets[2]; - const o3 = argOffsets[3]; - const o4 = argOffsets[4]; - if (retGetter === null) { - return function(a0, a1, a2, a3, a4) { - if (arguments.length !== 5) { - throwFFIArgCountError(5, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - writeNumericArg(view, i3, o3, a3, 3); - writeNumericArg(view, i4, o4, a4, 4); - rawFn(); - }; - } - return function(a0, a1, a2, a3, a4) { - if (arguments.length !== 5) { - throwFFIArgCountError(5, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - writeNumericArg(view, i3, o3, a3, 3); - writeNumericArg(view, i4, o4, a4, 4); - rawFn(); - return retGetter(view, 0, true); - }; - } - if (nargs === 6) { - const i0 = argInfos[0]; - const i1 = argInfos[1]; - const i2 = argInfos[2]; - const i3 = argInfos[3]; - const i4 = argInfos[4]; - const i5 = argInfos[5]; - const o0 = argOffsets[0]; - const o1 = argOffsets[1]; - const o2 = argOffsets[2]; - const o3 = argOffsets[3]; - const o4 = argOffsets[4]; - const o5 = argOffsets[5]; - if (retGetter === null) { - return function(a0, a1, a2, a3, a4, a5) { - if (arguments.length !== 6) { - throwFFIArgCountError(6, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - writeNumericArg(view, i3, o3, a3, 3); - writeNumericArg(view, i4, o4, a4, 4); - writeNumericArg(view, i5, o5, a5, 5); - rawFn(); - }; - } - return function(a0, a1, a2, a3, a4, a5) { - if (arguments.length !== 6) { - throwFFIArgCountError(6, arguments.length); - } - writeNumericArg(view, i0, o0, a0, 0); - writeNumericArg(view, i1, o1, a1, 1); - writeNumericArg(view, i2, o2, a2, 2); - writeNumericArg(view, i3, o3, a3, 3); - writeNumericArg(view, i4, o4, a4, 4); - writeNumericArg(view, i5, o5, a5, 5); - rawFn(); - return retGetter(view, 0, true); - }; - } - // 7+ args: further specialization is diminishing returns and bloats - // this builder. - if (retGetter === null) { - return function(...args) { - if (args.length !== nargs) { - throwFFIArgCountError(nargs, args.length); - } - for (let i = 0; i < nargs; i++) { - writeNumericArg(view, argInfos[i], argOffsets[i], args[i], i); - } - rawFn(); - }; - } - return function(...args) { - if (args.length !== nargs) { - throwFFIArgCountError(nargs, args.length); - } - for (let i = 0; i < nargs; i++) { - writeNumericArg(view, argInfos[i], argOffsets[i], args[i], i); - } - rawFn(); - return retGetter(view, 0, true); - }; -} - -// Accept-set mirrors the native `ParseFunctionSignature` in -// `src/ffi/types.cc`. `ParseFunctionSignature` additionally throws when -// multiple aliases are set at once. The wrapper runs before the native -// call, so those conflicts still surface from the native side regardless -// of which alias we happen to read here. -function sigParams(sig) { - return sig.parameters ?? sig.arguments ?? []; -} - -function sigResult(sig) { - return sig.result ?? sig.return ?? sig.returns ?? 'void'; -} - -// The native invoker for SB-eligible symbols is `InvokeFunctionSB`, which -// reads arguments from the shared buffer populated by -// `wrapWithSharedBuffer`. These patches make sure every path that surfaces -// a raw SB-eligible function to user code (`getFunction`, `getFunctions`, -// and the `functions` accessor) returns the wrapper instead. -const rawGetFunction = DynamicLibrary.prototype.getFunction; -const rawGetFunctions = DynamicLibrary.prototype.getFunctions; - -DynamicLibrary.prototype.getFunction = function getFunction(name, sig) { - // Native `DynamicLibrary::GetFunction` validates `sig`, so by the time - // we have `raw` we know `sig` is a valid object. - const raw = FunctionPrototypeCall(rawGetFunction, this, name, sig); - return wrapWithSharedBuffer(raw, sigParams(sig), sigResult(sig)); -}; - -DynamicLibrary.prototype.getFunctions = function getFunctions(definitions) { - // Native `GetFunctions` switches on `args.Length() > 0`. Zero args - // returns every cached function, one arg requires an object. Forwarding - // `undefined` would fail the object check, so drop it when omitted. - const raw = definitions === undefined ? - FunctionPrototypeCall(rawGetFunctions, this) : - FunctionPrototypeCall(rawGetFunctions, this, definitions); - if (raw === undefined || raw === null) return raw; - const keys = ObjectKeys(raw); - const out = { __proto__: null }; - for (let i = 0; i < keys.length; i++) { - const name = keys[i]; - // No `definitions`: native side returned every cached function, so we - // wrap using each function's own `kSbParams` / `kSbResult` metadata - // (same fallback as the `functions` accessor). - if (definitions === undefined) { - out[name] = wrapWithSharedBuffer(raw[name]); - } else { - const sig = definitions[name]; - out[name] = wrapWithSharedBuffer(raw[name], sigParams(sig), sigResult(sig)); - } - } - return out; -}; - -{ - // The native side installs `functions` as an accessor returning raw - // functions. Rewrap each access so `lib.functions.foo(...)` goes through - // the SB wrapper instead of invoking the fast path against an - // uninitialized buffer. - const functionsDescriptor = - ObjectGetOwnPropertyDescriptor(DynamicLibrary.prototype, 'functions'); - /* c8 ignore start */ - if (functionsDescriptor === undefined || !functionsDescriptor.get) { - // Missing getter means the native and JS sides are out of sync; silently - // skipping the patch would expose the fast-path-against-uninitialized-buffer - // footgun this whole block exists to prevent. - throw new ERR_INTERNAL_ASSERTION( - 'FFI: DynamicLibrary.prototype.functions accessor not found or has no getter'); - } - /* c8 ignore stop */ - const origGetter = functionsDescriptor.get; - ObjectDefineProperty(DynamicLibrary.prototype, 'functions', { - __proto__: null, - configurable: true, - enumerable: functionsDescriptor.enumerable, - get() { - const raw = FunctionPrototypeCall(origGetter, this); - if (raw === undefined || raw === null) return raw; - const wrapped = { __proto__: null }; - const keys = ObjectKeys(raw); - for (let i = 0; i < keys.length; i++) { - const name = keys[i]; - wrapped[name] = wrapWithSharedBuffer(raw[name]); - } - return wrapped; - }, - }); -} - -module.exports = { - wrapWithSharedBuffer, -}; diff --git a/node.gyp b/node.gyp index 77acf529698db1..17bc32216be483 100644 --- a/node.gyp +++ b/node.gyp @@ -39,6 +39,7 @@ 'node_use_quic%': 'false', 'node_use_sqlite%': 'true', 'node_use_ffi%': 'false', + 'node_use_ffi_fastcall%': 'false', 'node_use_v8_platform%': 'true', 'node_v8_options%': '', 'node_write_snapshot_as_string_literals': 'true', @@ -472,6 +473,13 @@ 'src/ffi/types.cc', 'src/ffi/types.h', ], + 'node_ffi_fastcall_sources': [ + 'src/ffi/fastcall/jit_memory.cc', + 'src/ffi/fastcall/jit_memory.h', + 'src/ffi/fastcall/stub_emitter.h', + 'src/ffi/fastcall/cfunction_info.cc', + 'src/ffi/fastcall/cfunction_info.h', + ], 'node_mksnapshot_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_mksnapshot<(EXECUTABLE_SUFFIX)', 'node_js2c_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_js2c<(EXECUTABLE_SUFFIX)', 'conditions': [ @@ -1010,6 +1018,24 @@ 'deps/libffi/libffi.gyp:libffi', ], }], + [ 'node_use_ffi_fastcall=="true"', { + 'defines': [ 'HAVE_FFI_FASTCALL=1' ], + 'sources': [ '<@(node_ffi_fastcall_sources)' ], + 'conditions': [ + [ 'target_arch=="arm64"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_aarch64.cc' ], + }], + [ 'target_arch=="x64" and OS!="win"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_sysv.cc' ], + }], + [ 'target_arch=="x64" and OS=="win"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_win.cc' ], + }], + [ 'target_arch=="arm"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_arm.cc' ], + }], + ], + }], ], }], [ 'node_shared=="true" and node_module_version!="" and OS!="win"', { @@ -1077,6 +1103,24 @@ 'deps/libffi/libffi.gyp:libffi', ], }], + [ 'node_use_ffi_fastcall=="true"', { + 'defines': [ 'HAVE_FFI_FASTCALL=1' ], + 'sources': [ '<@(node_ffi_fastcall_sources)' ], + 'conditions': [ + [ 'target_arch=="arm64"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_aarch64.cc' ], + }], + [ 'target_arch=="x64" and OS!="win"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_sysv.cc' ], + }], + [ 'target_arch=="x64" and OS=="win"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_win.cc' ], + }], + [ 'target_arch=="arm"', { + 'sources': [ 'src/ffi/fastcall/stub_emitter_arm.cc' ], + }], + ], + }], ], }], [ 'node_use_quic=="true"', { @@ -1408,6 +1452,18 @@ }, { 'sources!': [ '<@(node_cctest_quic_sources)' ], }], + [ 'node_use_ffi_fastcall=="true"', { + 'defines': [ + 'HAVE_FFI_FASTCALL=1', + ], + }, { + 'sources!': [ + 'test/cctest/test_ffi_fastcall_cfunction.cc', + 'test/cctest/test_ffi_fastcall_eligibility.cc', + 'test/cctest/test_ffi_fastcall_emitter.cc', + 'test/cctest/test_ffi_fastcall_jit.cc', + ], + }], ['v8_enable_inspector==1', { 'defines': [ 'HAVE_INSPECTOR=1', diff --git a/src/env_properties.h b/src/env_properties.h index 0fc7b2b66179e4..5417ad72d7c62f 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -46,10 +46,10 @@ #define PER_ISOLATE_SYMBOL_PROPERTIES(V) \ V(fs_use_promises_symbol, "fs_use_promises_symbol") \ V(async_id_symbol, "async_id_symbol") \ - V(ffi_sb_shared_buffer_symbol, "ffi_sb_shared_buffer_symbol") \ - V(ffi_sb_invoke_slow_symbol, "ffi_sb_invoke_slow_symbol") \ - V(ffi_sb_params_symbol, "ffi_sb_params_symbol") \ - V(ffi_sb_result_symbol, "ffi_sb_result_symbol") \ + V(ffi_fastcall_alive_symbol, "ffi.fastcall.alive") \ + V(ffi_fastcall_invoke_slow_symbol, "ffi.fastcall.invokeSlow") \ + V(ffi_fastcall_params_symbol, "ffi.fastcall.params") \ + V(ffi_fastcall_result_symbol, "ffi.fastcall.result") \ V(constructor_key_symbol, "constructor_key_symbol") \ V(handle_onclose_symbol, "handle_onclose") \ V(no_message_symbol, "no_message_symbol") \ diff --git a/src/ffi/fastcall/cfunction_info.cc b/src/ffi/fastcall/cfunction_info.cc new file mode 100644 index 00000000000000..33866c391e253a --- /dev/null +++ b/src/ffi/fastcall/cfunction_info.cc @@ -0,0 +1,135 @@ +#include "ffi/fastcall/cfunction_info.h" + +#ifdef HAVE_FFI_FASTCALL + +#include "ffi/types.h" +#include "util.h" + +namespace node::ffi::fastcall { + +namespace { + +// Map an ffi_type to (CTypeInfo::Type, ArgClass) for argument positions. +struct ArgMapping { + v8::CTypeInfo::Type ctype; + ArgClass cls; +}; + +// Map an ffi_type to (CTypeInfo::Type, ResultClass) for the return position. +struct ResultMapping { + v8::CTypeInfo::Type ctype; + ResultClass cls; +}; + +ArgMapping MapArgType(ffi_type* t) { + using T = v8::CTypeInfo::Type; + if (t == &ffi_type_sint8 || + t == &ffi_type_sint16 || + t == &ffi_type_sint32) return {T::kInt32, ArgClass::kGP}; + if (t == &ffi_type_uint8 || + t == &ffi_type_uint16) return {T::kInt32, ArgClass::kGP}; + if (t == &ffi_type_uint32) return {T::kUint32, ArgClass::kGP}; + if (t == &ffi_type_sint64) return {T::kInt64, ArgClass::kGP}; + if (t == &ffi_type_uint64 || + t == &ffi_type_pointer) return {T::kUint64, ArgClass::kGP}; + if (t == &ffi_type_float) return {T::kFloat32, ArgClass::kFP}; + if (t == &ffi_type_double) return {T::kFloat64, ArgClass::kFP}; + // Unreachable if eligibility was checked before calling BuildCFunctionInfo. + UNREACHABLE("FFI fast-call: MapArgType called with type that passed " + "eligibility but has no arg mapping"); +} + +ResultMapping MapResultType(ffi_type* t) { + using T = v8::CTypeInfo::Type; + if (t == &ffi_type_void) return {T::kVoid, ResultClass::kVoid}; + if (t == &ffi_type_sint8 || + t == &ffi_type_sint16 || + t == &ffi_type_sint32) return {T::kInt32, ResultClass::kGP}; + if (t == &ffi_type_uint8 || + t == &ffi_type_uint16) return {T::kInt32, ResultClass::kGP}; + if (t == &ffi_type_uint32) return {T::kUint32, ResultClass::kGP}; + if (t == &ffi_type_sint64) return {T::kInt64, ResultClass::kGP}; + if (t == &ffi_type_uint64 || + t == &ffi_type_pointer) return {T::kUint64, ResultClass::kGP}; + if (t == &ffi_type_float) return {T::kFloat32, ResultClass::kFP}; + if (t == &ffi_type_double) return {T::kFloat64, ResultClass::kFP}; + UNREACHABLE("FFI fast-call: MapResultType called with type that passed " + "eligibility but has no result mapping"); +} + +} // namespace + +CFunctionInfoBundle::~CFunctionInfoBundle() { + ::operator delete[](arg_types); + delete info; +} + +CFunctionInfoBundle::CFunctionInfoBundle(CFunctionInfoBundle&& o) noexcept + : info(o.info), + arg_types(o.arg_types), + arg_classes(std::move(o.arg_classes)), + result_class(o.result_class) { + o.info = nullptr; + o.arg_types = nullptr; + o.result_class = ResultClass::kVoid; +} + +CFunctionInfoBundle& CFunctionInfoBundle::operator=( + CFunctionInfoBundle&& o) noexcept { + if (this != &o) { + ::operator delete[](arg_types); + delete info; + info = o.info; + arg_types = o.arg_types; + arg_classes = std::move(o.arg_classes); + result_class = o.result_class; + o.info = nullptr; + o.arg_types = nullptr; + o.result_class = ResultClass::kVoid; + } + return *this; +} + +CFunctionInfoBundle BuildCFunctionInfo(const FFIFunction& fn) { + using T = v8::CTypeInfo::Type; + static_assert(std::is_trivially_destructible_v, + "CTypeInfo must be trivially destructible for placement-new " + "array without explicit element destruction"); + CFunctionInfoBundle b; + + // V8 wants the receiver as arg[0]. Total CTypeInfo entries = args + 1. + // CTypeInfo has no default constructor, so we allocate raw storage and + // placement-new each element individually. + size_t n = fn.args.size() + 1; + void* raw = ::operator new[](n * sizeof(v8::CTypeInfo)); + b.arg_types = static_cast(raw); + // Placement-new the receiver slot. + new (&b.arg_types[0]) v8::CTypeInfo(T::kV8Value); + // Placement-new the arg slots with a placeholder; overwritten below. + for (size_t i = 1; i < n; ++i) { + new (&b.arg_types[i]) v8::CTypeInfo(T::kVoid); + } + + b.arg_classes.reserve(fn.args.size()); + for (size_t i = 0; i < fn.args.size(); ++i) { + auto m = MapArgType(fn.args[i]); + b.arg_types[i + 1] = v8::CTypeInfo(m.ctype); + b.arg_classes.push_back(m.cls); + } + + auto rm = MapResultType(fn.return_type); + b.result_class = rm.cls; + v8::CTypeInfo return_info(rm.ctype); + + b.info = new v8::CFunctionInfo( + return_info, + static_cast(n), + b.arg_types, + v8::CFunctionInfo::Int64Representation::kBigInt); + + return b; +} + +} // namespace node::ffi::fastcall + +#endif // HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/cfunction_info.h b/src/ffi/fastcall/cfunction_info.h new file mode 100644 index 00000000000000..b2718fd4c867c2 --- /dev/null +++ b/src/ffi/fastcall/cfunction_info.h @@ -0,0 +1,39 @@ +#pragma once + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS && \ + defined(HAVE_FFI_FASTCALL) + +#include + +#include "ffi/fastcall/stub_emitter.h" +#include "node_ffi.h" +#include "v8-fast-api-calls.h" + +namespace node::ffi::fastcall { + +// Owns dynamically-allocated CFunctionInfo + CTypeInfo[] arrays for a +// single FFI function. Lifetime is tied to the FFIFunctionInfo. Free +// on weak-callback / dynamic library teardown. +struct CFunctionInfoBundle { + v8::CFunctionInfo* info = nullptr; + v8::CTypeInfo* arg_types = nullptr; + std::vector arg_classes; + ResultClass result_class = ResultClass::kVoid; + + CFunctionInfoBundle() = default; + ~CFunctionInfoBundle(); + CFunctionInfoBundle(const CFunctionInfoBundle&) = delete; + CFunctionInfoBundle& operator=(const CFunctionInfoBundle&) = delete; + CFunctionInfoBundle(CFunctionInfoBundle&&) noexcept; + CFunctionInfoBundle& operator=(CFunctionInfoBundle&&) noexcept; +}; + +// Build a CFunctionInfo + CTypeInfo[] for `fn`. The caller must already have +// verified `fn` via IsFastCallEligible before calling this. With +// -fno-exceptions, `new` aborts on OOM rather than throwing, so this function +// never fails — it returns directly instead of using optional. +CFunctionInfoBundle BuildCFunctionInfo(const FFIFunction& fn); + +} // namespace node::ffi::fastcall + +#endif // NODE_WANT_INTERNALS && HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/jit_memory.cc b/src/ffi/fastcall/jit_memory.cc new file mode 100644 index 00000000000000..50aa8e5699dcdd --- /dev/null +++ b/src/ffi/fastcall/jit_memory.cc @@ -0,0 +1,292 @@ +#include "ffi/fastcall/jit_memory.h" + +#ifdef HAVE_FFI_FASTCALL + +#include +#include + +#include "node_internals.h" +#include "util.h" + +#if defined(__POSIX__) +#include +#include +#endif + +#if defined(_WIN32) +#include +#endif + +namespace node::ffi::fastcall { + +namespace { + +size_t RoundUp(size_t v, size_t align) { + return (v + align - 1) & ~(align - 1); +} + +// Allocate a region that can later become executable. On macOS/arm64 the +// mapping must be created with MAP_JIT at allocation time; only then can +// mprotect(PROT_READ|PROT_EXEC) succeed after the Hardened Runtime check. +void* AllocJitPages(size_t size) { +#if defined(__APPLE__) + int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT; + void* p = mmap(nullptr, size, PROT_READ | PROT_WRITE, flags, -1, 0); + return (p == MAP_FAILED) ? nullptr : p; +#elif defined(__POSIX__) + void* p = mmap(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + return (p == MAP_FAILED) ? nullptr : p; +#elif defined(_WIN32) + return VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE); +#else + return nullptr; +#endif +} + +bool MakePageRX(void* addr, size_t size) { +#if defined(__POSIX__) + if (mprotect(addr, size, PROT_READ | PROT_EXEC) != 0) return false; + // Flush icache for the full range. + __builtin___clear_cache(static_cast(addr), + static_cast(addr) + size); + return true; +#elif defined(_WIN32) + DWORD old; + if (!VirtualProtect(addr, size, PAGE_EXECUTE_READ, &old)) return false; + FlushInstructionCache(GetCurrentProcess(), addr, size); + return true; +#else + return false; +#endif +} + +void FreeJitPages(void* addr, size_t size) { +#if defined(__POSIX__) + munmap(addr, size); +#elif defined(_WIN32) + VirtualFree(addr, 0, MEM_RELEASE); +#endif +} + +size_t SystemPageSize() { +#if defined(__POSIX__) + return static_cast(sysconf(_SC_PAGESIZE)); +#elif defined(_WIN32) + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwPageSize); +#else + return 4096; +#endif +} + +} // namespace + +JitMemory* JitMemory::Get(v8::Isolate* isolate) { + static JitMemory instance; + // The isolate parameter is retained for API symmetry; the implementation + // uses direct mmap/VirtualAlloc so no per-platform lookup is needed. + std::lock_guard lock(instance.mu_); + if (instance.page_size_ == 0) { + instance.page_size_ = SystemPageSize(); + } + return &instance; +} + +JitMemory::JitMemory() + : page_size_(0), + slot_align_(64), + live_bytes_(0) {} + +void* JitMemory::Allocate(size_t size, size_t* out_alloc_size) { + if (out_alloc_size) *out_alloc_size = 0; + if (size == 0) return nullptr; + + std::lock_guard lock(mu_); + if (page_size_ == 0) return nullptr; // Not initialised. + + size_t need = RoundUp(size, slot_align_); + + if (!pages_.empty()) { + auto& page = pages_.back(); + if (!page.executable && page.bump + need <= page.size) { + void* p = static_cast(page.base) + page.bump; + page.bump += need; + live_bytes_ += need; + if (out_alloc_size) *out_alloc_size = need; + return p; + } + } + + size_t page_bytes = RoundUp(std::max(page_size_, need), page_size_); + void* base = AllocJitPages(page_bytes); + if (base == nullptr) return nullptr; + + pages_.push_back(Page{base, page_bytes, need, false}); + live_bytes_ += need; + if (out_alloc_size) *out_alloc_size = need; + return base; +} + +bool JitMemory::MakeExecutable(void* ptr, size_t alloc_size) { + std::lock_guard lock(mu_); + if (page_size_ == 0) return false; + if (ptr == nullptr || alloc_size == 0) return false; + + uintptr_t addr = reinterpret_cast(ptr); + uintptr_t page_start = addr & ~(page_size_ - 1); + uintptr_t page_end = RoundUp(addr + alloc_size, page_size_); + + if (!MakePageRX(reinterpret_cast(page_start), + page_end - page_start)) { + return false; + } + + for (auto& page : pages_) { + uintptr_t base = reinterpret_cast(page.base); + if (addr >= base && addr < base + page.size) { + page.executable = true; + break; + } + } + return true; +} + +std::optional JitMemory::EmitStub(const uint8_t* code, + size_t size) { + if (code == nullptr || size == 0) return std::nullopt; + + std::lock_guard lock(mu_); + if (page_size_ == 0) return std::nullopt; + + size_t need = RoundUp(size, slot_align_); + + // Find or allocate a writable page. + void* dst = nullptr; + size_t alloc_size = need; + + if (!pages_.empty()) { + auto& page = pages_.back(); + if (!page.executable && page.bump + need <= page.size) { + dst = static_cast(page.base) + page.bump; + page.bump += need; + live_bytes_ += need; + } + } + + if (dst == nullptr) { + size_t page_bytes = RoundUp(std::max(page_size_, need), page_size_); + void* base = AllocJitPages(page_bytes); + if (base == nullptr) return std::nullopt; + pages_.push_back(Page{base, page_bytes, need, false}); + live_bytes_ += need; + dst = base; + } + + std::memcpy(dst, code, size); + + // Make executable — inline the mprotect/flush without re-locking. + uintptr_t addr = reinterpret_cast(dst); + uintptr_t page_start = addr & ~(page_size_ - 1); + uintptr_t page_end = RoundUp(addr + alloc_size, page_size_); + if (!MakePageRX(reinterpret_cast(page_start), + page_end - page_start)) { + // MakePageRX failed (e.g., mprotect rejected by hardened runtime). + // Roll back the live-bytes counter so the leak doesn't show up in + // MemoryTracker. We do NOT roll back page.bump — the failed slot is + // permanently wasted within the page, but the page itself stays in + // pages_ in RW state and the next EmitStub may successfully use the + // rest of it. The mapping is not returned to the OS. + live_bytes_ -= alloc_size; + return std::nullopt; + } + + for (auto& page : pages_) { + uintptr_t base = reinterpret_cast(page.base); + if (addr >= base && addr < base + page.size) { + page.executable = true; + break; + } + } + + return EmittedStub{dst, alloc_size}; +} + +void JitMemory::Free(void* ptr, size_t alloc_size) { + if (ptr == nullptr || alloc_size == 0) return; + std::lock_guard lock(mu_); + CHECK_GE(live_bytes_, alloc_size); + live_bytes_ -= alloc_size; +} + +size_t JitMemory::TotalLiveBytes() const { + std::lock_guard lock(mu_); + return live_bytes_; +} + +void JitMemory::ResetForTesting() { + std::lock_guard lock(mu_); + CHECK_EQ(live_bytes_, 0u); + for (auto& page : pages_) { + FreeJitPages(page.base, page.size); + } + pages_.clear(); +} + +bool JitMemory::SelfTest(v8::Isolate* isolate) { + static std::once_flag once; + static std::atomic result{-1}; + + std::call_once(once, [this, isolate]() { + Get(isolate); + + size_t alloc_size = 0; + void* p = Allocate(16, &alloc_size); + if (p == nullptr) { + result.store(0, std::memory_order_release); + return; + } + + uint8_t* code = static_cast(p); +#if defined(__x86_64__) || defined(_M_X64) + code[0] = 0xb8; code[1] = 0x2a; code[2] = 0x00; + code[3] = 0x00; code[4] = 0x00; + code[5] = 0xc3; +#elif defined(__aarch64__) || defined(_M_ARM64) + uint32_t* w = reinterpret_cast(code); + w[0] = 0x52800540; + w[1] = 0xd65f03c0; +#elif defined(__arm__) || defined(_M_ARM) + uint32_t* w = reinterpret_cast(code); + w[0] = 0xe3a0002a; + w[1] = 0xe12fff1e; +#else + Free(p, alloc_size); + result.store(1, std::memory_order_release); + return; +#endif + + if (!MakeExecutable(p, alloc_size)) { + Free(p, alloc_size); + result.store(0, std::memory_order_release); + return; + } + + using FnPtr = int (*)(); + int got = reinterpret_cast(p)(); + bool ok = (got == 42); + // The page stays executable, but the slot's bytes are no longer + // counted as live. Future allocations skip this page (it's marked + // executable), so the slot just sits unused — that's fine. + Free(p, alloc_size); + result.store(ok ? 1 : 0, std::memory_order_release); + }); + + return result.load(std::memory_order_acquire) == 1; +} + +} // namespace node::ffi::fastcall + +#endif // HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/jit_memory.h b/src/ffi/fastcall/jit_memory.h new file mode 100644 index 00000000000000..204ce268a8854f --- /dev/null +++ b/src/ffi/fastcall/jit_memory.h @@ -0,0 +1,87 @@ +#pragma once + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS && \ + defined(HAVE_FFI_FASTCALL) + +#include +#include +#include +#include +#include + +namespace v8 { class Isolate; } + +namespace node::ffi::fastcall { + +// Handle returned from EmitStub (and from the lower-level Allocate path). +struct EmittedStub { + void* entry; // CFunction.address + size_t alloc_size; +}; + +class JitMemory { + public: + // Returns the singleton. On first call, initialises internal state (page + // size detection). The isolate parameter is retained for API symmetry with + // future callers; the implementation uses direct mmap/VirtualAlloc rather + // than the v8::PageAllocator interface to ensure MAP_JIT on Apple Silicon. + // Subsequent calls may pass nullptr. + static JitMemory* Get(v8::Isolate* isolate); + + // Allocate `size` bytes (rounded up to slot alignment, currently 64). + // Returns nullptr on failure (zero size, allocator unavailable, OOM). + // The returned pointer is in RW state — write the stub bytes, then call + // MakeExecutable. + void* Allocate(size_t size, size_t* out_alloc_size); + + // Transition the page containing `ptr` (extent `alloc_size`) from RW to + // RX, flushing icache as needed. Returns true on success. After this + // returns, the page is full as far as Allocate is concerned — subsequent + // allocations go to a fresh page. + bool MakeExecutable(void* ptr, size_t alloc_size); + + // Decrement the live-byte counter. Memory is not returned to the OS. + void Free(void* ptr, size_t alloc_size); + + // Convenience overload for callers that hold an EmittedStub value. + // Documents the pairing with EmitStub at the type level. + void Free(const EmittedStub& stub) { + Free(stub.entry, stub.alloc_size); + } + + // Total bytes currently allocated to live stubs. + size_t TotalLiveBytes() const; + + // All-in-one: allocate, write code, transition to RX, return handle. + // Holds the mutex across the entire operation; safe under multi-isolate + // concurrent emit. Returns std::nullopt on any failure (OOM, MakeExecutable). + std::optional EmitStub(const uint8_t* code, size_t size); + + // For testing only: free all pages back to the OS. Asserts no live bytes. + void ResetForTesting(); + + // One-time process-wide self-test: allocate, write platform-native + // "return 42", make RX, call. Cached. On failure, sets a flag so future + // Allocate calls return nullptr; callers fall back to libffi. + bool SelfTest(v8::Isolate* isolate); + + private: + JitMemory(); + + struct Page { + void* base; + size_t size; + size_t bump; + bool executable; + }; + + size_t page_size_; // 0 means not yet initialised + size_t slot_align_; + std::vector pages_; + size_t live_bytes_; + mutable std::mutex mu_; +}; + +} // namespace node::ffi::fastcall + +#endif // NODE_WANT_INTERNALS && HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/stub_emitter.h b/src/ffi/fastcall/stub_emitter.h new file mode 100644 index 00000000000000..e4b636d72bfefb --- /dev/null +++ b/src/ffi/fastcall/stub_emitter.h @@ -0,0 +1,49 @@ +#pragma once + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS && \ + defined(HAVE_FFI_FASTCALL) + +#include +#include + +#include "ffi/fastcall/jit_memory.h" + +namespace v8 { class Isolate; } + +namespace node::ffi::fastcall { + +// One slot per native arg of the target. Does not include kVoid — that is +// only meaningful as a return type (see ResultClass). +enum class ArgClass : uint8_t { + kGP, // any 32/64-bit integer or pointer (single GP slot) + kFP, // float or double (single FP slot) + kGPPair, // 64-bit integer on AArch32 (two GP slots; not used on + // AArch64/x86_64 — unsupported in v1) +}; + +// Classification of the return type. Separate from ArgClass to make it +// impossible to pass kVoid as an argument or kGPPair as a result. +enum class ResultClass : uint8_t { + kGP, // integer or pointer return + kFP, // float or double return + kVoid, // void return +}; + +// Build a stub for the target. The receiver is implicit and consumes +// one GP slot from V8 ahead of the listed args. Returns std::nullopt on +// failure (out of JIT memory, signature exceeds ABI cap, unsupported +// platform, kGPPair on non-AArch32). +// +// `target` is the native function pointer to tail-call. +// `args` lists the target's args in order. The receiver is implicit. +// `result` is informational on most ABIs; the stub does not synthesize +// a return value, only forwards. +std::optional EmitForwarder( + v8::Isolate* isolate, + void* target, + const std::vector& args, + ResultClass result); + +} // namespace node::ffi::fastcall + +#endif // NODE_WANT_INTERNALS && HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/stub_emitter_aarch64.cc b/src/ffi/fastcall/stub_emitter_aarch64.cc new file mode 100644 index 00000000000000..4985f3b2869da4 --- /dev/null +++ b/src/ffi/fastcall/stub_emitter_aarch64.cc @@ -0,0 +1,95 @@ +#include "ffi/fastcall/stub_emitter.h" + +#if defined(HAVE_FFI_FASTCALL) && (defined(__aarch64__) || defined(_M_ARM64)) + +#include + +#include "ffi/fastcall/jit_memory.h" + +namespace node::ffi::fastcall { + +namespace { + +constexpr unsigned kMaxGPArgs = 7; // x0..x7 minus receiver + +// AArch64 instruction helpers. All instructions are 32 bits, little-endian. + +// MOV Xd, Xm (encoded as ORR Xd, XZR, Xm) +uint32_t MovReg(unsigned dst, unsigned src) { + return 0xAA0003E0u | (src << 16) | dst; +} + +// MOVZ Xd, #imm16, LSL #shift (shift in {0,16,32,48}) +uint32_t Movz(unsigned dst, uint16_t imm, unsigned shift) { + unsigned hw = shift / 16; + return 0xD2800000u | (hw << 21) | (uint32_t(imm) << 5) | dst; +} + +// MOVK Xd, #imm16, LSL #shift +uint32_t Movk(unsigned dst, uint16_t imm, unsigned shift) { + unsigned hw = shift / 16; + return 0xF2800000u | (hw << 21) | (uint32_t(imm) << 5) | dst; +} + +// BR Xn (unconditional branch to register; tail-call) +uint32_t BrReg(unsigned reg) { + return 0xD61F0000u | (reg << 5); +} + +} // namespace + +std::optional EmitForwarder( + v8::Isolate* isolate, + void* target, + const std::vector& args, + ResultClass /*result*/) { + unsigned gp = 0, fp = 0; + for (auto c : args) { + switch (c) { + case ArgClass::kGP: ++gp; break; + case ArgClass::kFP: ++fp; break; + case ArgClass::kGPPair: return std::nullopt; // unsupported on this ABI + } + } + if (gp > kMaxGPArgs) return std::nullopt; + if (fp > 8) return std::nullopt; + + uint32_t code[16] = {0}; + size_t off = 0; + + // Shift x1..xN -> x0..x(N-1). Forward order is safe: at iteration i + // we emit `MOV xi, x(i+1)`, writing only xi. At iteration i+1 we + // read x(i+2), which has not been written yet. Each register is + // written exactly once, and the read of register k always happens + // before any write to k. + for (unsigned i = 0; i < gp; ++i) { + code[off++] = MovReg(/*dst=*/i, /*src=*/i + 1); + } + + // Load target into x16 (intra-procedure-call scratch reg). Build the + // 64-bit address with MOVZ + up to 3 MOVKs. Always emit MOVZ for the + // bottom 16 bits even if zero; only emit higher MOVKs when their + // bits are non-zero, to keep stubs tight. + uint64_t addr = reinterpret_cast(target); + code[off++] = Movz(16, static_cast(addr & 0xFFFF), 0); + if (((addr >> 16) & 0xFFFF) != 0) { + code[off++] = Movk(16, static_cast((addr >> 16) & 0xFFFF), 16); + } + if (((addr >> 32) & 0xFFFF) != 0) { + code[off++] = Movk(16, static_cast((addr >> 32) & 0xFFFF), 32); + } + if (((addr >> 48) & 0xFFFF) != 0) { + code[off++] = Movk(16, static_cast((addr >> 48) & 0xFFFF), 48); + } + + // BR x16: tail-call. + code[off++] = BrReg(16); + + size_t bytes = off * 4; + return JitMemory::Get(isolate)->EmitStub( + reinterpret_cast(code), bytes); +} + +} // namespace node::ffi::fastcall + +#endif diff --git a/src/ffi/fastcall/stub_emitter_arm.cc b/src/ffi/fastcall/stub_emitter_arm.cc new file mode 100644 index 00000000000000..77391bab1a2f16 --- /dev/null +++ b/src/ffi/fastcall/stub_emitter_arm.cc @@ -0,0 +1,71 @@ +#include "ffi/fastcall/stub_emitter.h" + +#if defined(HAVE_FFI_FASTCALL) && defined(__arm__) + +#include +#include "ffi/fastcall/jit_memory.h" + +namespace node::ffi::fastcall { + +namespace { + +constexpr unsigned kMaxGPArgs = 3; // r1..r3 after stripping receiver in r0 + +uint32_t MovReg(unsigned dst, unsigned src) { + // mov rd, rm -> 0xE1A00000 | (dst<<12) | src + return 0xE1A00000u | (dst << 12) | src; +} + +uint32_t LdrLitR12() { + // ldr r12, [pc, #0] -> 0xE59FC000 + // pc-relative offset 0 means: literal at pc, where pc = ldr_pos + 8. + return 0xE59FC000u; +} + +uint32_t BxReg(unsigned reg) { + // bx rn -> 0xE12FFF10 | rn + return 0xE12FFF10u | reg; +} + +} // namespace + +std::optional EmitForwarder( + v8::Isolate* isolate, + void* target, + const std::vector& args, + ResultClass /*result*/) { + unsigned gp = 0, fp = 0; + for (auto c : args) { + switch (c) { + case ArgClass::kGP: ++gp; break; + case ArgClass::kFP: ++fp; break; + case ArgClass::kGPPair: return std::nullopt; // i64/u64 unsupported + } + } + if (gp > kMaxGPArgs) return std::nullopt; + if (fp > 16) return std::nullopt; + + uint32_t code[8] = {0}; + size_t off = 0; + + // Shift r1..rN -> r0..r(N-1). + for (unsigned i = 0; i < gp; ++i) { + code[off++] = MovReg(/*dst=*/i, /*src=*/i + 1); + } + + // Layout: ldr r12, [pc, #0] ; bx r12 ; .word target ; .word 0 + // After ldr executes, pc = ldr_addr + 8. With imm12 = 0, the ldr loads + // from pc + 0 = ldr_addr + 8. That's the address of the .word target. + code[off++] = LdrLitR12(); + code[off++] = BxReg(12); + code[off++] = static_cast(reinterpret_cast(target)); + code[off++] = 0; // padding to keep stub size a multiple of 8 + + size_t bytes = off * 4; + return JitMemory::Get(isolate)->EmitStub( + reinterpret_cast(code), bytes); +} + +} // namespace node::ffi::fastcall + +#endif diff --git a/src/ffi/fastcall/stub_emitter_x64_sysv.cc b/src/ffi/fastcall/stub_emitter_x64_sysv.cc new file mode 100644 index 00000000000000..20bc82975b8ed1 --- /dev/null +++ b/src/ffi/fastcall/stub_emitter_x64_sysv.cc @@ -0,0 +1,87 @@ +#include "ffi/fastcall/stub_emitter.h" + +#if defined(HAVE_FFI_FASTCALL) && defined(__x86_64__) && \ + (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)) + +#include +#include "ffi/fastcall/jit_memory.h" + +namespace node::ffi::fastcall { + +namespace { +// SysV x86_64 GP arg regs: RDI, RSI, RDX, RCX, R8, R9. +// FP arg regs: XMM0..XMM7. Receiver in RDI. +// +// Strip = shift GP regs down by one. FP regs are NOT shifted — receiver +// is GP. Up to 5 GP args: tail-call. 6 GP args: 7th GP slot is on the +// stack, use call+ret with stack rewrite. + +constexpr uint8_t kMoves[5][3] = { + {0x48, 0x89, 0xF7}, // mov rdi, rsi + {0x48, 0x89, 0xD6}, // mov rsi, rdx + {0x48, 0x89, 0xCA}, // mov rdx, rcx + {0x4C, 0x89, 0xC1}, // mov rcx, r8 + {0x4D, 0x89, 0xC8}, // mov r8, r9 +}; + +} // namespace + +std::optional EmitForwarder( + v8::Isolate* isolate, + void* target, + const std::vector& args, + ResultClass /*result*/) { + unsigned gp_count = 0; + unsigned fp_count = 0; + for (auto c : args) { + switch (c) { + case ArgClass::kGP: ++gp_count; break; + case ArgClass::kFP: ++fp_count; break; + case ArgClass::kGPPair: return std::nullopt; // unsupported on this ABI + } + } + if (gp_count > 6) return std::nullopt; + if (fp_count > 8) return std::nullopt; + + uint8_t buf[64] = {0}; + size_t off = 0; + + if (gp_count <= 5) { + // Tail-call path. + for (unsigned i = 0; i < gp_count; ++i) { + std::memcpy(buf + off, kMoves[i], 3); + off += 3; + } + // mov r10, imm64; jmp r10 + buf[off++] = 0x49; buf[off++] = 0xBA; + uint64_t addr = reinterpret_cast(target); + std::memcpy(buf + off, &addr, 8); off += 8; + buf[off++] = 0x41; buf[off++] = 0xFF; buf[off++] = 0xE2; + } else { + // gp_count == 6: 7th GP slot on stack. Use call+ret with stack rewrite. + // sub rsp, 8 -> 48 83 EC 08 + buf[off++] = 0x48; buf[off++] = 0x83; buf[off++] = 0xEC; buf[off++] = 0x08; + // shift first 5 GP regs + for (unsigned i = 0; i < 5; ++i) { + std::memcpy(buf + off, kMoves[i], 3); + off += 3; + } + // mov r9, [rsp+16] -> 4C 8B 4C 24 10 + buf[off++] = 0x4C; buf[off++] = 0x8B; buf[off++] = 0x4C; + buf[off++] = 0x24; buf[off++] = 0x10; + // mov r10, imm64; call r10 + buf[off++] = 0x49; buf[off++] = 0xBA; + uint64_t addr = reinterpret_cast(target); + std::memcpy(buf + off, &addr, 8); off += 8; + buf[off++] = 0x41; buf[off++] = 0xFF; buf[off++] = 0xD2; + // add rsp, 8; ret + buf[off++] = 0x48; buf[off++] = 0x83; buf[off++] = 0xC4; buf[off++] = 0x08; + buf[off++] = 0xC3; + } + + return JitMemory::Get(isolate)->EmitStub(buf, off); +} + +} // namespace node::ffi::fastcall + +#endif diff --git a/src/ffi/fastcall/stub_emitter_x64_win.cc b/src/ffi/fastcall/stub_emitter_x64_win.cc new file mode 100644 index 00000000000000..d78367ffe03bc0 --- /dev/null +++ b/src/ffi/fastcall/stub_emitter_x64_win.cc @@ -0,0 +1,80 @@ +#include "ffi/fastcall/stub_emitter.h" + +#if defined(HAVE_FFI_FASTCALL) && defined(__x86_64__) && defined(_WIN32) + +#include +#include "ffi/fastcall/jit_memory.h" + +namespace node::ffi::fastcall { + +namespace { +// Win64 ABI: GP regs rcx, rdx, r8, r9 (4); FP regs xmm0-3 (4). +// FP/GP slots are POSITIONAL — slot N is one of {rcx,rdx,r8,r9} or one of +// {xmm0..xmm3} depending on the type at position N. Strip = shift the Nth +// arg from slot N+1 to slot N, where slot K maps to gp_reg[K] for kGP and +// xmm[K] for kFP. +// +// Encoding tables, indexed by destination slot 0..2: +// GP from slot+1 to slot: +// slot 0: mov rcx, rdx 48 89 D1 +// slot 1: mov rdx, r8 4C 89 C2 +// slot 2: mov r8, r9 4D 89 C8 +// FP from slot+1 to slot: +// slot 0: movaps xmm0, xmm1 0F 28 C1 +// slot 1: movaps xmm1, xmm2 0F 28 CA +// slot 2: movaps xmm2, xmm3 0F 28 D3 +// +// Cap: args.size() <= 3 (4 register slots minus the receiver). Stack-arg +// shuffling is not implemented; signatures with more args fall back to libffi. + +const uint8_t kGPMov[3][3] = { + {0x48, 0x89, 0xD1}, // mov rcx, rdx + {0x4C, 0x89, 0xC2}, // mov rdx, r8 + {0x4D, 0x89, 0xC8}, // mov r8, r9 +}; + +const uint8_t kFPMov[3][3] = { + {0x0F, 0x28, 0xC1}, // movaps xmm0, xmm1 + {0x0F, 0x28, 0xCA}, // movaps xmm1, xmm2 + {0x0F, 0x28, 0xD3}, // movaps xmm2, xmm3 +}; + +} // namespace + +std::optional EmitForwarder( + v8::Isolate* isolate, + void* target, + const std::vector& args, + ResultClass /*result*/) { + if (args.size() > 3) return std::nullopt; // 4 reg slots minus receiver + for (auto c : args) { + switch (c) { + case ArgClass::kGP: + case ArgClass::kFP: + break; + case ArgClass::kGPPair: + return std::nullopt; // unsupported on Win64 + } + } + + uint8_t buf[64] = {0}; + size_t off = 0; + + for (size_t i = 0; i < args.size(); ++i) { + const uint8_t* enc = (args[i] == ArgClass::kGP) ? kGPMov[i] : kFPMov[i]; + std::memcpy(buf + off, enc, 3); + off += 3; + } + + // mov r10, imm64; jmp r10 + buf[off++] = 0x49; buf[off++] = 0xBA; + uint64_t addr = reinterpret_cast(target); + std::memcpy(buf + off, &addr, 8); off += 8; + buf[off++] = 0x41; buf[off++] = 0xFF; buf[off++] = 0xE2; + + return JitMemory::Get(isolate)->EmitStub(buf, off); +} + +} // namespace node::ffi::fastcall + +#endif diff --git a/src/ffi/types.cc b/src/ffi/types.cc index 4d5062cdf832fb..52c93193cfe9f9 100644 --- a/src/ffi/types.cc +++ b/src/ffi/types.cc @@ -230,101 +230,6 @@ bool SignaturesMatch(const FFIFunction& fn, return true; } -bool IsSBEligibleFFIType(ffi_type* type) { - return type == &ffi_type_void || type == &ffi_type_sint8 || - type == &ffi_type_uint8 || type == &ffi_type_sint16 || - type == &ffi_type_uint16 || type == &ffi_type_sint32 || - type == &ffi_type_uint32 || type == &ffi_type_sint64 || - type == &ffi_type_uint64 || type == &ffi_type_float || - type == &ffi_type_double || type == &ffi_type_pointer; -} - -bool IsSBEligibleSignature(const FFIFunction& fn) { - // The JS wrapper writes and reads the shared buffer little-endian while - // the C++ side uses memcpy in host order. On big-endian hosts these - // disagree, so the fast path is disabled there. - if constexpr (IsBigEndian()) { - return false; - } - // Zero-argument functions gain nothing from the shared-buffer path - // (no argument packing to skip) and measurably lose on tight native - // calls like `uv_os_getpid` due to the wrapper's fixed overhead. - if (fn.args.empty()) return false; - if (!IsSBEligibleFFIType(fn.return_type)) return false; - for (ffi_type* arg : fn.args) { - if (!IsSBEligibleFFIType(arg)) return false; - } - return true; -} - -bool SignatureHasPointerArgs(const FFIFunction& fn) { - for (ffi_type* arg : fn.args) { - if (arg == &ffi_type_pointer) return true; - } - return false; -} - -void ReadFFIArgFromBuffer(ffi_type* type, - const uint8_t* buffer, - size_t offset, - void* out) { - CHECK(IsSBEligibleFFIType(type)); - CHECK_LE(type->size, sizeof(uint64_t)); - // memcpy avoids the strict-aliasing violation that a direct typed load - // from the raw uint8_t buffer would incur. - const uint8_t* src = buffer + offset; - std::memcpy(out, src, type->size); -} - -void WriteFFIReturnToBuffer(ffi_type* type, - const void* result, - uint8_t* buffer, - size_t offset) { - CHECK(IsSBEligibleFFIType(type)); - uint8_t* dst = buffer + offset; - std::memset(dst, 0, 8); - - if (type == &ffi_type_void) { - return; - } - - // libffi promotes small integer return values to ffi_arg size, so these - // branches read as ffi_arg or ffi_sarg and then truncate back down. - if (type == &ffi_type_sint8) { - int8_t tmp = static_cast(*static_cast(result)); - std::memcpy(dst, &tmp, sizeof(tmp)); - return; - } - if (type == &ffi_type_uint8) { - uint8_t tmp = static_cast(*static_cast(result)); - std::memcpy(dst, &tmp, sizeof(tmp)); - return; - } - if (type == &ffi_type_sint16) { - int16_t tmp = static_cast(*static_cast(result)); - std::memcpy(dst, &tmp, sizeof(tmp)); - return; - } - if (type == &ffi_type_uint16) { - uint16_t tmp = static_cast(*static_cast(result)); - std::memcpy(dst, &tmp, sizeof(tmp)); - return; - } - if (type == &ffi_type_sint32) { - int32_t tmp = static_cast(*static_cast(result)); - std::memcpy(dst, &tmp, sizeof(tmp)); - return; - } - if (type == &ffi_type_uint32) { - uint32_t tmp = static_cast(*static_cast(result)); - std::memcpy(dst, &tmp, sizeof(tmp)); - return; - } - - // Remaining SB-eligible types (sint64, uint64, float, double, pointer) - // are not promoted by libffi and can be copied as-is. - std::memcpy(dst, result, type->size); -} v8::Maybe ToFFIType(Environment* env, std::string_view type_str) { if (type_str == "void") { @@ -364,9 +269,9 @@ v8::Maybe ToFFIType(Environment* env, std::string_view type_str) { } } -// The JS fast path in lib/internal/ffi-shared-buffer.js mirrors the -// validation below. `writeNumericArg` matches the numeric branches and -// `writePointerArg` matches the pointer-BigInt branch. Error codes and +// The JS fast path in lib/internal/ffi-fastcall.js mirrors the +// validation below. `validateAndCoerce` matches the numeric branches and +// `coercePointer` matches the pointer-BigInt branch. Error codes and // messages must stay identical across all three sites. Maybe ToFFIArgument(Environment* env, unsigned int index, @@ -738,6 +643,127 @@ bool ToFFIReturnValue(Local result, ffi_type* type, void* ret) { return true; } +#ifdef HAVE_FFI_FASTCALL + +namespace { + +bool IsFastCallEligibleFFIType(ffi_type* t) { + return t == &ffi_type_void || + t == &ffi_type_sint8 || t == &ffi_type_uint8 || + t == &ffi_type_sint16 || t == &ffi_type_uint16 || + t == &ffi_type_sint32 || t == &ffi_type_uint32 || + t == &ffi_type_sint64 || t == &ffi_type_uint64 || + t == &ffi_type_float || t == &ffi_type_double || + t == &ffi_type_pointer; +} + +bool IsFunctionTypeName(const std::string& s) { + return s == "function"; +} + +// Per-platform emitter GP caps (excluding the implicit receiver slot). +// Note: Win64 uses a combined gp+fp <= 3 positional-slot cap checked below; +// it does not use kFastcallMaxGPArgs. +#if defined(__aarch64__) || defined(_M_ARM64) +constexpr unsigned kFastcallMaxGPArgs = 7; +#elif defined(__x86_64__) && !defined(_WIN32) +constexpr unsigned kFastcallMaxGPArgs = 6; +#elif defined(__arm__) +constexpr unsigned kFastcallMaxGPArgs = 3; +#else +constexpr unsigned kFastcallMaxGPArgs = 0; // no emitter on this platform +#endif + +// Every supported emitter allows up to 8 FP args. Win64 is further +// constrained by the gp+fp<=3 positional-slot cap checked below. +constexpr unsigned kFastcallMaxFPArgs = 8; + +} // namespace + +bool IsFastCallEligible(const FFIFunction& fn, const char** out_reason) { + static const char* dummy = ""; + if (out_reason == nullptr) out_reason = &dummy; + + if (!IsFastCallEligibleFFIType(fn.return_type)) { + *out_reason = "unsupported return type"; + return false; + } + if (IsFunctionTypeName(fn.return_type_name)) { + *out_reason = "return type is function"; + return false; + } + + unsigned gp = 0, fp = 0; + for (size_t i = 0; i < fn.args.size(); ++i) { + ffi_type* t = fn.args[i]; + const std::string& name = fn.arg_type_names[i]; + + if (!IsFastCallEligibleFFIType(t)) { + *out_reason = "unsupported arg type"; + return false; + } + // `void` is fine as a return type but has no register slot, so it cannot + // appear in `args`. `IsFastCallEligibleFFIType` accepts it for the + // return-type check above; reject it explicitly here so the GP/FP + // counting below stays consistent and `MapArgType` is never called with + // it (where it would hit UNREACHABLE). + if (t == &ffi_type_void) { + *out_reason = "void cannot be an argument type"; + return false; + } + if (IsFunctionTypeName(name)) { + *out_reason = "arg is function"; + return false; + } + +#if defined(__arm__) + // AArch32: i64/u64 consume two GP regs with alignment rules. Not + // supported in the v1 stub emitter. + if (t == &ffi_type_sint64 || t == &ffi_type_uint64) { + *out_reason = "i64/u64 arg unsupported on aarch32"; + return false; + } +#endif + + if (t == &ffi_type_float || t == &ffi_type_double) { + ++fp; + } else { + ++gp; + } + } + +#if defined(__arm__) + if (fn.return_type == &ffi_type_sint64 || + fn.return_type == &ffi_type_uint64) { + *out_reason = "i64/u64 return unsupported on aarch32"; + return false; + } +#endif + +#if defined(__x86_64__) && defined(_WIN32) + // Win64: positional slots — only 4 total (one taken by receiver), so + // gp + fp must not exceed 3. + if (gp + fp > 3) { + *out_reason = "GP + FP arg count exceeds Win64 cap"; + return false; + } +#else + if (gp > kFastcallMaxGPArgs) { + *out_reason = "GP arg count exceeds ABI cap"; + return false; + } + if (fp > kFastcallMaxFPArgs) { + *out_reason = "FP arg count exceeds ABI cap"; + return false; + } +#endif + + *out_reason = ""; + return true; +} + +#endif // HAVE_FFI_FASTCALL + } // namespace ffi } // namespace node diff --git a/src/ffi/types.h b/src/ffi/types.h index 14f474a465aa4a..85ed27862fc81f 100644 --- a/src/ffi/types.h +++ b/src/ffi/types.h @@ -53,30 +53,17 @@ bool SignaturesMatch(const FFIFunction& fn, ffi_type* return_type, const std::vector& args); -// True if the FFI type can be read from / written to a raw byte buffer -// without needing V8 operations (conversion, allocation, etc.). -bool IsSBEligibleFFIType(ffi_type* type); - -// True if the signature's return type and all argument types are SB-eligible. -bool IsSBEligibleSignature(const FFIFunction& fn); - -// True if any argument is pointer-typed. For these, the JS wrapper must -// do a runtime type check to decide fast vs. slow path per call. -bool SignatureHasPointerArgs(const FFIFunction& fn); - -// Read a value of the given FFI type from buffer at the given byte offset -// into the output pointer (sized as uint64_t to hold any numeric type). -void ReadFFIArgFromBuffer(ffi_type* type, - const uint8_t* buffer, - size_t offset, - void* out); - -// Write a return value of the given FFI type from the ffi_call result -// into the buffer at the given byte offset. -void WriteFFIReturnToBuffer(ffi_type* type, - const void* result, - uint8_t* buffer, - size_t offset); +#ifdef HAVE_FFI_FASTCALL +// Returns true if `fn` can be invoked via the V8 fast-call path. On +// false, `*out_reason` is set to a static string describing why +// (never null after this returns; callers may pass nullptr to ignore). +// +// Eligibility = all of: every arg type and the return type are +// numeric-or-pointer, no `function`-typed args/return, GP arg count +// within ABI cap, FP arg count within ABI cap. Adds an AArch32-only +// rejection of i64/u64 args (kGPPair handling not in v1). +bool IsFastCallEligible(const FFIFunction& fn, const char** out_reason); +#endif } // namespace node::ffi diff --git a/src/node_builtins.cc b/src/node_builtins.cc index b098a41cca9ea4..8fa11e7e5ab9f2 100644 --- a/src/node_builtins.cc +++ b/src/node_builtins.cc @@ -141,7 +141,7 @@ BuiltinLoader::BuiltinCategories BuiltinLoader::GetBuiltinCategories() const { "internal/quic/state", #endif // !OPENSSL_NO_QUIC #if !HAVE_FFI - "internal/ffi-shared-buffer", + "internal/ffi-fastcall", #endif // !HAVE_FFI "ffi", // Experimental. "quic", // Experimental. diff --git a/src/node_ffi.cc b/src/node_ffi.cc index 89018c8e6e4b32..07cb3d69c47222 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -10,6 +10,16 @@ #include "ffi/data.h" #include "ffi/types.h" #include "node_errors.h" +#include "node_process-inl.h" + +#include +#include + +#ifdef HAVE_FFI_FASTCALL +#include "ffi/fastcall/cfunction_info.h" +#include "ffi/fastcall/jit_memory.h" +#include "ffi/fastcall/stub_emitter.h" +#endif namespace node { @@ -44,9 +54,37 @@ using v8::WeakCallbackType; namespace ffi { +#ifdef HAVE_FFI_FASTCALL +FastCallState::FastCallState( + v8::Isolate* isolate, + v8::Local slow_fn, + void* stub, + size_t alloc_size, + std::unique_ptr bndl) + : slow_invoke(isolate, slow_fn), + stub_entry(stub), + stub_alloc_size(alloc_size), + cfun_bundle(std::move(bndl)) {} + +FastCallState::~FastCallState() { + slow_invoke.Reset(); + cfun_bundle.reset(); + if (stub_entry != nullptr) { + node::ffi::fastcall::JitMemory::Get(/*isolate=*/nullptr) + ->Free(stub_entry, stub_alloc_size); + stub_entry = nullptr; + } +} +#endif + DynamicLibrary::DynamicLibrary(Environment* env, Local object) : BaseObject(env, object), lib_{}, handle_(nullptr), symbols_() { MakeWeak(); +#ifdef HAVE_FFI_FASTCALL + alive_backing_ = v8::ArrayBuffer::NewBackingStore(env->isolate(), 1); + CHECK(alive_backing_); + static_cast(alive_backing_->Data())[0] = 0; +#endif } DynamicLibrary::~DynamicLibrary() { @@ -66,12 +104,34 @@ void DynamicLibrary::MemoryInfo(MemoryTracker* tracker) const { tracker->TrackFieldWithSize( "symbols", symbols_size, "std::unordered_map"); - // FFIFunctionInfo instances and their sb_backing ArrayBuffers are - // owned by V8 function wrappers and reachable only via weak references, - // so they are deliberately not counted here. + // FFIFunctionInfo instances are owned by V8 function wrappers and reachable + // only via weak references, so they are deliberately not counted here. + +#ifdef HAVE_FFI_FASTCALL + // Process-wide JIT bytes from the singleton allocator. Per-library + // accounting would require plumbing per-stub bookkeeping through + // CreateFunction; for v1 we report the singleton total under each + // library's MemoryInfo, with a name that makes the scope clear. + size_t jit_bytes = + node::ffi::fastcall::JitMemory::Get(env()->isolate())->TotalLiveBytes(); + tracker->TrackFieldWithSize( + "jit_stubs_process_global", jit_bytes, + "ffi::FastCallStubs (process-wide singleton)"); +#endif } +#ifdef HAVE_FFI_FASTCALL +v8::Local DynamicLibrary::AliveBuffer(v8::Isolate* isolate) { + return v8::ArrayBuffer::New(isolate, alive_backing_); +} +#endif + void DynamicLibrary::Close() { +#ifdef HAVE_FFI_FASTCALL + if (alive_backing_) { + static_cast(alive_backing_->Data())[0] = 1; + } +#endif for (auto& [name, fn] : functions_) { fn->closed = true; fn->ptr = nullptr; @@ -192,6 +252,9 @@ Maybe DynamicLibrary::PrepareFunction( void DynamicLibrary::CleanupFunctionInfo( const WeakCallbackInfo& data) { FFIFunctionInfo* info = data.GetParameter(); +#ifdef HAVE_FFI_FASTCALL + info->fast.reset(); // FastCallState destructor frees the JIT stub +#endif info->fn.reset(); info->self.Reset(); delete info; @@ -211,20 +274,174 @@ MaybeLocal DynamicLibrary::CreateFunction( DCHECK_EQ(fn->args.size(), fn->arg_type_names.size()); - bool use_sb = IsSBEligibleSignature(*fn); - bool has_ptr_args = use_sb && SignatureHasPointerArgs(*fn); - Local data = External::New(isolate, info.get(), v8::kExternalPointerTypeTagDefault); - MaybeLocal maybe_ret = - Function::New(context, - use_sb ? DynamicLibrary::InvokeFunctionSB - : DynamicLibrary::InvokeFunction, - data); + // Internal properties are keyed by per-isolate Symbols (see + // `env_properties.h`) to keep them out of string-key reflection, and the + // `ReadOnly | DontEnum | DontDelete` attribute set blocks user code from + // reading, modifying, or deleting them. + PropertyAttribute internal_attrs = + static_cast(ReadOnly | DontEnum | DontDelete); + Local ret; - if (!maybe_ret.ToLocal(&ret)) { - return MaybeLocal(); + +#ifdef HAVE_FFI_FASTCALL + const char* fc_reason = ""; + // One-time process-wide self-test. On failure (entitlement missing, + // hardened runtime, SELinux execmem denial, etc.), every subsequent + // registration falls back to libffi. + static std::atomic selftest_result{-1}; + static std::once_flag selftest_warn_once; + int s = selftest_result.load(std::memory_order_acquire); + if (s == -1) { + bool ok = node::ffi::fastcall::JitMemory::Get(isolate)->SelfTest(isolate); + s = ok ? 1 : 0; + selftest_result.store(s, std::memory_order_release); + if (!ok) { + // The atomic above only dedupes the SelfTest *call*; two threads can + // both observe `s == -1` and both reach this branch. `call_once` + // ensures at most one warning is emitted per process across all + // racing first-callers. + bool warn_failed = false; + std::call_once(selftest_warn_once, [&]() { + if (ProcessEmitWarning(env, + "FFI fast-call self-test failed; falling back to libffi for " + "all FFI invocations") + .IsNothing()) { + warn_failed = true; + } + }); + if (warn_failed) return MaybeLocal(); + } + } + bool fc_ok = (s == 1) && node::ffi::IsFastCallEligible(*fn, &fc_reason); + if (fc_ok) { + auto bundle = node::ffi::fastcall::BuildCFunctionInfo(*fn); + { + auto stub = node::ffi::fastcall::EmitForwarder( + isolate, fn->ptr, bundle.arg_classes, bundle.result_class); + if (!stub.has_value()) { + // Warn at most once per process: a single FFI app may register hundreds + // of functions, and the failure cause (mprotect / SELinux / hardened + // runtime) is the same for every call. Repeated warnings would flood + // logs without adding signal. + static std::atomic warned{false}; + if (!warned.exchange(true, std::memory_order_acq_rel)) { + if (ProcessEmitWarning(env, + "FFI fast-call stub emission failed for an eligible " + "signature; falling back to libffi for this and possibly " + "future calls. This usually indicates a JIT memory or " + "permission issue (mprotect/SELinux/hardened runtime).") + .IsNothing()) { + return MaybeLocal(); + } + } + // fall through to libffi path + } else { + v8::CFunction cfun(stub->entry, bundle.info); + v8::Local tmpl = v8::FunctionTemplate::New( + isolate, + DynamicLibrary::InvokeFunction, + data, + v8::Local(), + static_cast(fn->args.size()), + v8::ConstructorBehavior::kThrow, + v8::SideEffectType::kHasSideEffect, + &cfun); + v8::Local fc_fn; + if (tmpl->GetFunction(context).ToLocal(&fc_fn)) { + // Build all V8 values before touching `info` fields, so that on + // any failure we can free the stub and return cleanly without + // leaking JIT memory. + bool metadata_ok = true; + + // Alive ArrayBuffer. + v8::Local alive_ab = AliveBuffer(isolate); + if (!fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_alive_symbol(), + alive_ab, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + + // Build the slow-invoke v8::Function for the wrapper's pointer + // fallback path. + Local slow_fn; + if (metadata_ok && + !Function::New(context, DynamicLibrary::InvokeFunction, data) + .ToLocal(&slow_fn)) { + metadata_ok = false; + } + if (metadata_ok && + !fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_invoke_slow_symbol(), + slow_fn, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + + // Attach params and result type names. + Local params_arr; + if (metadata_ok && + !ToV8Value(context, fn->arg_type_names, isolate) + .ToLocal(¶ms_arr)) { + metadata_ok = false; + } + if (metadata_ok && + !fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_params_symbol(), + params_arr, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + Local result_str; + if (metadata_ok && + !ToV8Value(context, fn->return_type_name, isolate) + .ToLocal(&result_str)) { + metadata_ok = false; + } + if (metadata_ok && + !fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_result_symbol(), + result_str, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + + if (!metadata_ok) { + // Free the stub to avoid a JIT memory leak. + node::ffi::fastcall::JitMemory::Get(isolate) + ->Free(*stub); + return MaybeLocal(); + } + + // All metadata attached successfully. Populate fast-call state. + info->fast = std::make_unique( + isolate, slow_fn, stub->entry, stub->alloc_size, + std::make_unique( + std::move(bundle))); + ret = fc_fn; + } else { + // Template-to-function failed (V8 has a pending exception). Free the + // stub since we won't use it; mirror the metadata-fail path and + // propagate the empty MaybeLocal so the caller surfaces the V8 + // exception. + node::ffi::fastcall::JitMemory::Get(isolate) + ->Free(*stub); + return MaybeLocal(); + } + } + } + } + if (ret.IsEmpty()) +#endif + { + MaybeLocal maybe_ret = + Function::New(context, DynamicLibrary::InvokeFunction, data); + if (!maybe_ret.ToLocal(&ret)) { + return MaybeLocal(); + } } Local name_str; @@ -243,74 +460,6 @@ MaybeLocal DynamicLibrary::CreateFunction( return MaybeLocal(); } - // Internal properties are keyed by per-isolate Symbols (see - // `env_properties.h`) to keep them out of string-key reflection, and the - // `ReadOnly | DontEnum | DontDelete` attribute set blocks user code from - // reading, modifying, or deleting them. - PropertyAttribute internal_attrs = - static_cast(ReadOnly | DontEnum | DontDelete); - - if (use_sb) { - size_t sb_size = 8 * (fn->args.size() + 1); - Local ab = ArrayBuffer::New(isolate, sb_size); - // The shared_ptr to the backing store keeps the memory alive while - // FFIFunctionInfo still references it. - info->sb_backing = ab->GetBackingStore(); - - if (!ret->DefineOwnProperty( - context, env->ffi_sb_shared_buffer_symbol(), ab, internal_attrs) - .FromMaybe(false)) { - return MaybeLocal(); - } - - // Signatures with pointer args also expose a slow-path invoker bound - // to the same FFIFunctionInfo. The JS wrapper routes through it when a - // pointer argument is anything other than a BigInt, null, or undefined - // (strings, Buffers, ArrayBuffers, and ArrayBufferViews). - if (has_ptr_args) { - Local slow_fn; - if (!Function::New(context, DynamicLibrary::InvokeFunction, data) - .ToLocal(&slow_fn)) { - return MaybeLocal(); - } - if (!ret->DefineOwnProperty(context, - env->ffi_sb_invoke_slow_symbol(), - slow_fn, - internal_attrs) - .FromMaybe(false)) { - return MaybeLocal(); - } - } - - // Attach the original signature type names so the JS wrapper can - // rebuild the signature from a raw function when the caller did not - // pass parameters and result explicitly. The `lib.functions` accessor - // path relies on this. - Local params_arr; - if (!ToV8Value(context, fn->arg_type_names, isolate).ToLocal(¶ms_arr)) { - return MaybeLocal(); - } - if (!ret->DefineOwnProperty(context, - env->ffi_sb_params_symbol(), - params_arr, - internal_attrs) - .FromMaybe(false)) { - return MaybeLocal(); - } - - Local result_name; - if (!ToV8Value(context, fn->return_type_name, isolate) - .ToLocal(&result_name)) { - return MaybeLocal(); - } - if (!ret->DefineOwnProperty(context, - env->ffi_sb_result_symbol(), - result_name, - internal_attrs) - .FromMaybe(false)) { - return MaybeLocal(); - } - } info->self.Reset(isolate, ret); info->self.SetWeak(info.release(), @@ -440,65 +589,6 @@ void DynamicLibrary::InvokeFunction(const FunctionCallbackInfo& args) { free(result); } -void DynamicLibrary::InvokeFunctionSB(const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - FFIFunctionInfo* info = - static_cast(args.Data().As()->Value()); - FFIFunction* fn = info->fn.get(); - - if (fn == nullptr || fn->closed || fn->ptr == nullptr) { - THROW_ERR_FFI_LIBRARY_CLOSED(env); - return; - } - - // Arguments reach the native invoker through the shared buffer, not - // through V8. The JS wrapper always calls the raw function as `rawFn()` - // so any non-zero argument count indicates that user code reached the - // raw SB function directly and is about to read stale buffer contents. - if (args.Length() != 0) { - THROW_ERR_INVALID_ARG_VALUE( - env, - "SB-invoked FFI functions receive arguments through the shared " - "buffer, not as JavaScript arguments"); - return; - } - - // A failure of either CHECK means the SB invoker ran against a function - // that `CreateFunction` did not set up for the fast path, which is a - // contract violation. They stay enabled in Release because each FFI call - // is already dominated by `ffi_call` itself. - CHECK(info->sb_backing); - CHECK_EQ(info->sb_backing->ByteLength(), 8u * (info->fn->args.size() + 1)); - - uint8_t* buffer = static_cast(info->sb_backing->Data()); - unsigned int nargs = fn->args.size(); - - // Layout is 8 bytes per slot. The return value lives at offset 0 and - // argument i lives at offset 8*(i+1). - std::vector values(nargs, 0); - std::vector ffi_args(nargs, nullptr); - - for (unsigned int i = 0; i < nargs; i++) { - ReadFFIArgFromBuffer(fn->args[i], buffer, 8 * (i + 1), &values[i]); - ffi_args[i] = &values[i]; - } - - // The storage must cover both the ffi_arg width that libffi uses for - // promoted small integer returns and the 8 bytes needed for non-promoted - // SB-eligible returns like f64, i64, and u64. `sizeof(ffi_arg)` is only - // 4 on 32-bit ARM, so take the max. - constexpr size_t kSBResultStorageSize = - sizeof(ffi_arg) > 8 ? sizeof(ffi_arg) : 8; - alignas(8) uint8_t result_storage[kSBResultStorageSize] = {0}; - void* result = (fn->return_type != &ffi_type_void) ? result_storage : nullptr; - - ffi_call(&fn->cif, FFI_FN(fn->ptr), result, ffi_args.data()); - - if (result != nullptr) { - WriteFFIReturnToBuffer(fn->return_type, result, buffer, 0); - } -} - // This is the function that will be called by libffi when a callback // is invoked from a dlopen library. It converts the arguments to JavaScript // values and calls the original JavaScript callback function. @@ -1141,8 +1231,8 @@ static void Initialize(Local target, SetMethod(context, target, "setFloat64", SetFloat64); // ToFFIType maps `char` to sint8 or uint8 based on `CHAR_MIN < 0` at C++ - // build time. Exposing the same decision to JS lets the shared-buffer - // wrapper's range check match `ToFFIArgument` on every platform. + // build time. Exposing the same decision to JS lets the fast-call wrapper's + // range check match `ToFFIArgument` on every platform. Isolate* isolate = env->isolate(); target ->Set(context, @@ -1150,10 +1240,9 @@ static void Initialize(Local target, Boolean::New(isolate, CHAR_MIN < 0)) .Check(); - // The shared-buffer fast path uses `uintptrMax` to reject pointer BigInts - // that would otherwise be silently truncated by `ReadFFIArgFromBuffer`'s - // `memcpy(..., type->size, ...)` on 32-bit platforms. The slow path - // rejects the same values through `ToFFIArgument`. + // `uintptrMax` is used by the fast-call JS wrapper to reject pointer BigInts + // that exceed the platform's pointer range on 32-bit platforms. The slow + // path rejects the same values through `ToFFIArgument`. target ->Set(context, FIXED_ONE_BYTE_STRING(isolate, "uintptrMax"), @@ -1162,28 +1251,28 @@ static void Initialize(Local target, static_cast(std::numeric_limits::max()))) .Check(); - // Per-isolate Symbols used by `lib/internal/ffi-shared-buffer.js` to key - // shared-buffer internal state on raw FFI functions. +#ifdef HAVE_FFI_FASTCALL target ->Set(context, - FIXED_ONE_BYTE_STRING(isolate, "kSbSharedBuffer"), - env->ffi_sb_shared_buffer_symbol()) + FIXED_ONE_BYTE_STRING(isolate, "kFastcallAlive"), + env->ffi_fastcall_alive_symbol()) .Check(); target ->Set(context, - FIXED_ONE_BYTE_STRING(isolate, "kSbInvokeSlow"), - env->ffi_sb_invoke_slow_symbol()) + FIXED_ONE_BYTE_STRING(isolate, "kFastcallInvokeSlow"), + env->ffi_fastcall_invoke_slow_symbol()) .Check(); target ->Set(context, - FIXED_ONE_BYTE_STRING(isolate, "kSbParams"), - env->ffi_sb_params_symbol()) + FIXED_ONE_BYTE_STRING(isolate, "kFastcallParams"), + env->ffi_fastcall_params_symbol()) .Check(); target ->Set(context, - FIXED_ONE_BYTE_STRING(isolate, "kSbResult"), - env->ffi_sb_result_symbol()) + FIXED_ONE_BYTE_STRING(isolate, "kFastcallResult"), + env->ffi_fastcall_result_symbol()) .Check(); +#endif } } // namespace ffi diff --git a/src/node_ffi.h b/src/node_ffi.h index 14362963f288ce..8efbb7a1a026d4 100644 --- a/src/node_ffi.h +++ b/src/node_ffi.h @@ -13,6 +13,17 @@ #include #include +#ifdef HAVE_FFI_FASTCALL +// jit_memory.h only uses standard-library headers so it is safe to include +// here. cfunction_info.h includes node_ffi.h, so it cannot be included here +// (cycle), but a forward declaration suffices for the unique_ptr member. +#include "ffi/fastcall/jit_memory.h" // for JitMemory, EmittedStub + +namespace node::ffi::fastcall { +struct CFunctionInfoBundle; +} // namespace node::ffi::fastcall +#endif + namespace node::ffi { class DynamicLibrary; @@ -29,10 +40,56 @@ struct FFIFunction { std::string return_type_name; }; +#ifdef HAVE_FFI_FASTCALL +// Owns the per-function fast-call state for a single FFI registration. +// Three resources are owned and freed by the destructor: +// 1. The JIT stub bytes — released via +// `JitMemory::Free(stub_entry, stub_alloc_size)`. +// 2. The slow-path libffi `v8::Global` — reset. +// 3. The `CFunctionInfoBundle` (which itself owns heap-allocated +// `v8::CFunctionInfo` and `v8::CTypeInfo[]`) — destroyed via the +// bundle's destructor. +// +// `FFIFunctionInfo::fast` is null for ineligible signatures (libffi-only +// path); a non-null `fast` always represents a fully-constructed +// fast-call state. +struct FastCallState { + v8::Global slow_invoke; + void* stub_entry = nullptr; + size_t stub_alloc_size = 0; + std::unique_ptr cfun_bundle; + + // Default-construction is forbidden — every FastCallState must hold a + // valid stub/bundle/slow-invoke triple. Construct via the value form. + FastCallState() = delete; + + // Construct a fully-armed FastCallState. All resources move into the + // new object atomically (under -fno-exceptions, member init runs + // top-to-bottom; an OOM in any V8 handle allocation aborts the + // process, which is the project contract). + // Preconditions: + // - `slow_fn` is a non-empty Local. + // - `stub` is a non-null pointer returned from JitMemory::EmitStub. + // - `alloc_size` is the matching allocation size from EmittedStub. + // - `bndl` is a non-null unique_ptr. + FastCallState(v8::Isolate* isolate, + v8::Local slow_fn, + void* stub, + size_t alloc_size, + std::unique_ptr bndl); + ~FastCallState(); + FastCallState(const FastCallState&) = delete; + FastCallState& operator=(const FastCallState&) = delete; +}; +#endif + struct FFIFunctionInfo { std::shared_ptr fn; v8::Global self; - std::shared_ptr sb_backing; + +#ifdef HAVE_FFI_FASTCALL + std::unique_ptr fast; // null when ineligible +#endif }; struct FFICallback { @@ -78,7 +135,6 @@ class DynamicLibrary : public BaseObject { static void New(const v8::FunctionCallbackInfo& args); static void Close(const v8::FunctionCallbackInfo& args); static void InvokeFunction(const v8::FunctionCallbackInfo& args); - static void InvokeFunctionSB(const v8::FunctionCallbackInfo& args); static void InvokeCallback(ffi_cif* cif, void* ret, void** args, @@ -99,6 +155,16 @@ class DynamicLibrary : public BaseObject { SET_MEMORY_INFO_NAME(DynamicLibrary) SET_SELF_SIZE(DynamicLibrary) +#ifdef HAVE_FFI_FASTCALL + public: + // Returns the per-library "alive" ArrayBuffer. The first byte is 0 + // while the library is open and 1 after Close() runs. Wrappers in + // lib/internal/ffi-fastcall.js use this to throw ERR_FFI_LIBRARY_CLOSED + // before the V8 fast-call path enters a stub pointing at unmapped + // library memory. + v8::Local AliveBuffer(v8::Isolate* isolate); +#endif + private: void Close(); v8::Maybe ResolveSymbol(Environment* env, const std::string& name); @@ -123,6 +189,9 @@ class DynamicLibrary : public BaseObject { std::unordered_map symbols_; std::unordered_map> functions_; std::unordered_map> callbacks_; +#ifdef HAVE_FFI_FASTCALL + std::shared_ptr alive_backing_; +#endif }; void GetInt8(const v8::FunctionCallbackInfo& args); diff --git a/test/cctest/test_ffi_fastcall_cfunction.cc b/test/cctest/test_ffi_fastcall_cfunction.cc new file mode 100644 index 00000000000000..720c4988b17b84 --- /dev/null +++ b/test/cctest/test_ffi_fastcall_cfunction.cc @@ -0,0 +1,116 @@ +#include "gtest/gtest.h" + +#ifdef HAVE_FFI_FASTCALL + +#include +#include + +#include "ffi/fastcall/cfunction_info.h" +#include "node_ffi.h" + +using node::ffi::FFIFunction; +using node::ffi::fastcall::ArgClass; +using node::ffi::fastcall::BuildCFunctionInfo; +using node::ffi::fastcall::ResultClass; + +namespace { +FFIFunction MakeFn(ffi_type* ret, + std::string ret_name, + std::vector args, + std::vector arg_names) { + FFIFunction fn{}; + fn.return_type = ret; + fn.return_type_name = std::move(ret_name); + fn.args = std::move(args); + fn.arg_type_names = std::move(arg_names); + return fn; +} +} // namespace + +TEST(FFIFastCallCFunction, NumericSignature) { + auto fn = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_sint32, &ffi_type_sint32}, + {"i32", "i32"}); + auto bundle = BuildCFunctionInfo(fn); + // Receiver + 2 args = 3 CTypeInfo entries. + EXPECT_EQ(bundle.info->ArgumentCount(), 3u); + EXPECT_EQ(bundle.arg_classes.size(), 2u); + EXPECT_EQ(bundle.arg_classes[0], ArgClass::kGP); + EXPECT_EQ(bundle.arg_classes[1], ArgClass::kGP); + EXPECT_EQ(bundle.result_class, ResultClass::kGP); +} + +TEST(FFIFastCallCFunction, FloatSignature) { + auto fn = MakeFn(&ffi_type_double, "double", + {&ffi_type_float, &ffi_type_double}, + {"float", "double"}); + auto bundle = BuildCFunctionInfo(fn); + EXPECT_EQ(bundle.arg_classes[0], ArgClass::kFP); + EXPECT_EQ(bundle.arg_classes[1], ArgClass::kFP); + EXPECT_EQ(bundle.result_class, ResultClass::kFP); +} + +TEST(FFIFastCallCFunction, VoidReturn) { + auto fn = MakeFn(&ffi_type_void, "void", + {&ffi_type_sint32}, {"i32"}); + auto bundle = BuildCFunctionInfo(fn); + EXPECT_EQ(bundle.result_class, ResultClass::kVoid); +} + +TEST(FFIFastCallCFunction, PointerSignature) { + auto fn = MakeFn(&ffi_type_pointer, "pointer", + {&ffi_type_pointer}, {"pointer"}); + auto bundle = BuildCFunctionInfo(fn); + EXPECT_EQ(bundle.arg_classes[0], ArgClass::kGP); + EXPECT_EQ(bundle.result_class, ResultClass::kGP); +} + +TEST(FFIFastCallCFunction, MoveCleanupSafe) { + auto fn = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_sint32}, {"i32"}); + auto a = BuildCFunctionInfo(fn); + // Move into a new bundle and let it drop. The moved-from bundle's + // destructor must not double-free. + auto b = std::move(a); + EXPECT_EQ(a.info, nullptr); + EXPECT_EQ(a.arg_types, nullptr); + // b's destructor runs at end of scope; no crash expected. +} + +TEST(FFIFastCallCFunction, MoveAssignmentSelfSafe) { + auto fn = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_sint32}, {"i32"}); + auto bundle = BuildCFunctionInfo(fn); + ASSERT_NE(bundle.info, nullptr); + ASSERT_NE(bundle.arg_types, nullptr); + ASSERT_EQ(bundle.arg_classes.size(), 1u); + ASSERT_EQ(bundle.result_class, node::ffi::fastcall::ResultClass::kGP); + + // Self-move-assign must be a no-op (or at least not double-free). + // The self-assignment guard should keep all four bundle fields valid. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wself-move" + bundle = std::move(bundle); +#pragma GCC diagnostic pop + EXPECT_NE(bundle.info, nullptr); + EXPECT_NE(bundle.arg_types, nullptr); + EXPECT_EQ(bundle.arg_classes.size(), 1u); + EXPECT_EQ(bundle.result_class, node::ffi::fastcall::ResultClass::kGP); +} + +TEST(FFIFastCallCFunction, MoveAssignmentReplaces) { + auto fn_a = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_sint32}, {"i32"}); + auto fn_b = MakeFn(&ffi_type_double, "double", + {&ffi_type_double}, {"double"}); + auto bundle_a = BuildCFunctionInfo(fn_a); + auto bundle_b = BuildCFunctionInfo(fn_b); + const void* old_a_info = bundle_a.info; + bundle_a = std::move(bundle_b); + // bundle_a now holds what bundle_b had; bundle_b is nulled out. + EXPECT_EQ(bundle_b.info, nullptr); + EXPECT_NE(bundle_a.info, nullptr); + EXPECT_NE(bundle_a.info, old_a_info); +} + +#endif // HAVE_FFI_FASTCALL diff --git a/test/cctest/test_ffi_fastcall_eligibility.cc b/test/cctest/test_ffi_fastcall_eligibility.cc new file mode 100644 index 00000000000000..7b523e0503b254 --- /dev/null +++ b/test/cctest/test_ffi_fastcall_eligibility.cc @@ -0,0 +1,233 @@ +#include "gtest/gtest.h" + +#ifdef HAVE_FFI_FASTCALL + +#include +#include + +#include "ffi.h" +#include "ffi/types.h" +#include "node_ffi.h" + +using node::ffi::FFIFunction; +using node::ffi::IsFastCallEligible; + +namespace { +FFIFunction MakeFn(ffi_type* ret, + std::string ret_name, + std::vector args, + std::vector arg_names) { + FFIFunction fn{}; + fn.return_type = ret; + fn.return_type_name = std::move(ret_name); + fn.args = std::move(args); + fn.arg_type_names = std::move(arg_names); + return fn; +} +} // namespace + +TEST(FFIFastCallEligibility, AllNumeric) { + auto fn = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_sint32, &ffi_type_sint32}, + {"i32", "i32"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, VoidReturnIsOk) { + auto fn = MakeFn(&ffi_type_void, "void", + {&ffi_type_sint32}, {"i32"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, FloatsOk) { + auto fn = MakeFn(&ffi_type_double, "double", + {&ffi_type_float, &ffi_type_double}, + {"float", "double"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, PointerOk) { + auto fn = MakeFn(&ffi_type_pointer, "pointer", + {&ffi_type_pointer}, {"pointer"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, FunctionTypeRejected) { + auto fn = MakeFn(&ffi_type_void, "void", + {&ffi_type_pointer}, {"function"}); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); + ASSERT_NE(reason, nullptr); + EXPECT_NE(std::string(reason).find("function"), std::string::npos); +} + +TEST(FFIFastCallEligibility, FunctionReturnRejected) { + auto fn = MakeFn(&ffi_type_pointer, "function", + {&ffi_type_sint32}, {"i32"}); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, TooManyGPArgsRejected) { + // 8 GP args is one over AArch64's cap of 7 (the largest GP cap of any + // supported platform), so this is rejected on every supported emitter. + std::vector args(8, &ffi_type_sint32); + std::vector names(8, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); + ASSERT_NE(reason, nullptr); + // On non-Win64 platforms the reason mentions "GP arg"; on Win64 the combined + // gp+fp cap fires with "Win64 cap". Accept either. + const std::string r(reason); + EXPECT_TRUE(r.find("GP arg") != std::string::npos || + r.find("Win64 cap") != std::string::npos); +} + +TEST(FFIFastCallEligibility, TooManyFPArgsRejected) { + // 9 FP args is one over the FP cap of 8. + std::vector args(9, &ffi_type_double); + std::vector names(9, "double"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); + ASSERT_NE(reason, nullptr); + // On non-Win64 the reason mentions "FP arg"; on Win64 the combined cap fires. + const std::string r(reason); + EXPECT_TRUE(r.find("FP arg") != std::string::npos || + r.find("Win64 cap") != std::string::npos); +} + +TEST(FFIFastCallEligibility, VoidArgRejected) { + auto fn = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_void}, {"void"}); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); + ASSERT_NE(reason, nullptr); + EXPECT_NE(std::string(reason).find("void"), std::string::npos); +} + +TEST(FFIFastCallEligibility, NullOutReasonOk) { + auto fn = MakeFn(&ffi_type_sint32, "i32", + {&ffi_type_sint32}, {"i32"}); + EXPECT_TRUE(IsFastCallEligible(fn, nullptr)); +} + +// Per-platform GP cap boundary tests. + +#if defined(__aarch64__) || defined(_M_ARM64) +// AArch64 allows up to 7 GP args — the largest GP cap of any supported +// platform. Verify that 7 is accepted and 8 is rejected. +TEST(FFIFastCallEligibility, AArch64GPCapBoundary) { + std::vector args(7, &ffi_type_sint32); + std::vector names(7, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} +#endif // AArch64 + +#if defined(__x86_64__) && \ + (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)) +// SysV (Linux/macOS/FreeBSD x86_64) allows up to 6 GP args. Verify that 6 +// is accepted and 7 is rejected. +TEST(FFIFastCallEligibility, SysVGPCapBoundary) { + std::vector args(6, &ffi_type_sint32); + std::vector names(6, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, SysVGPCapExceeded) { + std::vector args(7, &ffi_type_sint32); + std::vector names(7, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); +} +#endif // SysV x86_64 + +#if defined(_WIN32) && (defined(__x86_64__) || defined(_M_X64)) +// Win64 allows at most 3 combined GP+FP args. Verify the boundary. +TEST(FFIFastCallEligibility, Win64CombinedCapBoundary) { + std::vector args(3, &ffi_type_sint32); + std::vector names(3, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, Win64CombinedCapExceeded) { + std::vector args(4, &ffi_type_sint32); + std::vector names(4, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); +} +#endif // Win64 + +TEST(FFIFastCallEligibility, StringTypeNameOk) { + auto fn = MakeFn(&ffi_type_pointer, "pointer", + {&ffi_type_pointer}, {"string"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, BufferTypeNameOk) { + auto fn = MakeFn(&ffi_type_pointer, "pointer", + {&ffi_type_pointer}, {"buffer"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, ArraybufferTypeNameOk) { + auto fn = MakeFn(&ffi_type_pointer, "pointer", + {&ffi_type_pointer}, {"arraybuffer"}); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +#if defined(__arm__) +TEST(FFIFastCallEligibility, ARM32GPCapBoundary) { + std::vector args(3, &ffi_type_sint32); + std::vector names(3, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", + std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, ARM32GPCapExceeded) { + std::vector args(4, &ffi_type_sint32); + std::vector names(4, "i32"); + auto fn = MakeFn(&ffi_type_void, "void", + std::move(args), std::move(names)); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); +} + +TEST(FFIFastCallEligibility, ARM32I64ArgRejected) { + auto fn = MakeFn(&ffi_type_void, "void", + {&ffi_type_sint64}, {"i64"}); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); + ASSERT_NE(reason, nullptr); + EXPECT_NE(std::string(reason).find("aarch32"), std::string::npos); +} + +TEST(FFIFastCallEligibility, ARM32I64ReturnRejected) { + auto fn = MakeFn(&ffi_type_sint64, "i64", + {&ffi_type_sint32}, {"i32"}); + const char* reason = nullptr; + EXPECT_FALSE(IsFastCallEligible(fn, &reason)); + ASSERT_NE(reason, nullptr); + EXPECT_NE(std::string(reason).find("aarch32"), std::string::npos); +} +#endif // __arm__ + +#endif // HAVE_FFI_FASTCALL diff --git a/test/cctest/test_ffi_fastcall_emitter.cc b/test/cctest/test_ffi_fastcall_emitter.cc new file mode 100644 index 00000000000000..561e396facb6b4 --- /dev/null +++ b/test/cctest/test_ffi_fastcall_emitter.cc @@ -0,0 +1,233 @@ +#include "gtest/gtest.h" + +#ifdef HAVE_FFI_FASTCALL + +#include + +#include "ffi/fastcall/jit_memory.h" +#include "ffi/fastcall/stub_emitter.h" +#include "node_test_fixture.h" + +using node::ffi::fastcall::ArgClass; +using node::ffi::fastcall::EmitForwarder; +using node::ffi::fastcall::JitMemory; +using node::ffi::fastcall::ResultClass; + +class StubEmitterTest : public NodeTestFixture { + protected: + void TearDown() override { + JitMemory::Get(isolate_)->ResetForTesting(); + NodeTestFixture::TearDown(); + } +}; + +// Test target functions. Plain static so the optimizer doesn't merge +// them with anything else. +namespace { +int target_zero_args() { return 1234; } +int target_one_int(int a) { return a + 100; } +int target_two_int(int a, int b) { return a - b; } +int target_six_int(int a, int b, int c, int d, int e, int f) { + return a + b + c + d + e + f; +} +int target_seven_int(int a, int b, int c, int d, int e, int f, int g) { + return a + b + c + d + e + f + g; +} +double target_one_double(double a) { return a * 2.0; } +int target_int_double(int a, double b) { + return a + static_cast(b); +} +[[maybe_unused]] int target_fp_int(double a, int b) { return static_cast(a) + b; } +[[maybe_unused]] int target_three_int(int a, int b, int c) { return a + b + c; } +[[maybe_unused]] int target_four_int(int a, int b, int c, int d) { + return a + b + c + d; +} +} // namespace + +#if defined(__aarch64__) || defined(_M_ARM64) || \ + defined(__x86_64__) || defined(_M_X64) + +TEST_F(StubEmitterTest, ZeroArgs) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_zero_args), + {}, + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(1234, fn(reinterpret_cast(0xdeadbeefULL))); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, OneIntArg) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_one_int), + {ArgClass::kGP}, + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(142, fn(nullptr, 42)); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, TwoIntArgs) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_two_int), + {ArgClass::kGP, ArgClass::kGP}, + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int, int); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(7, fn(nullptr, 10, 3)); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, OneDoubleArg) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_one_double), + {ArgClass::kFP}, + ResultClass::kFP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = double (*)(void* receiver, double); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_DOUBLE_EQ(6.28, fn(nullptr, 3.14)); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, MixedIntDouble) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_int_double), + {ArgClass::kGP, ArgClass::kFP}, + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int, double); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(13, fn(nullptr, 10, 3.7)); + JitMemory::Get(isolate_)->Free(*stub); +} + +// SixIntArgs: passes on AArch64 (cap=7) and SysV x86_64 (cap=6), but fails +// on Win64 (combined GP+FP cap=3). Gate it out on Win64. +#if defined(__aarch64__) || defined(_M_ARM64) || \ + (defined(__x86_64__) && !defined(_WIN32)) +TEST_F(StubEmitterTest, SixIntArgs) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_six_int), + std::vector(6, ArgClass::kGP), + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int, int, int, int, int, int); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(21, fn(nullptr, 1, 2, 3, 4, 5, 6)); + JitMemory::Get(isolate_)->Free(*stub); +} +#endif // AArch64 || (x86_64 && !Win64) + +// SevenIntArgs: only AArch64 supports 7 GP args (cap=7). SysV cap is 6, +// Win64 cap is 3 combined — both would reject 7 GP args. +#if defined(__aarch64__) || defined(_M_ARM64) +TEST_F(StubEmitterTest, SevenIntArgs) { + // AArch64 supports up to 7 GP args after the receiver (kMaxGPArgs=7). + auto stub = EmitForwarder( + isolate_, reinterpret_cast(target_seven_int), + std::vector(7, ArgClass::kGP), ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int, int, int, int, int, int, int); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(28, fn(nullptr, 1, 2, 3, 4, 5, 6, 7)); + JitMemory::Get(isolate_)->Free(*stub); +} +#endif // AArch64 + +// SysV (Linux/macOS/FreeBSD x86_64) cap is exactly 6 GP args. 7 must fail. +#if defined(__x86_64__) && \ + (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)) +TEST_F(StubEmitterTest, SevenGPArgsRejectedSysV) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_seven_int), + std::vector(7, ArgClass::kGP), + ResultClass::kGP); + EXPECT_FALSE(stub.has_value()); +} +#endif // SysV x86_64 + +TEST_F(StubEmitterTest, EightIntArgsRejected) { + // 8 GP args would require stack handling not supported in v1. + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_seven_int), + std::vector(8, ArgClass::kGP), + ResultClass::kGP); + EXPECT_FALSE(stub.has_value()); +} + +TEST_F(StubEmitterTest, GPPairRejected) { + // kGPPair is for AArch32 only; reject on AArch64/x86_64. + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_one_int), + {ArgClass::kGPPair}, + ResultClass::kGP); + EXPECT_FALSE(stub.has_value()); +} + +#if defined(__aarch64__) || defined(_M_ARM64) +TEST_F(StubEmitterTest, NineFPArgsRejected) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_one_double), + std::vector(9, ArgClass::kFP), + ResultClass::kGP); + EXPECT_FALSE(stub.has_value()); +} +#endif // __aarch64__ + +#if defined(_WIN32) && (defined(__x86_64__) || defined(_M_X64)) +TEST_F(StubEmitterTest, Win64MixedGPThenFP) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_int_double), + {ArgClass::kGP, ArgClass::kFP}, + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int, double); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(13, fn(nullptr, 10, 3.7)); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, Win64FPThenGP) { + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_fp_int), + {ArgClass::kFP, ArgClass::kGP}, + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, double, int); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(13, fn(nullptr, 3.7, 10)); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, Win64ThreeArgsAccepted) { + // 3 args is the Win64 cap (4 register slots minus the receiver). + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_three_int), + std::vector(3, ArgClass::kGP), + ResultClass::kGP); + ASSERT_TRUE(stub.has_value()); + using FnPtr = int (*)(void* receiver, int, int, int); + FnPtr fn = reinterpret_cast(stub->entry); + EXPECT_EQ(6, fn(nullptr, 1, 2, 3)); + JitMemory::Get(isolate_)->Free(*stub); +} + +TEST_F(StubEmitterTest, Win64FourArgsRejected) { + // 4 args exceeds the Win64 register-slot cap. + auto stub = EmitForwarder(isolate_, + reinterpret_cast(target_four_int), + std::vector(4, ArgClass::kGP), + ResultClass::kGP); + EXPECT_FALSE(stub.has_value()); +} +#endif // _WIN32 && x86_64 + +#endif // __aarch64__ || x86_64 + +#endif // HAVE_FFI_FASTCALL diff --git a/test/cctest/test_ffi_fastcall_jit.cc b/test/cctest/test_ffi_fastcall_jit.cc new file mode 100644 index 00000000000000..473be6abb5e7dd --- /dev/null +++ b/test/cctest/test_ffi_fastcall_jit.cc @@ -0,0 +1,159 @@ +#include "gtest/gtest.h" + +#ifdef HAVE_FFI_FASTCALL + +#include +#include + +#if defined(__POSIX__) +#include +#endif + +#include "ffi/fastcall/jit_memory.h" +#include "node_test_fixture.h" + +namespace { +size_t SysPageSize() { +#if defined(__POSIX__) + return static_cast(sysconf(_SC_PAGESIZE)); +#else + return 4096; +#endif +} +} // namespace + +using node::ffi::fastcall::JitMemory; + +class JitMemoryTest : public NodeTestFixture { + protected: + void TearDown() override { + JitMemory::Get(isolate_)->ResetForTesting(); + NodeTestFixture::TearDown(); + } +}; + +TEST_F(JitMemoryTest, AllocateZeroFails) { + auto* jit = JitMemory::Get(isolate_); + size_t alloc_size = 0; + EXPECT_EQ(nullptr, jit->Allocate(0, &alloc_size)); +} + +TEST_F(JitMemoryTest, AllocateSmallReturnsAligned) { + auto* jit = JitMemory::Get(isolate_); + size_t alloc_size = 0; + void* p = jit->Allocate(32, &alloc_size); + ASSERT_NE(nullptr, p); + EXPECT_GE(alloc_size, 32u); + EXPECT_EQ(0u, reinterpret_cast(p) % 64); + jit->Free(p, alloc_size); +} + +TEST_F(JitMemoryTest, FreeIsNoOp) { + auto* jit = JitMemory::Get(isolate_); + size_t a = 0, b = 0; + void* p1 = jit->Allocate(48, &a); + ASSERT_NE(nullptr, p1); + size_t live_before = jit->TotalLiveBytes(); + jit->Free(p1, a); + EXPECT_LT(jit->TotalLiveBytes(), live_before); + void* p2 = jit->Allocate(48, &b); + ASSERT_NE(nullptr, p2); + EXPECT_NE(p1, p2); + jit->Free(p2, b); +} + +TEST_F(JitMemoryTest, ManySmallAllocsFit) { + auto* jit = JitMemory::Get(isolate_); + std::vector> ptrs; + for (int i = 0; i < 200; ++i) { + size_t a = 0; + void* p = jit->Allocate(40, &a); + ASSERT_NE(nullptr, p) << "alloc " << i; + ptrs.push_back({p, a}); + } + for (auto& [p, a] : ptrs) jit->Free(p, a); +} + +#if defined(__x86_64__) || defined(_M_X64) || \ + defined(__aarch64__) || defined(_M_ARM64) +TEST_F(JitMemoryTest, ExecutableStubReturnsValue) { + auto* jit = JitMemory::Get(isolate_); + size_t alloc_size = 0; + void* p = jit->Allocate(64, &alloc_size); + ASSERT_NE(nullptr, p); + + uint8_t* code = static_cast(p); +#if defined(__x86_64__) || defined(_M_X64) + // mov eax, 42; ret + code[0] = 0xb8; code[1] = 0x2a; code[2] = 0x00; + code[3] = 0x00; code[4] = 0x00; + code[5] = 0xc3; +#elif defined(__aarch64__) || defined(_M_ARM64) + uint32_t* w = reinterpret_cast(code); + w[0] = 0x52800540; // mov w0, #42 + w[1] = 0xd65f03c0; // ret +#endif + + ASSERT_TRUE(jit->MakeExecutable(p, alloc_size)); + using FnPtr = int (*)(); + EXPECT_EQ(42, reinterpret_cast(p)()); + jit->Free(p, alloc_size); +} +#endif // x86_64 || aarch64 + +TEST_F(JitMemoryTest, SelfTestPasses) { + EXPECT_TRUE(JitMemory::Get(isolate_)->SelfTest(isolate_)); +} + +TEST_F(JitMemoryTest, MultiplePagesAllocate) { + auto* jit = JitMemory::Get(isolate_); + // Force at least 2 pages by allocating many large chunks. + // 1000 × 1024 bytes = ~1 MB, guaranteed to span pages on any platform. + std::vector> ptrs; + for (int i = 0; i < 1000; ++i) { + size_t a = 0; + void* p = jit->Allocate(1024, &a); + ASSERT_NE(nullptr, p) << "alloc " << i; + ptrs.push_back({p, a}); + } + // Sanity: at least two distinct page bases observed. + const size_t page_mask = ~(SysPageSize() - 1); + std::set page_bases; + for (auto& [p, ignored] : ptrs) { + page_bases.insert(reinterpret_cast(p) & page_mask); + } + EXPECT_GE(page_bases.size(), 2u); + for (auto& [p, a] : ptrs) jit->Free(p, a); +} + +TEST_F(JitMemoryTest, MakeExecutableNullArgsSafe) { + auto* jit = JitMemory::Get(isolate_); + // Verify graceful handling of nullptr / zero size. The page_size_==0 + // uninitialized branch is unreachable here because Get(isolate_) already + // initialized the singleton; that branch is defense-in-depth only. + EXPECT_FALSE(jit->MakeExecutable(nullptr, 64)); + EXPECT_FALSE(jit->MakeExecutable(reinterpret_cast(0x1), 0)); +} + +TEST_F(JitMemoryTest, AllocateAfterMakeExecutableSkipsSealedPage) { + auto* jit = JitMemory::Get(isolate_); + size_t a1 = 0; + void* p1 = jit->Allocate(64, &a1); + ASSERT_NE(nullptr, p1); + ASSERT_TRUE(jit->MakeExecutable(p1, a1)); + + // After sealing, a new alloc should NOT be in the same page. + size_t a2 = 0; + void* p2 = jit->Allocate(64, &a2); + ASSERT_NE(nullptr, p2); + const uintptr_t page_mask = ~(SysPageSize() - 1); + uintptr_t page1 = reinterpret_cast(p1) & page_mask; + uintptr_t page2 = reinterpret_cast(p2) & page_mask; + EXPECT_NE(page1, page2); + + jit->Free(p2, a2); + // p1 was already made executable; freeing is fine (counter only). + jit->Free(p1, a1); +} + +#endif // HAVE_FFI_FASTCALL diff --git a/test/ffi/fixture_library/ffi_test_library.c b/test/ffi/fixture_library/ffi_test_library.c index 4a57b9c9970a9a..776c0bc01a1c9b 100644 --- a/test/ffi/fixture_library/ffi_test_library.c +++ b/test/ffi/fixture_library/ffi_test_library.c @@ -444,6 +444,24 @@ FFI_EXPORT void array_set_f64(double* arr, size_t index, double value) { arr[index] = value; } +// Arity boundary test helpers. + +FFI_EXPORT int add_5(int a, int b, int c, int d, int e) { + return a + b + c + d + e; +} + +FFI_EXPORT int add_6(int a, int b, int c, int d, int e, int f) { + return a + b + c + d + e + f; +} + +FFI_EXPORT int add_7(int a, int b, int c, int d, int e, int f, int g) { + return a + b + c + d + e + f + g; +} + +// Pointer echo for pointer-fallback test. + +FFI_EXPORT void* return_pointer_arg(void* p) { return p; } + #ifndef _WIN32 FFI_EXPORT void* readonly_memory() { // TODO(bengl) Add a Windows version of this. diff --git a/test/ffi/test-ffi-dynamic-library.js b/test/ffi/test-ffi-dynamic-library.js index a80fe5286afab9..6c475f7b73bfe0 100644 --- a/test/ffi/test-ffi-dynamic-library.js +++ b/test/ffi/test-ffi-dynamic-library.js @@ -50,8 +50,8 @@ test('dlopen resolves functions from definitions', () => { assert.strictEqual(functions.add_f32(1.25, 2.75), 4); assert.strictEqual(functions.add_u64(20n, 22n), 42n); assert.strictEqual(functions.add_i32.name, 'add_i32'); - // Shared-buffer wrapper sets `length` to the FFI signature's arity - // (see `inheritMetadata` in lib/internal/ffi-shared-buffer.js). The raw + // Fast-call wrapper sets `length` to the FFI signature's arity + // (see `inheritMetadata` in lib/internal/ffi-fastcall.js). The raw // native function has length 0, but the wrapper exposes the parameter // count so `fn.length` is useful for introspection. assert.strictEqual(functions.add_i32.length, 2); diff --git a/test/ffi/test-ffi-fastcall-arity.js b/test/ffi/test-ffi-fastcall-arity.js new file mode 100644 index 00000000000000..41073cd6e860e6 --- /dev/null +++ b/test/ffi/test-ffi-fastcall-arity.js @@ -0,0 +1,90 @@ +// Flags: --experimental-ffi +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); + +const assert = require('node:assert'); +const { test } = require('node:test'); +const ffi = require('node:ffi'); +const { libraryPath } = require('./ffi-test-common'); + +test('0 args (getpid-like)', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + char_is_signed: { result: 'i32', parameters: [] }, + }); + try { + const v = functions.char_is_signed(); + assert.strictEqual(typeof v, 'number'); + // char_is_signed returns 0 or 1 depending on platform. + assert.ok(v === 0 || v === 1); + } finally { lib.close(); } +}); + +test('0-arg function rejects extra args', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + char_is_signed: { result: 'i32', parameters: [] }, + }); + try { + assert.throws(() => functions.char_is_signed(1), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.char_is_signed(1, 2), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { lib.close(); } +}); + +test('3 GP args', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + sum_3_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32'] }, + }); + try { + assert.strictEqual(functions.sum_3_i32(1, 2, 3), 6); + } finally { lib.close(); } +}); + +test('4 GP args', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + sum_4_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32'] }, + }); + try { + assert.strictEqual(functions.sum_4_i32(1, 2, 3, 4), 10); + } finally { lib.close(); } +}); + +test('wrong argument count throws', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + try { + assert.throws(() => functions.add_i32(1), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_i32(1, 2, 3), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { lib.close(); } +}); + +test('5 GP args (SysV register limit)', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_5: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32'] }, + }); + try { + assert.strictEqual(functions.add_5(1, 2, 3, 4, 5), 15); + } finally { lib.close(); } +}); + +test('6 GP args (SysV stack overflow boundary)', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_6: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32'] }, + }); + try { + assert.strictEqual(functions.add_6(1, 2, 3, 4, 5, 6), 21); + } finally { lib.close(); } +}); + +test('7 GP args (AArch64 boundary)', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_7: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32'] }, + }); + try { + assert.strictEqual(functions.add_7(1, 2, 3, 4, 5, 6, 7), 28); + } finally { lib.close(); } +}); diff --git a/test/ffi/test-ffi-fastcall-close.js b/test/ffi/test-ffi-fastcall-close.js new file mode 100644 index 00000000000000..68a93a9578f6ed --- /dev/null +++ b/test/ffi/test-ffi-fastcall-close.js @@ -0,0 +1,56 @@ +// Flags: --experimental-ffi +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); + +const assert = require('node:assert'); +const { test } = require('node:test'); +const ffi = require('node:ffi'); +const { libraryPath } = require('./ffi-test-common'); + +test('close throws ERR_FFI_LIBRARY_CLOSED on subsequent fast-call', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + // Warm up TurboFan with many calls. + for (let i = 0; i < 10000; i++) { + functions.add_i32(i, i); + } + lib.close(); + assert.throws(() => functions.add_i32(1, 2), + { code: 'ERR_FFI_LIBRARY_CLOSED' }); +}); + +test('close throws ERR_FFI_LIBRARY_CLOSED via libffi slow path (ineligible signature)', () => { + // 'function' as a parameter type makes IsFastCallEligible return false, so + // the fast-call wrapper is never applied and the function dispatches directly + // to InvokeFunction (libffi slow path). This exercises InvokeFunction's own + // fn->closed check rather than the JS wrapper's alive-buffer check. + // We declare return_pointer_arg with a 'function' param even though its real + // ABI takes a void*; the closed check fires before any ABI work, so the + // mismatch doesn't matter. + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['function'] }, + }); + // Don't call pre-close: the ABI-mismatched signature would produce garbage. + lib.close(); + assert.throws(() => functions.return_pointer_arg(0n), + { code: 'ERR_FFI_LIBRARY_CLOSED' }); +}); + +test('ERR_FFI_LIBRARY_CLOSED is an Error instance with the right name', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + lib.close(); + let caught; + try { + functions.add_i32(1, 2); + } catch (err) { + caught = err; + } + assert.ok(caught instanceof Error); + assert.strictEqual(caught.code, 'ERR_FFI_LIBRARY_CLOSED'); + assert.strictEqual(caught.name, 'Error'); + assert.strictEqual(caught.message, 'Library is closed'); +}); diff --git a/test/ffi/test-ffi-fastcall-eligibility.js b/test/ffi/test-ffi-fastcall-eligibility.js new file mode 100644 index 00000000000000..5b55a098b49b94 --- /dev/null +++ b/test/ffi/test-ffi-fastcall-eligibility.js @@ -0,0 +1,50 @@ +// Flags: --experimental-ffi --expose-internals +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); + +const assert = require('node:assert'); +const { test } = require('node:test'); + +// Capture the unpatched method BEFORE requiring node:ffi. +// ffi-fastcall.js patches DynamicLibrary.prototype.getFunction when +// node:ffi is loaded; grabbing the raw binding first lets us call +// getFunction without the wrapper, so we can inspect the Symbol-keyed +// metadata that the fast-call path attaches. +const { internalBinding } = require('internal/test/binding'); +const ffiBinding = internalBinding('ffi'); +const rawGetFunction = ffiBinding.DynamicLibrary.prototype.getFunction; + +require('node:ffi'); // load AFTER capturing the unpatched method +const { libraryPath } = require('./ffi-test-common'); + +const fastcallEnabled = ffiBinding.kFastcallAlive !== undefined; + +test('eligible signature has fastcall metadata', { skip: !fastcallEnabled }, () => { + const lib = new ffiBinding.DynamicLibrary(libraryPath); + try { + const raw = rawGetFunction.call(lib, 'add_i32', + { result: 'i32', parameters: ['i32', 'i32'] }); + assert.notStrictEqual(raw[ffiBinding.kFastcallAlive], undefined); + assert.notStrictEqual(raw[ffiBinding.kFastcallInvokeSlow], undefined); + assert.notStrictEqual(raw[ffiBinding.kFastcallParams], undefined); + assert.notStrictEqual(raw[ffiBinding.kFastcallResult], undefined); + } finally { + lib.close(); + } +}); + +test('signature with function type lacks fastcall metadata', + { skip: !fastcallEnabled }, () => { + const lib = new ffiBinding.DynamicLibrary(libraryPath); + try { + // Use a real fixture symbol but declare the signature with a function-typed + // arg. Eligibility should reject this; the inner v8::Function is plain + // (no fastcall metadata). + const raw = rawGetFunction.call(lib, 'add_i32', + { result: 'i32', parameters: ['function', 'i32'] }); + assert.strictEqual(raw[ffiBinding.kFastcallAlive], undefined); + } finally { + lib.close(); + } +}); diff --git a/test/ffi/test-ffi-fastcall-optimized.js b/test/ffi/test-ffi-fastcall-optimized.js new file mode 100644 index 00000000000000..624fb605c9a231 --- /dev/null +++ b/test/ffi/test-ffi-fastcall-optimized.js @@ -0,0 +1,160 @@ +// Flags: --experimental-ffi --allow-natives-syntax --no-warnings +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); + +// Verify strict-numeric signatures still throw ERR_INVALID_ARG_VALUE for +// non-conforming inputs after V8 fast-call engages. Without per-arg +// validation in the wrapper, V8's CheckedNumberAsWord32 / CheckedBigIntTrunc- +// atingWord64 silently truncate out-of-range and non-integer inputs in the +// fast path — yielding tier-dependent semantics where the same FFI binding +// throws cold and silently corrupts hot. The wrapper must run validateAnd- +// Coerce before the fast-call so the contract holds across all tiers. + +const assert = require('node:assert'); +const { test } = require('node:test'); +const ffi = require('node:ffi'); +const { libraryPath } = require('./ffi-test-common'); + +function optimize(fn) { + // %PrepareFunctionForOptimization marks the function and runs profiling; + // %OptimizeFunctionOnNextCall forces TurboFan on the next invocation. + // After that call, the function is fully optimized and the inner + // v8::Function fast-call path is engaged for matching arg shapes. + eval('%PrepareFunctionForOptimization(fn)'); + fn(); + eval('%OptimizeFunctionOnNextCall(fn)'); + fn(); +} + +test('i32 strict validation holds after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + try { + const callOk = () => assert.strictEqual(functions.add_i32(1, 2), 3); + optimize(callOk); + + // Out-of-range int: must throw, not silently wrap to -2147483648. + assert.throws(() => functions.add_i32(2147483648, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + // Non-integer number: must throw, not silently floor to 1. + assert.throws(() => functions.add_i32(1.5, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + // NaN: must throw, not coerce to 0. + assert.throws(() => functions.add_i32(NaN, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('u32 strict validation holds after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_u32: { result: 'u32', parameters: ['u32', 'u32'] }, + }); + try { + const callOk = () => assert.strictEqual(functions.add_u32(1, 2), 3); + optimize(callOk); + + assert.throws(() => functions.add_u32(-1, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_u32(4294967296, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_u32(1.5, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('i64 strict validation holds after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i64: { result: 'i64', parameters: ['i64', 'i64'] }, + }); + try { + const callOk = () => assert.strictEqual(functions.add_i64(1n, 2n), 3n); + optimize(callOk); + + // Out-of-range BigInt: V8's CheckedBigIntTruncatingWord64 would silently + // wrap to low 64 bits in the fast path. The JS validation must catch it. + assert.throws(() => functions.add_i64(1n << 100n, 0n), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_i64(1n << 63n, 0n), + { code: 'ERR_INVALID_ARG_VALUE' }); + // Number passed where BigInt expected. + assert.throws(() => functions.add_i64(1, 2), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('u64 strict validation holds after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_u64: { result: 'u64', parameters: ['u64', 'u64'] }, + }); + try { + const callOk = () => assert.strictEqual(functions.add_u64(1n, 2n), 3n); + optimize(callOk); + + // Negative BigInt passed to u64: V8 truncation would wrap to 2^64-1. + assert.throws(() => functions.add_u64(-1n, 0n), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_u64(1n << 64n, 0n), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('f64 type validation holds after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_f64: { result: 'f64', parameters: ['f64', 'f64'] }, + }); + try { + const callOk = () => assert.strictEqual(functions.add_f64(1.5, 2.5), 4); + optimize(callOk); + + // Non-number must throw (V8's fast-call accepts any number including + // NaN/Inf, but rejects non-numbers via deopt to slow path). + assert.throws(() => functions.add_f64('1', 2), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_f64(1n, 2), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('arity errors hold after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + try { + const callOk = () => assert.strictEqual(functions.add_i32(1, 2), 3); + optimize(callOk); + + assert.throws(() => functions.add_i32(1), + { code: 'ERR_INVALID_ARG_VALUE', + message: /Invalid argument count: expected 2/ }); + assert.throws(() => functions.add_i32(1, 2, 3), + { code: 'ERR_INVALID_ARG_VALUE', + message: /Invalid argument count: expected 2/ }); + } finally { + lib.close(); + } +}); + +test('close-while-live check holds after fast-call engages', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + const callOk = () => assert.strictEqual(functions.add_i32(1, 2), 3); + optimize(callOk); + + lib.close(); + + assert.throws(() => functions.add_i32(1, 2), + { code: 'ERR_FFI_LIBRARY_CLOSED' }); +}); diff --git a/test/ffi/test-ffi-fastcall-pointer-fallback.js b/test/ffi/test-ffi-fastcall-pointer-fallback.js new file mode 100644 index 00000000000000..db5c3a855ffa78 --- /dev/null +++ b/test/ffi/test-ffi-fastcall-pointer-fallback.js @@ -0,0 +1,105 @@ +// Flags: --experimental-ffi +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); + +const assert = require('node:assert'); +const { test } = require('node:test'); +const ffi = require('node:ffi'); +const { libraryPath } = require('./ffi-test-common'); + +test('null pointer takes fast path (no throw)', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['pointer'] }, + }); + try { + // Fast path: BigInt + assert.strictEqual(functions.return_pointer_arg(0xDEADBEEFn), 0xDEADBEEFn); + // Fast path: null + assert.strictEqual(functions.return_pointer_arg(null), 0n); + // Slow path: Buffer (routes via slowInvoke) + const buf = Buffer.from([1, 2, 3, 4]); + const ptr = functions.return_pointer_arg(buf); + assert.strictEqual(typeof ptr, 'bigint'); + assert.notStrictEqual(ptr, 0n); + } finally { + lib.close(); + } +}); + +test('ArrayBuffer routes through slow path', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['pointer'] }, + }); + try { + const ab = new ArrayBuffer(8); + const ptr = functions.return_pointer_arg(ab); + assert.strictEqual(typeof ptr, 'bigint'); + assert.notStrictEqual(ptr, 0n); + } finally { lib.close(); } +}); + +test('ArrayBufferView with byteOffset routes through slow path', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['pointer'] }, + }); + try { + const ab = new ArrayBuffer(16); + const view = new Uint8Array(ab, 4, 8); // non-zero byteOffset + const ptr = functions.return_pointer_arg(view); + assert.strictEqual(typeof ptr, 'bigint'); + // Pointer should be ab.base + 4, but we don't know the base. + // Just verify it's non-zero and a bigint. + assert.notStrictEqual(ptr, 0n); + } finally { lib.close(); } +}); + +test('ArrayBufferView with byteOffset propagates the offset', () => { + // Use return_pointer_arg to echo back the pointer. Call it with two views + // into the same ArrayBuffer at different offsets; the returned pointers + // must differ by exactly the offset delta (4 bytes). + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['pointer'] }, + }); + try { + const ab = new ArrayBuffer(32); + const view0 = new Uint8Array(ab, 0, 16); + const view4 = new Uint8Array(ab, 4, 16); + const ptr0 = functions.return_pointer_arg(view0); + const ptr4 = functions.return_pointer_arg(view4); + assert.strictEqual(typeof ptr0, 'bigint'); + assert.strictEqual(typeof ptr4, 'bigint'); + assert.strictEqual(ptr4 - ptr0, 4n, + `expected ptr4 - ptr0 === 4n, got ${ptr4 - ptr0}`); + } finally { lib.close(); } +}); + +test('string routes through slow path with declared "string" type', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['string'] }, + }); + try { + const ptr = functions.return_pointer_arg('hello'); + assert.strictEqual(typeof ptr, 'bigint'); + assert.notStrictEqual(ptr, 0n); + } finally { lib.close(); } +}); + +test('undefined pointer maps to 0n', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['pointer'] }, + }); + try { + assert.strictEqual(functions.return_pointer_arg(undefined), 0n); + } finally { lib.close(); } +}); + +test('out-of-range BigInt rejected at wrapper', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + return_pointer_arg: { result: 'pointer', parameters: ['pointer'] }, + }); + try { + assert.throws(() => functions.return_pointer_arg(-1n), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { lib.close(); } +}); diff --git a/test/ffi/test-ffi-fastcall-types.js b/test/ffi/test-ffi-fastcall-types.js new file mode 100644 index 00000000000000..fdf6637cc62738 --- /dev/null +++ b/test/ffi/test-ffi-fastcall-types.js @@ -0,0 +1,83 @@ +// Flags: --experimental-ffi +'use strict'; +const common = require('../common'); +common.skipIfFFIMissing(); + +const assert = require('node:assert'); +const { test } = require('node:test'); +const ffi = require('node:ffi'); +const { libraryPath } = require('./ffi-test-common'); + +test('i32 add', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, + }); + try { + assert.strictEqual(functions.add_i32(20, 22), 42); + assert.strictEqual(functions.add_i32(-100, 50), -50); + assert.strictEqual(functions.add_i32(2147483647, 0), 2147483647); + assert.throws(() => functions.add_i32(2147483648, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('i64 add via BigInt', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i64: { result: 'i64', parameters: ['i64', 'i64'] }, + }); + try { + assert.strictEqual(functions.add_i64(1n, 2n), 3n); + assert.strictEqual(functions.add_i64(-1n, 1n), 0n); + assert.throws(() => functions.add_i64(1, 2), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('i8 add with range checks', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_i8: { result: 'i8', parameters: ['i8', 'i8'] }, + }); + try { + assert.strictEqual(functions.add_i8(10, 20), 30); + assert.strictEqual(functions.add_i8(-128, 0), -128); + assert.throws(() => functions.add_i8(128, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_i8(-129, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('u8 add with range checks', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_u8: { result: 'u8', parameters: ['u8', 'u8'] }, + }); + try { + assert.strictEqual(functions.add_u8(0, 0), 0); + assert.strictEqual(functions.add_u8(255, 0), 255); + assert.throws(() => functions.add_u8(-1, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + assert.throws(() => functions.add_u8(256, 0), + { code: 'ERR_INVALID_ARG_VALUE' }); + } finally { + lib.close(); + } +}); + +test('float and double round-trip', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + add_f32: { result: 'f32', parameters: ['f32', 'f32'] }, + add_f64: { result: 'f64', parameters: ['f64', 'f64'] }, + }); + try { + assert.ok(Math.abs(functions.add_f32(1.5, 2.5) - 4.0) < 1e-6); + assert.strictEqual(functions.add_f64(1.5, 2.5), 4.0); + } finally { + lib.close(); + } +}); diff --git a/test/ffi/test-ffi-shared-buffer.js b/test/ffi/test-ffi-shared-buffer.js deleted file mode 100644 index 43d76a4da18315..00000000000000 --- a/test/ffi/test-ffi-shared-buffer.js +++ /dev/null @@ -1,806 +0,0 @@ -// Flags: --experimental-ffi --expose-internals -'use strict'; -const common = require('../common'); -common.skipIfFFIMissing(); - -const assert = require('node:assert'); -const { test } = require('node:test'); - -// Capture the unpatched DynamicLibrary.prototype.getFunction BEFORE loading -// `node:ffi`, which patches it. The SB-metadata test below uses the raw -// method to inspect Symbol-keyed internals that `inheritMetadata` -// deliberately does not forward onto the wrapper. -const { internalBinding } = require('internal/test/binding'); -const ffiBinding = internalBinding('ffi'); -const { - kSbInvokeSlow, - kSbParams, - kSbResult, - kSbSharedBuffer, -} = ffiBinding; -const rawGetFunctionUnpatched = ffiBinding.DynamicLibrary.prototype.getFunction; - -const ffi = require('node:ffi'); -const { libraryPath } = require('./ffi-test-common'); - -test('numeric-only i32 function uses SB path', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - }); - try { - assert.strictEqual(functions.add_i32(20, 22), 42); - assert.strictEqual(functions.add_i32(-10, 10), 0); - assert.strictEqual(functions.add_i32(0, 0), 0); - assert.strictEqual(functions.add_i32(2147483647, 0), 2147483647); - } finally { - lib.close(); - } -}); - -test('i8/u8/i16/u16 round-trip', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i8: { result: 'i8', parameters: ['i8', 'i8'] }, - add_u8: { result: 'u8', parameters: ['u8', 'u8'] }, - add_i16: { result: 'i16', parameters: ['i16', 'i16'] }, - add_u16: { result: 'u16', parameters: ['u16', 'u16'] }, - }); - try { - assert.strictEqual(functions.add_i8(10, 20), 30); - assert.strictEqual(functions.add_u8(100, 155), 255); - assert.strictEqual(functions.add_i16(1000, 2000), 3000); - assert.strictEqual(functions.add_u16(30000, 35535), 65535); - } finally { - lib.close(); - } -}); - -test('f32/f64 round-trip', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_f32: { result: 'f32', parameters: ['f32', 'f32'] }, - add_f64: { result: 'f64', parameters: ['f64', 'f64'] }, - }); - try { - // 1.25 and 2.75 are exactly representable in float32, so the sum is exact. - assert.strictEqual(functions.add_f32(1.25, 2.75), 4.0); - assert.strictEqual(functions.add_f64(1.5, 2.5), 4.0); - } finally { - lib.close(); - } -}); - -test('i64/u64 BigInt round-trip', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i64: { result: 'i64', parameters: ['i64', 'i64'] }, - add_u64: { result: 'u64', parameters: ['u64', 'u64'] }, - }); - try { - assert.strictEqual(functions.add_i64(10n, 20n), 30n); - assert.strictEqual(functions.add_u64(10n, 20n), 30n); - } finally { - lib.close(); - } -}); - -test('zero-arg function', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - char_is_signed: { result: 'i32', parameters: [] }, - }); - try { - const result = functions.char_is_signed(); - assert.strictEqual(typeof result, 'number'); - assert.ok(result === 0 || result === 1); - } finally { - lib.close(); - } -}); - -test('6-arg numeric function', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - sum_6_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32'] }, - }); - try { - assert.strictEqual(functions.sum_6_i32(1, 2, 3, 4, 5, 6), 21); - } finally { - lib.close(); - } -}); - -test('pointer args: fast path (BigInt/null) and slow-path fallback (Buffer/ArrayBuffer)', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - identity_pointer: { result: 'pointer', parameters: ['pointer'] }, - pointer_to_usize: { result: 'u64', parameters: ['pointer'] }, - }); - try { - assert.strictEqual(functions.identity_pointer(0n), 0n); - assert.strictEqual(functions.identity_pointer(0x1234n), 0x1234n); - assert.strictEqual(functions.identity_pointer(null), 0n); - assert.strictEqual(functions.identity_pointer(undefined), 0n); - assert.strictEqual(functions.pointer_to_usize(0x42n), 0x42n); - - const buf = Buffer.from('hello'); - const bufPtr = functions.identity_pointer(buf); - assert.strictEqual(typeof bufPtr, 'bigint'); - assert.strictEqual(bufPtr, ffi.getRawPointer(buf)); - - const abPtr = functions.identity_pointer(new ArrayBuffer(16)); - assert.strictEqual(typeof abPtr, 'bigint'); - assert.ok(abPtr !== 0n); - } finally { - lib.close(); - } -}); - -test('string pointer uses slow-path fallback', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - string_length: { result: 'u64', parameters: ['pointer'] }, - }); - try { - assert.strictEqual(functions.string_length('hello'), 5n); - // strlen(NULL) is UB, so use a NUL-terminated Buffer for the fast path. - assert.strictEqual(functions.string_length(Buffer.from('world\0')), 5n); - } finally { - lib.close(); - } -}); - -test('non-SB-eligible signature falls back to raw function', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - string_duplicate: { result: 'pointer', parameters: ['pointer'] }, - free_string: { result: 'void', parameters: ['pointer'] }, - }); - try { - const dup = functions.string_duplicate('round-trip'); - assert.strictEqual(typeof dup, 'bigint'); - assert.ok(dup !== 0n); - functions.free_string(dup); - } finally { - lib.close(); - } -}); - -test('reentrancy across two FFI symbols', () => { - // A JS callback invoked by one FFI function reenters a different FFI - // function. Each has its own ArrayBuffer; neither may clobber the other. - const { lib, functions } = ffi.dlopen(libraryPath, { - call_int_callback: { result: 'i32', parameters: ['pointer', 'i32'] }, - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - }); - - let callDepth = 0; - let innerResult = -1; - const callback = lib.registerCallback( - { result: 'i32', parameters: ['i32'] }, - (x) => { - callDepth++; - if (callDepth === 1) innerResult = functions.add_i32(x, 100); - return x * 2; - }, - ); - - try { - const outer = functions.call_int_callback(callback, 7); - assert.strictEqual(innerResult, 107); - assert.strictEqual(outer, 14); - } finally { - lib.unregisterCallback(callback); - lib.close(); - } -}); - -test('arity mismatch throws ERR_INVALID_ARG_VALUE', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - }); - try { - assert.throws(() => functions.add_i32(1), { - code: 'ERR_INVALID_ARG_VALUE', - message: /Invalid argument count: expected 2, got 1/, - }); - assert.throws(() => functions.add_i32(1, 2, 3), { - code: 'ERR_INVALID_ARG_VALUE', - message: /Invalid argument count: expected 2, got 3/, - }); - } finally { - lib.close(); - } -}); - -test('arity 7+ uses the generic rest-params branch', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - sum_7_i32: { - result: 'i32', - parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32'], - }, - }); - try { - assert.strictEqual(functions.sum_7_i32(1, 2, 3, 4, 5, 6, 7), 28); - assert.throws( - () => functions.sum_7_i32(1, 2, 3, 4, 5, 6), - { code: 'ERR_INVALID_ARG_VALUE', message: /expected 7, got 6/ }, - ); - } finally { - lib.close(); - } -}); - -test('wrappers preserve name/length/pointer and the functions accessor returns wrappers', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - identity_pointer: { result: 'pointer', parameters: ['pointer'] }, - }); - try { - assert.strictEqual(functions.add_i32.name, 'add_i32'); - assert.strictEqual(typeof functions.add_i32.pointer, 'bigint'); - assert.ok(functions.add_i32.pointer !== 0n); - - assert.strictEqual(functions.identity_pointer.name, 'identity_pointer'); - assert.strictEqual(typeof functions.identity_pointer.pointer, 'bigint'); - assert.ok(functions.identity_pointer.pointer !== 0n); - - // `lib.functions.*` must also go through the SB wrapper. - assert.strictEqual(typeof lib.functions.add_i32, 'function'); - assert.strictEqual(lib.functions.add_i32(20, 22), 42); - assert.strictEqual(lib.functions.identity_pointer(0x1234n), 0x1234n); - } finally { - lib.close(); - } -}); - -test('integer boundaries for i8/u8/i16/u16/i32/u32', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i8: { result: 'i8', parameters: ['i8', 'i8'] }, - add_u8: { result: 'u8', parameters: ['u8', 'u8'] }, - add_i16: { result: 'i16', parameters: ['i16', 'i16'] }, - add_u16: { result: 'u16', parameters: ['u16', 'u16'] }, - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - add_u32: { result: 'u32', parameters: ['u32', 'u32'] }, - }); - - try { - assert.strictEqual(functions.add_i8(127, 0), 127); - assert.strictEqual(functions.add_i8(-128, 0), -128); - assert.strictEqual(functions.add_u8(255, 0), 255); - assert.strictEqual(functions.add_u8(0, 0), 0); - assert.strictEqual(functions.add_i16(32767, 0), 32767); - assert.strictEqual(functions.add_i16(-32768, 0), -32768); - assert.strictEqual(functions.add_u16(65535, 0), 65535); - assert.strictEqual(functions.add_i32(2147483647, 0), 2147483647); - assert.strictEqual(functions.add_i32(-2147483648, 0), -2147483648); - assert.strictEqual(functions.add_u32(4294967295, 0), 4294967295); - assert.strictEqual(functions.add_u32(0, 0), 0); - - const expect = { code: 'ERR_INVALID_ARG_VALUE' }; - assert.throws(() => functions.add_i8(128, 0), expect); - assert.throws(() => functions.add_i8(-129, 0), expect); - assert.throws(() => functions.add_u8(256, 0), expect); - assert.throws(() => functions.add_u8(-1, 0), expect); - assert.throws(() => functions.add_i16(32768, 0), expect); - assert.throws(() => functions.add_i16(-32769, 0), expect); - assert.throws(() => functions.add_u16(65536, 0), expect); - assert.throws(() => functions.add_u16(-1, 0), expect); - assert.throws(() => functions.add_i32(2147483648, 0), expect); - assert.throws(() => functions.add_i32(-2147483649, 0), expect); - assert.throws(() => functions.add_u32(4294967296, 0), expect); - assert.throws(() => functions.add_u32(-1, 0), expect); - - assert.throws(() => functions.add_i32(1.5, 0), expect); - assert.throws(() => functions.add_i32(NaN, 0), expect); - assert.throws(() => functions.add_i32(Infinity, 0), expect); - assert.throws(() => functions.add_i32('1', 0), expect); - } finally { - lib.close(); - } -}); - -test('i64/u64 BigInt boundaries and Number/BigInt type mismatches', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i64: { result: 'i64', parameters: ['i64', 'i64'] }, - add_u64: { result: 'u64', parameters: ['u64', 'u64'] }, - }); - - try { - const I64_MAX = (1n << 63n) - 1n; - const I64_MIN = -(1n << 63n); - const U64_MAX = (1n << 64n) - 1n; - - assert.strictEqual(functions.add_i64(I64_MAX, 0n), I64_MAX); - assert.strictEqual(functions.add_i64(I64_MIN, 0n), I64_MIN); - assert.strictEqual(functions.add_u64(U64_MAX, 0n), U64_MAX); - assert.strictEqual(functions.add_u64(0n, 0n), 0n); - - const expect = { code: 'ERR_INVALID_ARG_VALUE' }; - assert.throws(() => functions.add_i64(I64_MAX + 1n, 0n), expect); - assert.throws(() => functions.add_i64(I64_MIN - 1n, 0n), expect); - assert.throws(() => functions.add_u64(U64_MAX + 1n, 0n), expect); - assert.throws(() => functions.add_u64(-1n, 0n), expect); - - assert.throws(() => functions.add_i64(1, 2n), expect); - assert.throws(() => functions.add_i64(1n, '2'), expect); - } finally { - lib.close(); - } -}); - -test('char type picks signed/unsigned range based on host ABI', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - char_is_signed: { result: 'i32', parameters: [] }, - identity_char: { result: 'char', parameters: ['char'] }, - }); - - try { - const isSigned = functions.char_is_signed() !== 0; - const expect = { code: 'ERR_INVALID_ARG_VALUE' }; - - assert.strictEqual(functions.identity_char(65), 65); - - if (isSigned) { - assert.strictEqual(functions.identity_char(-128), -128); - assert.strictEqual(functions.identity_char(127), 127); - assert.throws(() => functions.identity_char(128), expect); - assert.throws(() => functions.identity_char(-129), expect); - } else { - assert.strictEqual(functions.identity_char(255), 255); - assert.strictEqual(functions.identity_char(0), 0); - assert.throws(() => functions.identity_char(256), expect); - assert.throws(() => functions.identity_char(-1), expect); - } - } finally { - lib.close(); - } -}); - -test('SB metadata is Symbol-keyed, attribute-hardened, and not leaked onto the wrapper', () => { - const rawLib = new ffiBinding.DynamicLibrary(libraryPath); - try { - const rawFn = rawGetFunctionUnpatched.call( - rawLib, 'add_i32', { result: 'i32', parameters: ['i32', 'i32'] }); - - for (const [name, sym] of [ - ['kSbSharedBuffer', kSbSharedBuffer], - ['kSbInvokeSlow', kSbInvokeSlow], - ['kSbParams', kSbParams], - ['kSbResult', kSbResult], - ]) { - assert.strictEqual(typeof sym, 'symbol', `${name} must be a Symbol`); - } - - // Numeric-only signature: kSbInvokeSlow absent; the rest present and hardened. - for (const [name, sym] of [ - ['kSbSharedBuffer', kSbSharedBuffer], - ['kSbParams', kSbParams], - ['kSbResult', kSbResult], - ]) { - const desc = Object.getOwnPropertyDescriptor(rawFn, sym); - assert.ok(desc !== undefined, `${name} missing on pure-numeric SB function`); - assert.strictEqual(desc.enumerable, false); - assert.strictEqual(desc.configurable, false); - assert.strictEqual(desc.writable, false); - } - assert.strictEqual( - Object.getOwnPropertyDescriptor(rawFn, kSbInvokeSlow), undefined); - - // Pointer signature: kSbInvokeSlow must exist (and be hardened). - const rawPtrFn = rawGetFunctionUnpatched.call( - rawLib, 'identity_pointer', { result: 'pointer', parameters: ['pointer'] }); - const slowDesc = Object.getOwnPropertyDescriptor(rawPtrFn, kSbInvokeSlow); - assert.ok(slowDesc !== undefined); - assert.strictEqual(slowDesc.enumerable, false); - assert.strictEqual(slowDesc.configurable, false); - assert.strictEqual(slowDesc.writable, false); - - assert.deepStrictEqual(Object.keys(rawFn), ['pointer']); - const ownSyms = Object.getOwnPropertySymbols(rawFn); - assert.ok(ownSyms.includes(kSbSharedBuffer)); - assert.ok(ownSyms.includes(kSbParams)); - assert.ok(ownSyms.includes(kSbResult)); - - // Internals must not be forwarded by `inheritMetadata`. - const { lib, functions } = ffi.dlopen(libraryPath, { - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - }); - try { - assert.strictEqual(functions.add_i32[kSbSharedBuffer], undefined); - assert.strictEqual(functions.add_i32[kSbInvokeSlow], undefined); - assert.strictEqual(functions.add_i32[kSbParams], undefined); - assert.strictEqual(functions.add_i32[kSbResult], undefined); - } finally { - lib.close(); - } - } finally { - rawLib.close(); - } -}); - -test('pointer fast-path range check: [0, 2^64 - 1]', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - identity_pointer: { result: 'pointer', parameters: ['pointer'] }, - }); - try { - assert.strictEqual(functions.identity_pointer(0n), 0n); - assert.strictEqual(functions.identity_pointer((1n << 64n) - 1n), (1n << 64n) - 1n); - - const expect = { code: 'ERR_INVALID_ARG_VALUE' }; - assert.throws(() => functions.identity_pointer(-1n), expect); - assert.throws(() => functions.identity_pointer(1n << 64n), expect); - } finally { - lib.close(); - } -}); - -test('self-recursive reentrancy: a single function\'s ArrayBuffer survives a nested call', () => { - // Stricter invariant than the two-symbol case: `InvokeFunctionSB` must - // copy args out of the ArrayBuffer to stack before `ffi_call` so a recursive - // call can reuse the same buffer without clobbering the outer frame. - const { lib, functions } = ffi.dlopen(libraryPath, { - call_binary_int_callback: { - result: 'i32', - parameters: ['function', 'i32', 'i32'], - }, - }); - - try { - let depth = 0; - const callback = lib.registerCallback( - { result: 'i32', parameters: ['i32', 'i32'] }, - common.mustCall((a, b) => { - depth++; - if (depth === 1) { - const inner = functions.call_binary_int_callback(callback, 100, 200); - assert.strictEqual(inner, 300); - } - return a + b; - }, 2), - ); - - try { - assert.strictEqual(functions.call_binary_int_callback(callback, 10, 20), 30); - } finally { - lib.unregisterCallback(callback); - } - } finally { - lib.close(); - } -}); - -test('void-return 0-arg wrapper branch', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - reset_counter: { result: 'void', parameters: [] }, - increment_counter: { result: 'void', parameters: [] }, - get_counter: { result: 'i32', parameters: [] }, - }); - try { - assert.strictEqual(functions.reset_counter(), undefined); - assert.strictEqual(functions.get_counter(), 0); - - functions.increment_counter(); - functions.increment_counter(); - functions.increment_counter(); - assert.strictEqual(functions.get_counter(), 3); - - assert.strictEqual(functions.reset_counter(), undefined); - assert.strictEqual(functions.get_counter(), 0); - } finally { - lib.close(); - } -}); - -test('void-return wrapper at every specialized arity observes side effects', () => { - // The arity ladder has a separate void-return closure for each arity. - // A wiring bug in a mid-arity void specialization would not be caught - // by the 0-arg void test above, so exercise the side effects directly - // at every arity the ladder specializes (1..6) plus the 7+ rest-params - // fallback. - const { lib, functions } = ffi.dlopen(libraryPath, { - store_i32: { result: 'void', parameters: ['i32'] }, - store_sum_2_i32: { result: 'void', parameters: ['i32', 'i32'] }, - store_sum_3_i32: { result: 'void', parameters: ['i32', 'i32', 'i32'] }, - store_sum_4_i32: { - result: 'void', - parameters: ['i32', 'i32', 'i32', 'i32'], - }, - store_sum_5_i32: { - result: 'void', - parameters: ['i32', 'i32', 'i32', 'i32', 'i32'], - }, - store_sum_6_i32: { - result: 'void', - parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32'], - }, - store_sum_8_i32: { - result: 'void', - parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32'], - }, - get_scratch: { result: 'i32', parameters: [] }, - }); - try { - // Powers-of-two summands detect a dropped or duplicated slot at each - // arity. - assert.strictEqual(functions.store_i32(7), undefined); - assert.strictEqual(functions.get_scratch(), 7); - - assert.strictEqual(functions.store_sum_2_i32(10, 32), undefined); - assert.strictEqual(functions.get_scratch(), 42); - - assert.strictEqual(functions.store_sum_3_i32(1, 2, 4), undefined); - assert.strictEqual(functions.get_scratch(), 7); - - assert.strictEqual(functions.store_sum_4_i32(1, 2, 4, 8), undefined); - assert.strictEqual(functions.get_scratch(), 15); - - assert.strictEqual(functions.store_sum_5_i32(1, 2, 4, 8, 16), undefined); - assert.strictEqual(functions.get_scratch(), 31); - - assert.strictEqual( - functions.store_sum_6_i32(1, 2, 4, 8, 16, 32), undefined); - assert.strictEqual(functions.get_scratch(), 63); - - // 7+ args takes the generic rest-params void branch rather than a - // per-arity specialization. - assert.strictEqual( - functions.store_sum_8_i32(1, 2, 4, 8, 16, 32, 64, 128), undefined); - assert.strictEqual(functions.get_scratch(), 255); - - // Validation still runs on every void-return branch, including the - // rest-params fallback. - assert.throws( - () => functions.store_i32(1.5), - { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws( - () => functions.store_sum_2_i32(1.5, 2), - { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws( - () => functions.store_sum_3_i32(1, 1.5, 3), - { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws( - () => functions.store_sum_4_i32(1, 2, 1.5, 4), - { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws( - () => functions.store_sum_5_i32(1, 2, 3, 1.5, 5), - { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws( - () => functions.store_sum_6_i32(1, 2, 3, 4, 5), - { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws( - () => functions.store_sum_8_i32(1, 2, 3, 4, 5, 6, 7, 1.5), - { code: 'ERR_INVALID_ARG_VALUE' }); - - // Wrong arity hits the `throwFFIArgCountError` branch inside each - // specialization (1..6 and the 7+ rest-params fallback). - for (const [name, expected, badArgs] of [ - ['store_i32', 1, []], - ['store_sum_2_i32', 2, [1]], - ['store_sum_3_i32', 3, [1, 2]], - ['store_sum_4_i32', 4, [1, 2, 3]], - ['store_sum_5_i32', 5, [1, 2, 3, 4]], - ['store_sum_6_i32', 6, [1, 2, 3, 4, 5]], - ['store_sum_8_i32', 8, [1, 2, 3, 4, 5, 6, 7]], - ]) { - assert.throws( - () => functions[name](...badArgs), - { - code: 'ERR_INVALID_ARG_VALUE', - message: new RegExp(`expected ${expected}, got ${badArgs.length}`), - }); - } - } finally { - lib.close(); - } -}); - -test('value-return wrapper arity mismatch hits every specialized branch', () => { - // `sum_7_i32` already exercises the 7+ rest-params branch elsewhere; - // this test targets the per-arity `throwFFIArgCountError` call in the - // value-return closures for arities 1..6 so each specialization's - // argument-count guard runs at least once. - const { lib, functions } = ffi.dlopen(libraryPath, { - logical_not: { result: 'i32', parameters: ['i32'] }, - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - sum_3_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32'] }, - sum_4_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32'] }, - sum_five_i32: { - result: 'i32', - parameters: ['i32', 'i32', 'i32', 'i32', 'i32'], - }, - sum_6_i32: { - result: 'i32', - parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32'], - }, - }); - try { - for (const [name, expected, badArgs] of [ - ['logical_not', 1, []], - ['add_i32', 2, [1]], - ['sum_3_i32', 3, [1, 2]], - ['sum_4_i32', 4, [1, 2, 3]], - ['sum_five_i32', 5, [1, 2, 3, 4]], - ['sum_6_i32', 6, [1, 2, 3, 4, 5]], - ]) { - assert.throws( - () => functions[name](...badArgs), - { - code: 'ERR_INVALID_ARG_VALUE', - message: new RegExp(`expected ${expected}, got ${badArgs.length}`), - }); - } - - // Sanity-check that a correct call still returns a value at each - // arity — a bug that swallowed the return on the value-return path - // would be caught here. - assert.strictEqual(functions.logical_not(0), 1); - assert.strictEqual(functions.add_i32(1, 2), 3); - assert.strictEqual(functions.sum_3_i32(1, 2, 4), 7); - assert.strictEqual(functions.sum_4_i32(1, 2, 4, 8), 15); - assert.strictEqual(functions.sum_five_i32(1, 2, 4, 8, 16), 31); - assert.strictEqual(functions.sum_6_i32(1, 2, 4, 8, 16, 32), 63); - } finally { - lib.close(); - } -}); - -test('pointer-dispatch wrapper rejects wrong-arity calls', () => { - // Pointer signatures share a single rest-params wrapper rather than the - // per-arity ladder, but it still has its own `throwFFIArgCountError` - // branch that needs to be exercised. - const { lib, functions } = ffi.dlopen(libraryPath, { - identity_pointer: { result: 'pointer', parameters: ['pointer'] }, - }); - try { - assert.throws( - () => functions.identity_pointer(), - { - code: 'ERR_INVALID_ARG_VALUE', - message: /expected 1, got 0/, - }); - assert.throws( - () => functions.identity_pointer(0n, 0n), - { - code: 'ERR_INVALID_ARG_VALUE', - message: /expected 1, got 2/, - }); - } finally { - lib.close(); - } -}); - -test('mid-arity wrappers (1, 3, 4, 5)', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - logical_not: { result: 'i32', parameters: ['i32'] }, - sum_3_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32'] }, - sum_4_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32'] }, - sum_five_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32'] }, - }); - try { - assert.strictEqual(functions.logical_not(0), 1); - assert.strictEqual(functions.logical_not(42), 0); - - // Powers-of-two summands: a dropped or duplicated slot would change the total. - assert.strictEqual(functions.sum_3_i32(1, 2, 4), 7); - assert.strictEqual(functions.sum_4_i32(1, 2, 4, 8), 15); - assert.strictEqual(functions.sum_five_i32(1, 2, 4, 8, 16), 31); - } finally { - lib.close(); - } -}); - -test('float specials: NaN, ±Infinity, -0 round-trip bit-exact', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - add_f64: { result: 'f64', parameters: ['f64', 'f64'] }, - multiply_f64: { result: 'f64', parameters: ['f64', 'f64'] }, - }); - try { - assert.ok(Number.isNaN(functions.add_f64(NaN, 1.0))); - assert.strictEqual(functions.add_f64(Infinity, 1.0), Infinity); - assert.strictEqual(functions.add_f64(-Infinity, 1.0), -Infinity); - assert.ok(Object.is(functions.multiply_f64(-0, 1.0), -0)); - } finally { - lib.close(); - } -}); - -test('arity-7+ branch still runs per-arg validation', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - sum_7_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32'] }, - }); - try { - assert.throws( - () => functions.sum_7_i32(1, 2, 3, 1.5, 5, 6, 7), - { code: 'ERR_INVALID_ARG_VALUE' }, - ); - } finally { - lib.close(); - } -}); - -test('mixed-kind signature (i32, f32, f64, u32) dispatches the right writer per slot', () => { - // Four distinct `sbTypeInfo.kind` values (int, float, float, int) — a - // wiring bug that reused one writer across slots would surface here. - const { lib, functions } = ffi.dlopen(libraryPath, { - mixed_operation: { parameters: ['i32', 'f32', 'f64', 'u32'], result: 'f64' }, - }); - - try { - assert.strictEqual(functions.mixed_operation(10, 2.5, 3.5, 4), 20); - assert.strictEqual(functions.mixed_operation(-1, 0.25, 0.75, 0), 0); - - const expect = { code: 'ERR_INVALID_ARG_VALUE' }; - // -1 on u32 slot: distinguishes u32 writer from i32 (i32 accepts -1). - assert.throws(() => functions.mixed_operation(0, 0.0, 0.0, -1), expect); - // 2^31 on i32 slot: distinguishes i32 writer from u32 (u32 accepts it). - assert.throws(() => functions.mixed_operation(2147483648, 0.0, 0.0, 0), expect); - // Float slots reject BigInt / string (the int/float writers both gate on `typeof`). - assert.throws(() => functions.mixed_operation(0, 1n, 0.0, 0), expect); - assert.throws(() => functions.mixed_operation(0, 0.0, 'x', 0), expect); - } finally { - lib.close(); - } -}); - -test('lib.getFunctions() with no arguments wraps every cached function', () => { - // Regression: the no-args branch previously returned raw native functions - // whose shared buffer was uninitialized, producing garbage numeric results. - // Mix SB-eligible signatures with one that is not (`string_length` takes a - // string, which bypasses the fast path) so the no-args branch has to walk - // the early-return path in `wrapWithSharedBuffer` alongside the wrapped - // branch. - const { lib } = ffi.dlopen(libraryPath, { - add_i32: { result: 'i32', parameters: ['i32', 'i32'] }, - add_f64: { result: 'f64', parameters: ['f64', 'f64'] }, - mixed_operation: { parameters: ['i32', 'f32', 'f64', 'u32'], result: 'f64' }, - identity_pointer: { result: 'pointer', parameters: ['pointer'] }, - string_length: { result: 'u64', parameters: ['string'] }, - }); - - try { - const all = lib.getFunctions(); - assert.strictEqual(Object.getPrototypeOf(all), null); - - // SB-eligible entries go through the shared-buffer wrapper. - assert.strictEqual(all.add_i32(20, 22), 42); - assert.strictEqual(all.add_f64(1.5, 2.5), 4.0); - assert.strictEqual(all.identity_pointer(0x42n), 0x42n); - - // Non-eligible entry returns its raw native wrapper unchanged; it still - // has to be callable from the object returned by `getFunctions()`. - assert.strictEqual(all.string_length('hello'), 5n); - - assert.deepStrictEqual( - Object.keys(all).sort(), - ['add_f64', 'add_i32', 'identity_pointer', - 'mixed_operation', 'string_length']); - - assert.throws(() => all.add_i32(1), { code: 'ERR_INVALID_ARG_VALUE' }); - assert.throws(() => all.add_i32(1.5, 0), { code: 'ERR_INVALID_ARG_VALUE' }); - - assert.strictEqual(typeof all.add_i32.pointer, 'bigint'); - assert.ok(all.add_i32.pointer !== 0n); - - // The wrapper object is no longer frozen; nothing in the SB design - // requires it. - assert.ok(!Object.isFrozen(all)); - } finally { - lib.close(); - } -}); - -test('mixed pointer + numeric signature uses the pointer-dispatch wrapper', () => { - const { lib, functions } = ffi.dlopen(libraryPath, { - call_int_callback: { result: 'i32', parameters: ['pointer', 'i32'] }, - }); - - try { - const cb = lib.registerCallback( - { result: 'i32', parameters: ['i32'] }, - (x) => x * 2, - ); - try { - assert.strictEqual(functions.call_int_callback(cb, 7), 14); - // Negative i32 must land in the numeric writer (not the pointer writer, - // which would reject a negative BigInt). - assert.strictEqual(functions.call_int_callback(cb, -5), -10); - } finally { - lib.unregisterCallback(cb); - } - } finally { - lib.close(); - } -}); From c308d4d9955c4de016b6817e56ac6fcababb38b0 Mon Sep 17 00:00:00 2001 From: Bryan English Date: Wed, 6 May 2026 12:03:38 -0400 Subject: [PATCH 2/4] fixup! ffi: add V8 fast-call path --- lib/internal/ffi-fastcall.js | 173 ++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 75 deletions(-) diff --git a/lib/internal/ffi-fastcall.js b/lib/internal/ffi-fastcall.js index 3c0f435f0855d9..6af9dd719cbc9e 100644 --- a/lib/internal/ffi-fastcall.js +++ b/lib/internal/ffi-fastcall.js @@ -90,7 +90,48 @@ function throwArg(msg) { throw err; } +// Per-kind specialized validators. Picked once at wrapper-build time, so +// the hot path doesn't re-dispatch on info.kind per call. Each returns the +// arg unchanged on success. +// +// i32/u32 use the bitwise-coerce-and-compare idiom. `(a | 0)` first ToInt32- +// coerces a (NaN/Infinity → 0, non-numbers → 0, fractional → trunc, out-of- +// range → wrap mod 2^32 with sign extension); the `=== a` then rejects every +// case where the coercion changed the value. Same logic for `>>> 0` against +// uint32. This collapses what would otherwise be typeof + NumberIsInteger + +// two range comparisons into one branch. +function validateI32(arg, idx) { + if ((arg | 0) !== arg) throwArg(`Argument ${idx} must be an int32`); + return arg; +} +function validateU32(arg, idx) { + if ((arg >>> 0) !== arg) throwArg(`Argument ${idx} must be a uint32`); + return arg; +} +function validateI64(arg, idx) { + if (typeof arg !== 'bigint' || arg < I64_MIN || arg > I64_MAX) { + throwArg(`Argument ${idx} must be an int64`); + } + return arg; +} +function validateU64(arg, idx) { + if (typeof arg !== 'bigint' || arg < 0n || arg > U64_MAX) { + throwArg(`Argument ${idx} must be a uint64`); + } + return arg; +} +function validateF32(arg, idx) { + if (typeof arg !== 'number') throwArg(`Argument ${idx} must be a float`); + return arg; +} +function validateF64(arg, idx) { + if (typeof arg !== 'number') throwArg(`Argument ${idx} must be a double`); + return arg; +} +// Generic validator for narrow ints (i8/u8/i16/u16/bool/char) — used by the +// full wrapper, never by the strict-numeric one. The narrower-int range is +// per-type so it stays generic. function validateAndCoerce(info, arg, idx) { const k = info.kind; if (k === 'int') { @@ -100,42 +141,30 @@ function validateAndCoerce(info, arg, idx) { } return arg; } - if (k === 'int32') { - if (typeof arg !== 'number' || !NumberIsInteger(arg) || - arg < -2147483648 || arg > 2147483647) { - throwArg(`Argument ${idx} must be ${info.label}`); - } - return arg; - } - if (k === 'uint32') { - if (typeof arg !== 'number' || !NumberIsInteger(arg) || - arg < 0 || arg > 4294967295) { - throwArg(`Argument ${idx} must be ${info.label}`); - } - return arg; - } - if (k === 'i64') { - if (typeof arg !== 'bigint' || arg < I64_MIN || arg > I64_MAX) { - throwArg(`Argument ${idx} must be ${info.label}`); - } - return arg; - } - if (k === 'u64') { - if (typeof arg !== 'bigint' || arg < 0n || arg > U64_MAX) { - throwArg(`Argument ${idx} must be ${info.label}`); - } - return arg; - } - if (k === 'float' || k === 'double') { - if (typeof arg !== 'number') { - throwArg(`Argument ${idx} must be ${info.label}`); - } - return arg; - } + if (k === 'int32') return validateI32(arg, idx); + if (k === 'uint32') return validateU32(arg, idx); + if (k === 'i64') return validateI64(arg, idx); + if (k === 'u64') return validateU64(arg, idx); + if (k === 'float') return validateF32(arg, idx); + if (k === 'double') return validateF64(arg, idx); throw new ERR_INTERNAL_ASSERTION( `FFI: unexpected kind="${k}" in validateAndCoerce`); } +function validatorFor(info) { + switch (info.kind) { + case 'int32': return validateI32; + case 'uint32': return validateU32; + case 'i64': return validateI64; + case 'u64': return validateU64; + case 'float': return validateF32; + case 'double': return validateF64; + default: + throw new ERR_INTERNAL_ASSERTION( + `FFI: validatorFor called with non-strict-numeric kind "${info.kind}"`); + } +} + const SLOW_PATH = Symbol('slow-path'); function coercePointer(arg, idx) { if (typeof arg === 'bigint') { @@ -148,15 +177,17 @@ function coercePointer(arg, idx) { return SLOW_PATH; } -// Specialized wrapper for strict-numeric signatures (no pointers). Validates -// each arg the same way the full wrapper does — V8's fast-call coercion -// silently truncates non-integers and out-of-range values when the call site -// is fully optimized (e.g. CheckedNumberAsWord32 wraps mod 2^32; for i64/u64, -// CheckedBigIntTruncatingWord64 wraps to low 64 bits). Skipping the JS check -// would make the same FFI binding throw cold and silently corrupt hot — so -// the per-arg validation stays. The win over the full wrapper is the dropped -// per-arg `if (isPointer)` branch and the smaller closure (no slowInvoke, -// no isPointer table). +// Specialized wrapper for strict-numeric signatures (no pointers). Each +// argument is validated by a per-kind validator picked once at build time +// — V8 inline-caches the call to the specific function, the kind branch +// is gone from the hot path, and i32/u32 use the `(a | 0) === a` and +// `(a >>> 0) === a` idioms (see comment on validateI32 above). +// +// The wrapper still validates every arg because V8's fast-call coercion +// silently truncates non-integers and out-of-range values when the call +// site is fully optimized (CheckedNumberAsWord32 wraps mod 2^32, Checked- +// BigIntTruncatingWord64 wraps to low 64 bits). Without the JS check the +// same FFI binding would throw cold and silently corrupt hot. function buildStrictNumericWrapper(rawFn, alive, infos, nargs) { switch (nargs) { case 0: @@ -168,86 +199,77 @@ function buildStrictNumericWrapper(rawFn, alive, infos, nargs) { return rawFn(); }; case 1: { - const i0 = infos[0]; + const v0 = validatorFor(infos[0]); return function(a0) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (arguments.length !== 1) { throwArg(`Invalid argument count: expected 1, got ${arguments.length}`); } - return rawFn(validateAndCoerce(i0, a0, 0)); + return rawFn(v0(a0, 0)); }; } case 2: { - const i0 = infos[0]; const i1 = infos[1]; + const v0 = validatorFor(infos[0]); const v1 = validatorFor(infos[1]); return function(a0, a1) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (arguments.length !== 2) { throwArg(`Invalid argument count: expected 2, got ${arguments.length}`); } - const v0 = validateAndCoerce(i0, a0, 0); - const v1 = validateAndCoerce(i1, a1, 1); - return rawFn(v0, v1); + return rawFn(v0(a0, 0), v1(a1, 1)); }; } case 3: { - const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; + const v0 = validatorFor(infos[0]); const v1 = validatorFor(infos[1]); + const v2 = validatorFor(infos[2]); return function(a0, a1, a2) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (arguments.length !== 3) { throwArg(`Invalid argument count: expected 3, got ${arguments.length}`); } - const v0 = validateAndCoerce(i0, a0, 0); - const v1 = validateAndCoerce(i1, a1, 1); - const v2 = validateAndCoerce(i2, a2, 2); - return rawFn(v0, v1, v2); + return rawFn(v0(a0, 0), v1(a1, 1), v2(a2, 2)); }; } case 4: { - const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; + const v0 = validatorFor(infos[0]); const v1 = validatorFor(infos[1]); + const v2 = validatorFor(infos[2]); const v3 = validatorFor(infos[3]); return function(a0, a1, a2, a3) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (arguments.length !== 4) { throwArg(`Invalid argument count: expected 4, got ${arguments.length}`); } - const v0 = validateAndCoerce(i0, a0, 0); - const v1 = validateAndCoerce(i1, a1, 1); - const v2 = validateAndCoerce(i2, a2, 2); - const v3 = validateAndCoerce(i3, a3, 3); - return rawFn(v0, v1, v2, v3); + return rawFn(v0(a0, 0), v1(a1, 1), v2(a2, 2), v3(a3, 3)); }; } case 5: { - const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; const i4 = infos[4]; + const v0 = validatorFor(infos[0]); const v1 = validatorFor(infos[1]); + const v2 = validatorFor(infos[2]); const v3 = validatorFor(infos[3]); + const v4 = validatorFor(infos[4]); return function(a0, a1, a2, a3, a4) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (arguments.length !== 5) { throwArg(`Invalid argument count: expected 5, got ${arguments.length}`); } - const v0 = validateAndCoerce(i0, a0, 0); - const v1 = validateAndCoerce(i1, a1, 1); - const v2 = validateAndCoerce(i2, a2, 2); - const v3 = validateAndCoerce(i3, a3, 3); - const v4 = validateAndCoerce(i4, a4, 4); - return rawFn(v0, v1, v2, v3, v4); + return rawFn(v0(a0, 0), v1(a1, 1), v2(a2, 2), v3(a3, 3), v4(a4, 4)); }; } case 6: { - const i0 = infos[0]; const i1 = infos[1]; const i2 = infos[2]; const i3 = infos[3]; const i4 = infos[4]; const i5 = infos[5]; + const v0 = validatorFor(infos[0]); const v1 = validatorFor(infos[1]); + const v2 = validatorFor(infos[2]); const v3 = validatorFor(infos[3]); + const v4 = validatorFor(infos[4]); const v5 = validatorFor(infos[5]); return function(a0, a1, a2, a3, a4, a5) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (arguments.length !== 6) { throwArg(`Invalid argument count: expected 6, got ${arguments.length}`); } - const v0 = validateAndCoerce(i0, a0, 0); - const v1 = validateAndCoerce(i1, a1, 1); - const v2 = validateAndCoerce(i2, a2, 2); - const v3 = validateAndCoerce(i3, a3, 3); - const v4 = validateAndCoerce(i4, a4, 4); - const v5 = validateAndCoerce(i5, a5, 5); - return rawFn(v0, v1, v2, v3, v4, v5); + return rawFn(v0(a0, 0), v1(a1, 1), v2(a2, 2), v3(a3, 3), + v4(a4, 4), v5(a5, 5)); }; } - default: + default: { + const validators = []; + for (let i = 0; i < nargs; i++) { + ArrayPrototypePush(validators, validatorFor(infos[i])); + } return function(...args) { if (alive[0] !== 0) throw new ERR_FFI_LIBRARY_CLOSED(); if (args.length !== nargs) { @@ -255,10 +277,11 @@ function buildStrictNumericWrapper(rawFn, alive, infos, nargs) { } const out = []; for (let i = 0; i < nargs; i++) { - ArrayPrototypePush(out, validateAndCoerce(infos[i], args[i], i)); + ArrayPrototypePush(out, validators[i](args[i], i)); } return ReflectApply(rawFn, undefined, out); }; + } } } From 56ffab4b406e08be5cb73e97ef3b5b36a4c8f14b Mon Sep 17 00:00:00 2001 From: Bryan English Date: Wed, 6 May 2026 13:02:04 -0400 Subject: [PATCH 3/4] deps: patch v8 fast-call to support no-receiver mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds CFunctionInfo::HasReceiver = kNo. When set, V8's TurboFan and Turboshaft fast-call lowering omits the JS receiver from the C call — the C function pointer is invoked with user args only, no receiver in the first parameter register. For Node's FFI fast-call path this means the receiver-strip JIT stub is no longer needed: dlsym'd target functions can be registered directly with V8 as the C address. Eliminates ~7 instructions per call (AArch64) plus V8's own receiver-into-arg0 setup. Yields +3-24% over the prior validators+stub path on FFI microbenchmarks (largest gains on many-args and pointer-bigint), and beats the pre-fix silent-truncation thin wrapper on every numeric and pointer benchmark while preserving strict validation. The change is gated by an enum on CFunctionInfo (default kYes, backward-compatible). Existing fast-call users (DOM bindings, V8 internals) are unaffected. Patches in deps/v8 cover the API header, the constructor, the GetFastApiCallTarget overload-matching, and the js-call-reducer input-layout setup; the simplified-lowering and turboshaft graph-builder loops already iterate from input 0 over ArgumentCount() inputs and pick up the new layout automatically. Signed-off-by: Bryan English --- deps/v8/include/v8-fast-api-calls.h | 19 ++- deps/v8/src/api/api.cc | 4 +- deps/v8/src/compiler/fast-api-calls.cc | 8 +- deps/v8/src/compiler/js-call-reducer.cc | 13 +- src/ffi/fastcall/cfunction_info.cc | 36 ++-- src/node_ffi.cc | 185 +++++++++------------ test/cctest/test_ffi_fastcall_cfunction.cc | 4 +- 7 files changed, 132 insertions(+), 137 deletions(-) diff --git a/deps/v8/include/v8-fast-api-calls.h b/deps/v8/include/v8-fast-api-calls.h index 58ec34fa050c91..537765c9d77b24 100644 --- a/deps/v8/include/v8-fast-api-calls.h +++ b/deps/v8/include/v8-fast-api-calls.h @@ -308,6 +308,19 @@ class V8_EXPORT CFunctionInfo { kBigInt = 1, // Use BigInts to represent 64 bit integers. }; + // Whether the C function takes a JS receiver as its first argument. + // Most fast-call C functions do (matching how V8 wires up FunctionTemplate + // callbacks). Embedders that want to register a plain C function pointer + // — e.g. an FFI dispatcher that has no use for the receiver — can set this + // to kNo. In that mode V8 omits the receiver from the C call: arg_info[0] + // is the first user argument, ArgumentCount() returns the user-arg count, + // and the JS receiver value is discarded by the lowering instead of being + // passed in the first parameter register. + enum class HasReceiver : uint8_t { + kYes = 0, + kNo = 1, + }; + // Construct a struct to hold a CFunction's type information. // |return_info| describes the function's return type. // |arg_info| is an array of |arg_count| CTypeInfos describing the @@ -315,7 +328,8 @@ class V8_EXPORT CFunctionInfo { // CTypeInfo::kCallbackOptionsType. CFunctionInfo(const CTypeInfo& return_info, unsigned int arg_count, const CTypeInfo* arg_info, - Int64Representation repr = Int64Representation::kNumber); + Int64Representation repr = Int64Representation::kNumber, + HasReceiver has_receiver = HasReceiver::kYes); const CTypeInfo& ReturnInfo() const { return return_info_; } @@ -327,6 +341,8 @@ class V8_EXPORT CFunctionInfo { Int64Representation GetInt64Representation() const { return repr_; } + bool HasReceiverArg() const { return has_receiver_ == HasReceiver::kYes; } + // |index| must be less than ArgumentCount(). // Note: if the last argument passed on construction of CFunctionInfo // has type CTypeInfo::kCallbackOptionsType, it is not included in @@ -342,6 +358,7 @@ class V8_EXPORT CFunctionInfo { private: const CTypeInfo return_info_; const Int64Representation repr_; + const HasReceiver has_receiver_; const unsigned int arg_count_; const CTypeInfo* arg_info_; }; diff --git a/deps/v8/src/api/api.cc b/deps/v8/src/api/api.cc index 9ef4e3b4a66006..60e67faddf5f25 100644 --- a/deps/v8/src/api/api.cc +++ b/deps/v8/src/api/api.cc @@ -11992,9 +11992,11 @@ CFunction::CFunction(const void* address, const CFunctionInfo* type_info) CFunctionInfo::CFunctionInfo(const CTypeInfo& return_info, unsigned int arg_count, const CTypeInfo* arg_info, - Int64Representation repr) + Int64Representation repr, + HasReceiver has_receiver) : return_info_(return_info), repr_(repr), + has_receiver_(has_receiver), arg_count_(arg_count), arg_info_(arg_info) { DCHECK(repr == Int64Representation::kNumber || diff --git a/deps/v8/src/compiler/fast-api-calls.cc b/deps/v8/src/compiler/fast-api-calls.cc index 0ffacd1078a96c..24bd5f27b7ea7a 100644 --- a/deps/v8/src/compiler/fast-api-calls.cc +++ b/deps/v8/src/compiler/fast-api-calls.cc @@ -385,10 +385,14 @@ FastApiCallFunction GetFastApiCallTarget( function_template_info.c_signatures(broker); const size_t overloads_count = signatures.size(); - // Only considers entries whose type list length matches arg_count. + // Only considers entries whose type list length matches arg_count. For + // signatures registered with HasReceiver=kNo, the C-side ArgumentCount + // already excludes the receiver, so we don't subtract it here. for (size_t i = 0; i < overloads_count; i++) { const CFunctionInfo* c_signature = signatures[i]; - const size_t len = c_signature->ArgumentCount() - kReceiver; + const size_t len = + c_signature->ArgumentCount() - + (c_signature->HasReceiverArg() ? kReceiver : 0); bool optimize_to_fast_call = (len == arg_count) && fast_api_call::CanOptimizeFastSignature(c_signature); diff --git a/deps/v8/src/compiler/js-call-reducer.cc b/deps/v8/src/compiler/js-call-reducer.cc index ececdd1d7e95b0..439951a75b7059 100644 --- a/deps/v8/src/compiler/js-call-reducer.cc +++ b/deps/v8/src/compiler/js-call-reducer.cc @@ -662,7 +662,9 @@ class FastApiCallReducerAssembler : public JSCallReducerAssembler { // arguments, so extract c_argument_count from the first function. const int c_argument_count = static_cast(c_function_.signature->ArgumentCount()); - CHECK_GE(c_argument_count, kReceiver); + if (c_function_.signature->HasReceiverArg()) { + CHECK_GE(c_argument_count, kReceiver); + } const int slow_arg_count = // Arguments for CallApiCallbackOptimizedXXX builtin including @@ -677,11 +679,16 @@ class FastApiCallReducerAssembler : public JSCallReducerAssembler { base::SmallVector inputs(value_input_count + kEffectAndControl); int cursor = 0; - inputs[cursor++] = n.receiver(); + const bool has_receiver_arg = + c_function_.signature->HasReceiverArg(); + if (has_receiver_arg) { + inputs[cursor++] = n.receiver(); + } // TODO(turbofan): Consider refactoring CFunctionInfo to distinguish // between receiver and arguments, simplifying this (and related) spots. - int js_args_count = c_argument_count - kReceiver; + int js_args_count = + c_argument_count - (has_receiver_arg ? kReceiver : 0); for (int i = 0; i < js_args_count; ++i) { if (i < n.ArgumentCount()) { inputs[cursor++] = n.Argument(i); diff --git a/src/ffi/fastcall/cfunction_info.cc b/src/ffi/fastcall/cfunction_info.cc index 33866c391e253a..f99d794f0e0361 100644 --- a/src/ffi/fastcall/cfunction_info.cc +++ b/src/ffi/fastcall/cfunction_info.cc @@ -91,30 +91,25 @@ CFunctionInfoBundle& CFunctionInfoBundle::operator=( } CFunctionInfoBundle BuildCFunctionInfo(const FFIFunction& fn) { - using T = v8::CTypeInfo::Type; static_assert(std::is_trivially_destructible_v, "CTypeInfo must be trivially destructible for placement-new " "array without explicit element destruction"); CFunctionInfoBundle b; - // V8 wants the receiver as arg[0]. Total CTypeInfo entries = args + 1. - // CTypeInfo has no default constructor, so we allocate raw storage and - // placement-new each element individually. - size_t n = fn.args.size() + 1; - void* raw = ::operator new[](n * sizeof(v8::CTypeInfo)); - b.arg_types = static_cast(raw); - // Placement-new the receiver slot. - new (&b.arg_types[0]) v8::CTypeInfo(T::kV8Value); - // Placement-new the arg slots with a placeholder; overwritten below. - for (size_t i = 1; i < n; ++i) { - new (&b.arg_types[i]) v8::CTypeInfo(T::kVoid); - } - - b.arg_classes.reserve(fn.args.size()); - for (size_t i = 0; i < fn.args.size(); ++i) { - auto m = MapArgType(fn.args[i]); - b.arg_types[i + 1] = v8::CTypeInfo(m.ctype); - b.arg_classes.push_back(m.cls); + // We register the C function with HasReceiver=kNo, so V8 does not pass a + // JS receiver in the first parameter register. The CTypeInfo[] holds only + // user-arg types — no receiver slot. CTypeInfo has no default constructor, + // so we allocate raw storage and placement-new each element. + size_t n = fn.args.size(); + if (n > 0) { + void* raw = ::operator new[](n * sizeof(v8::CTypeInfo)); + b.arg_types = static_cast(raw); + b.arg_classes.reserve(n); + for (size_t i = 0; i < n; ++i) { + auto m = MapArgType(fn.args[i]); + new (&b.arg_types[i]) v8::CTypeInfo(m.ctype); + b.arg_classes.push_back(m.cls); + } } auto rm = MapResultType(fn.return_type); @@ -125,7 +120,8 @@ CFunctionInfoBundle BuildCFunctionInfo(const FFIFunction& fn) { return_info, static_cast(n), b.arg_types, - v8::CFunctionInfo::Int64Representation::kBigInt); + v8::CFunctionInfo::Int64Representation::kBigInt, + v8::CFunctionInfo::HasReceiver::kNo); return b; } diff --git a/src/node_ffi.cc b/src/node_ffi.cc index 07cb3d69c47222..9935d532ad32e7 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -319,118 +319,85 @@ MaybeLocal DynamicLibrary::CreateFunction( if (fc_ok) { auto bundle = node::ffi::fastcall::BuildCFunctionInfo(*fn); { - auto stub = node::ffi::fastcall::EmitForwarder( - isolate, fn->ptr, bundle.arg_classes, bundle.result_class); - if (!stub.has_value()) { - // Warn at most once per process: a single FFI app may register hundreds - // of functions, and the failure cause (mprotect / SELinux / hardened - // runtime) is the same for every call. Repeated warnings would flood - // logs without adding signal. - static std::atomic warned{false}; - if (!warned.exchange(true, std::memory_order_acq_rel)) { - if (ProcessEmitWarning(env, - "FFI fast-call stub emission failed for an eligible " - "signature; falling back to libffi for this and possibly " - "future calls. This usually indicates a JIT memory or " - "permission issue (mprotect/SELinux/hardened runtime).") - .IsNothing()) { - return MaybeLocal(); - } + // Register the dlsym'd target function pointer directly with V8 as the + // C function. With CFunctionInfo built using HasReceiver=kNo, V8's fast- + // call lowering omits the JS receiver from the C call — so no receiver- + // strip stub is needed. The target's plain C signature matches the + // CFunctionInfo exactly. + v8::CFunction cfun(fn->ptr, bundle.info); + v8::Local tmpl = v8::FunctionTemplate::New( + isolate, + DynamicLibrary::InvokeFunction, + data, + v8::Local(), + static_cast(fn->args.size()), + v8::ConstructorBehavior::kThrow, + v8::SideEffectType::kHasSideEffect, + &cfun); + v8::Local fc_fn; + if (tmpl->GetFunction(context).ToLocal(&fc_fn)) { + bool metadata_ok = true; + + v8::Local alive_ab = AliveBuffer(isolate); + if (!fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_alive_symbol(), + alive_ab, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; } - // fall through to libffi path - } else { - v8::CFunction cfun(stub->entry, bundle.info); - v8::Local tmpl = v8::FunctionTemplate::New( - isolate, - DynamicLibrary::InvokeFunction, - data, - v8::Local(), - static_cast(fn->args.size()), - v8::ConstructorBehavior::kThrow, - v8::SideEffectType::kHasSideEffect, - &cfun); - v8::Local fc_fn; - if (tmpl->GetFunction(context).ToLocal(&fc_fn)) { - // Build all V8 values before touching `info` fields, so that on - // any failure we can free the stub and return cleanly without - // leaking JIT memory. - bool metadata_ok = true; - - // Alive ArrayBuffer. - v8::Local alive_ab = AliveBuffer(isolate); - if (!fc_fn->DefineOwnProperty( - context, env->ffi_fastcall_alive_symbol(), - alive_ab, internal_attrs) - .FromMaybe(false)) { - metadata_ok = false; - } - - // Build the slow-invoke v8::Function for the wrapper's pointer - // fallback path. - Local slow_fn; - if (metadata_ok && - !Function::New(context, DynamicLibrary::InvokeFunction, data) - .ToLocal(&slow_fn)) { - metadata_ok = false; - } - if (metadata_ok && - !fc_fn->DefineOwnProperty( - context, env->ffi_fastcall_invoke_slow_symbol(), - slow_fn, internal_attrs) - .FromMaybe(false)) { - metadata_ok = false; - } - - // Attach params and result type names. - Local params_arr; - if (metadata_ok && - !ToV8Value(context, fn->arg_type_names, isolate) - .ToLocal(¶ms_arr)) { - metadata_ok = false; - } - if (metadata_ok && - !fc_fn->DefineOwnProperty( - context, env->ffi_fastcall_params_symbol(), - params_arr, internal_attrs) - .FromMaybe(false)) { - metadata_ok = false; - } - Local result_str; - if (metadata_ok && - !ToV8Value(context, fn->return_type_name, isolate) - .ToLocal(&result_str)) { - metadata_ok = false; - } - if (metadata_ok && - !fc_fn->DefineOwnProperty( - context, env->ffi_fastcall_result_symbol(), - result_str, internal_attrs) - .FromMaybe(false)) { - metadata_ok = false; - } - - if (!metadata_ok) { - // Free the stub to avoid a JIT memory leak. - node::ffi::fastcall::JitMemory::Get(isolate) - ->Free(*stub); - return MaybeLocal(); - } - - // All metadata attached successfully. Populate fast-call state. - info->fast = std::make_unique( - isolate, slow_fn, stub->entry, stub->alloc_size, - std::make_unique( - std::move(bundle))); - ret = fc_fn; - } else { - // Template-to-function failed (V8 has a pending exception). Free the - // stub since we won't use it; mirror the metadata-fail path and - // propagate the empty MaybeLocal so the caller surfaces the V8 - // exception. - node::ffi::fastcall::JitMemory::Get(isolate) - ->Free(*stub); + + Local slow_fn; + if (metadata_ok && + !Function::New(context, DynamicLibrary::InvokeFunction, data) + .ToLocal(&slow_fn)) { + metadata_ok = false; + } + if (metadata_ok && + !fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_invoke_slow_symbol(), + slow_fn, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + + Local params_arr; + if (metadata_ok && + !ToV8Value(context, fn->arg_type_names, isolate) + .ToLocal(¶ms_arr)) { + metadata_ok = false; + } + if (metadata_ok && + !fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_params_symbol(), + params_arr, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + Local result_str; + if (metadata_ok && + !ToV8Value(context, fn->return_type_name, isolate) + .ToLocal(&result_str)) { + metadata_ok = false; + } + if (metadata_ok && + !fc_fn->DefineOwnProperty( + context, env->ffi_fastcall_result_symbol(), + result_str, internal_attrs) + .FromMaybe(false)) { + metadata_ok = false; + } + + if (!metadata_ok) { return MaybeLocal(); } + + info->fast = std::make_unique( + isolate, slow_fn, /*stub=*/nullptr, /*alloc_size=*/0, + std::make_unique( + std::move(bundle))); + ret = fc_fn; + } else { + return MaybeLocal(); } } } diff --git a/test/cctest/test_ffi_fastcall_cfunction.cc b/test/cctest/test_ffi_fastcall_cfunction.cc index 720c4988b17b84..d5b2e063db7347 100644 --- a/test/cctest/test_ffi_fastcall_cfunction.cc +++ b/test/cctest/test_ffi_fastcall_cfunction.cc @@ -33,7 +33,9 @@ TEST(FFIFastCallCFunction, NumericSignature) { {"i32", "i32"}); auto bundle = BuildCFunctionInfo(fn); // Receiver + 2 args = 3 CTypeInfo entries. - EXPECT_EQ(bundle.info->ArgumentCount(), 3u); + // No-receiver mode: ArgumentCount counts user args only (no leading + // v8::Value receiver slot). + EXPECT_EQ(bundle.info->ArgumentCount(), 2u); EXPECT_EQ(bundle.arg_classes.size(), 2u); EXPECT_EQ(bundle.arg_classes[0], ArgClass::kGP); EXPECT_EQ(bundle.arg_classes[1], ArgClass::kGP); From 1fd74c1d7209d6804d339b4bbe6868b79155148a Mon Sep 17 00:00:00 2001 From: Bryan English Date: Wed, 6 May 2026 13:25:30 -0400 Subject: [PATCH 4/4] ffi: remove receiver-strip stub emitter and JIT memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With v8 fast-call patched to support HasReceiver=kNo (previous commit), the dlsym'd target function pointer is registered directly with v8 as the C function address — no JIT'd receiver-strip trampoline is needed. This removes the entire stub-emitter and JIT-memory infrastructure, along with the platform-specific argument-cap logic (v8's own 8-arg fast-call cap takes over) and the boot-time self-test that verified JIT pages were executable. Deleted: src/ffi/fastcall/jit_memory.{h,cc}, stub_emitter.h, the four per-platform stub_emitter_*.cc files (aarch64, arm, x64_sysv, x64_win), test/cctest/test_ffi_fastcall_{emitter,jit}.cc, and the related node.gyp / config.gypi entries. The CFunctionInfoBundle drops its arg_classes and result_class members (they only existed to feed the stub emitters). IsFastCallEligible loses the per-platform GP/FP/Win64 caps and the AArch32 i64-arg rejection — v8 now handles those uniformly. FastCall- State drops stub_entry/stub_alloc_size; its destructor no longer needs to free JIT pages. Net change: ~2000 lines deleted across src/ and test/cctest/. Functional behavior is unchanged: all 15 FFI tests and the FFI cctests still pass. The benchmark gain over the prior shipping wrapper is +2-24% on AArch64 macOS (largest on many-args at +24% — saved register- shifts in the deleted stub). Signed-off-by: Bryan English --- node.gyp | 33 --- src/ffi/fastcall/cfunction_info.cc | 63 ++-- src/ffi/fastcall/cfunction_info.h | 16 +- src/ffi/fastcall/jit_memory.cc | 292 ------------------- src/ffi/fastcall/jit_memory.h | 87 ------ src/ffi/fastcall/stub_emitter.h | 49 ---- src/ffi/fastcall/stub_emitter_aarch64.cc | 95 ------ src/ffi/fastcall/stub_emitter_arm.cc | 71 ----- src/ffi/fastcall/stub_emitter_x64_sysv.cc | 87 ------ src/ffi/fastcall/stub_emitter_x64_win.cc | 80 ----- src/ffi/types.cc | 73 +---- src/node_ffi.cc | 58 +--- src/node_ffi.h | 32 +- test/cctest/test_ffi_fastcall_cfunction.cc | 29 +- test/cctest/test_ffi_fastcall_eligibility.cc | 118 +------- test/cctest/test_ffi_fastcall_emitter.cc | 233 --------------- test/cctest/test_ffi_fastcall_jit.cc | 159 ---------- 17 files changed, 55 insertions(+), 1520 deletions(-) delete mode 100644 src/ffi/fastcall/jit_memory.cc delete mode 100644 src/ffi/fastcall/jit_memory.h delete mode 100644 src/ffi/fastcall/stub_emitter.h delete mode 100644 src/ffi/fastcall/stub_emitter_aarch64.cc delete mode 100644 src/ffi/fastcall/stub_emitter_arm.cc delete mode 100644 src/ffi/fastcall/stub_emitter_x64_sysv.cc delete mode 100644 src/ffi/fastcall/stub_emitter_x64_win.cc delete mode 100644 test/cctest/test_ffi_fastcall_emitter.cc delete mode 100644 test/cctest/test_ffi_fastcall_jit.cc diff --git a/node.gyp b/node.gyp index 17bc32216be483..aa219f5c819bae 100644 --- a/node.gyp +++ b/node.gyp @@ -474,9 +474,6 @@ 'src/ffi/types.h', ], 'node_ffi_fastcall_sources': [ - 'src/ffi/fastcall/jit_memory.cc', - 'src/ffi/fastcall/jit_memory.h', - 'src/ffi/fastcall/stub_emitter.h', 'src/ffi/fastcall/cfunction_info.cc', 'src/ffi/fastcall/cfunction_info.h', ], @@ -1021,20 +1018,6 @@ [ 'node_use_ffi_fastcall=="true"', { 'defines': [ 'HAVE_FFI_FASTCALL=1' ], 'sources': [ '<@(node_ffi_fastcall_sources)' ], - 'conditions': [ - [ 'target_arch=="arm64"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_aarch64.cc' ], - }], - [ 'target_arch=="x64" and OS!="win"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_sysv.cc' ], - }], - [ 'target_arch=="x64" and OS=="win"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_win.cc' ], - }], - [ 'target_arch=="arm"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_arm.cc' ], - }], - ], }], ], }], @@ -1106,20 +1089,6 @@ [ 'node_use_ffi_fastcall=="true"', { 'defines': [ 'HAVE_FFI_FASTCALL=1' ], 'sources': [ '<@(node_ffi_fastcall_sources)' ], - 'conditions': [ - [ 'target_arch=="arm64"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_aarch64.cc' ], - }], - [ 'target_arch=="x64" and OS!="win"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_sysv.cc' ], - }], - [ 'target_arch=="x64" and OS=="win"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_x64_win.cc' ], - }], - [ 'target_arch=="arm"', { - 'sources': [ 'src/ffi/fastcall/stub_emitter_arm.cc' ], - }], - ], }], ], }], @@ -1460,8 +1429,6 @@ 'sources!': [ 'test/cctest/test_ffi_fastcall_cfunction.cc', 'test/cctest/test_ffi_fastcall_eligibility.cc', - 'test/cctest/test_ffi_fastcall_emitter.cc', - 'test/cctest/test_ffi_fastcall_jit.cc', ], }], ['v8_enable_inspector==1', { diff --git a/src/ffi/fastcall/cfunction_info.cc b/src/ffi/fastcall/cfunction_info.cc index f99d794f0e0361..e7844309028d67 100644 --- a/src/ffi/fastcall/cfunction_info.cc +++ b/src/ffi/fastcall/cfunction_info.cc @@ -9,50 +9,38 @@ namespace node::ffi::fastcall { namespace { -// Map an ffi_type to (CTypeInfo::Type, ArgClass) for argument positions. -struct ArgMapping { - v8::CTypeInfo::Type ctype; - ArgClass cls; -}; - -// Map an ffi_type to (CTypeInfo::Type, ResultClass) for the return position. -struct ResultMapping { - v8::CTypeInfo::Type ctype; - ResultClass cls; -}; - -ArgMapping MapArgType(ffi_type* t) { +v8::CTypeInfo::Type MapArgType(ffi_type* t) { using T = v8::CTypeInfo::Type; if (t == &ffi_type_sint8 || t == &ffi_type_sint16 || - t == &ffi_type_sint32) return {T::kInt32, ArgClass::kGP}; + t == &ffi_type_sint32) return T::kInt32; if (t == &ffi_type_uint8 || - t == &ffi_type_uint16) return {T::kInt32, ArgClass::kGP}; - if (t == &ffi_type_uint32) return {T::kUint32, ArgClass::kGP}; - if (t == &ffi_type_sint64) return {T::kInt64, ArgClass::kGP}; + t == &ffi_type_uint16) return T::kInt32; + if (t == &ffi_type_uint32) return T::kUint32; + if (t == &ffi_type_sint64) return T::kInt64; if (t == &ffi_type_uint64 || - t == &ffi_type_pointer) return {T::kUint64, ArgClass::kGP}; - if (t == &ffi_type_float) return {T::kFloat32, ArgClass::kFP}; - if (t == &ffi_type_double) return {T::kFloat64, ArgClass::kFP}; + t == &ffi_type_pointer) return T::kUint64; + if (t == &ffi_type_float) return T::kFloat32; + if (t == &ffi_type_double) return T::kFloat64; // Unreachable if eligibility was checked before calling BuildCFunctionInfo. UNREACHABLE("FFI fast-call: MapArgType called with type that passed " "eligibility but has no arg mapping"); } -ResultMapping MapResultType(ffi_type* t) { +v8::CTypeInfo::Type MapResultType(ffi_type* t) { using T = v8::CTypeInfo::Type; - if (t == &ffi_type_void) return {T::kVoid, ResultClass::kVoid}; + if (t == &ffi_type_void) return T::kVoid; if (t == &ffi_type_sint8 || t == &ffi_type_sint16 || - t == &ffi_type_sint32) return {T::kInt32, ResultClass::kGP}; + t == &ffi_type_sint32) return T::kInt32; if (t == &ffi_type_uint8 || - t == &ffi_type_uint16) return {T::kInt32, ResultClass::kGP}; - if (t == &ffi_type_uint32) return {T::kUint32, ResultClass::kGP}; - if (t == &ffi_type_sint64) return {T::kInt64, ResultClass::kGP}; + t == &ffi_type_uint16) return T::kInt32; + if (t == &ffi_type_uint32) return T::kUint32; + if (t == &ffi_type_sint64) return T::kInt64; if (t == &ffi_type_uint64 || - t == &ffi_type_pointer) return {T::kUint64, ResultClass::kGP}; - if (t == &ffi_type_float) return {T::kFloat32, ResultClass::kFP}; - if (t == &ffi_type_double) return {T::kFloat64, ResultClass::kFP}; + t == &ffi_type_pointer) return T::kUint64; + if (t == &ffi_type_float) return T::kFloat32; + if (t == &ffi_type_double) return T::kFloat64; UNREACHABLE("FFI fast-call: MapResultType called with type that passed " "eligibility but has no result mapping"); } @@ -66,12 +54,9 @@ CFunctionInfoBundle::~CFunctionInfoBundle() { CFunctionInfoBundle::CFunctionInfoBundle(CFunctionInfoBundle&& o) noexcept : info(o.info), - arg_types(o.arg_types), - arg_classes(std::move(o.arg_classes)), - result_class(o.result_class) { + arg_types(o.arg_types) { o.info = nullptr; o.arg_types = nullptr; - o.result_class = ResultClass::kVoid; } CFunctionInfoBundle& CFunctionInfoBundle::operator=( @@ -81,11 +66,8 @@ CFunctionInfoBundle& CFunctionInfoBundle::operator=( delete info; info = o.info; arg_types = o.arg_types; - arg_classes = std::move(o.arg_classes); - result_class = o.result_class; o.info = nullptr; o.arg_types = nullptr; - o.result_class = ResultClass::kVoid; } return *this; } @@ -104,17 +86,12 @@ CFunctionInfoBundle BuildCFunctionInfo(const FFIFunction& fn) { if (n > 0) { void* raw = ::operator new[](n * sizeof(v8::CTypeInfo)); b.arg_types = static_cast(raw); - b.arg_classes.reserve(n); for (size_t i = 0; i < n; ++i) { - auto m = MapArgType(fn.args[i]); - new (&b.arg_types[i]) v8::CTypeInfo(m.ctype); - b.arg_classes.push_back(m.cls); + new (&b.arg_types[i]) v8::CTypeInfo(MapArgType(fn.args[i])); } } - auto rm = MapResultType(fn.return_type); - b.result_class = rm.cls; - v8::CTypeInfo return_info(rm.ctype); + v8::CTypeInfo return_info(MapResultType(fn.return_type)); b.info = new v8::CFunctionInfo( return_info, diff --git a/src/ffi/fastcall/cfunction_info.h b/src/ffi/fastcall/cfunction_info.h index b2718fd4c867c2..e60bc4f2ca74b3 100644 --- a/src/ffi/fastcall/cfunction_info.h +++ b/src/ffi/fastcall/cfunction_info.h @@ -3,22 +3,16 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS && \ defined(HAVE_FFI_FASTCALL) -#include - -#include "ffi/fastcall/stub_emitter.h" #include "node_ffi.h" #include "v8-fast-api-calls.h" namespace node::ffi::fastcall { // Owns dynamically-allocated CFunctionInfo + CTypeInfo[] arrays for a -// single FFI function. Lifetime is tied to the FFIFunctionInfo. Free -// on weak-callback / dynamic library teardown. +// single FFI function. Lifetime is tied to the FFIFunctionInfo. struct CFunctionInfoBundle { v8::CFunctionInfo* info = nullptr; v8::CTypeInfo* arg_types = nullptr; - std::vector arg_classes; - ResultClass result_class = ResultClass::kVoid; CFunctionInfoBundle() = default; ~CFunctionInfoBundle(); @@ -28,10 +22,10 @@ struct CFunctionInfoBundle { CFunctionInfoBundle& operator=(CFunctionInfoBundle&&) noexcept; }; -// Build a CFunctionInfo + CTypeInfo[] for `fn`. The caller must already have -// verified `fn` via IsFastCallEligible before calling this. With -// -fno-exceptions, `new` aborts on OOM rather than throwing, so this function -// never fails — it returns directly instead of using optional. +// Build a CFunctionInfo + CTypeInfo[] for `fn` in HasReceiver=kNo mode. +// The caller must already have verified `fn` via IsFastCallEligible. With +// -fno-exceptions, `new` aborts on OOM rather than throwing, so this +// function never fails — it returns directly instead of using optional. CFunctionInfoBundle BuildCFunctionInfo(const FFIFunction& fn); } // namespace node::ffi::fastcall diff --git a/src/ffi/fastcall/jit_memory.cc b/src/ffi/fastcall/jit_memory.cc deleted file mode 100644 index 50aa8e5699dcdd..00000000000000 --- a/src/ffi/fastcall/jit_memory.cc +++ /dev/null @@ -1,292 +0,0 @@ -#include "ffi/fastcall/jit_memory.h" - -#ifdef HAVE_FFI_FASTCALL - -#include -#include - -#include "node_internals.h" -#include "util.h" - -#if defined(__POSIX__) -#include -#include -#endif - -#if defined(_WIN32) -#include -#endif - -namespace node::ffi::fastcall { - -namespace { - -size_t RoundUp(size_t v, size_t align) { - return (v + align - 1) & ~(align - 1); -} - -// Allocate a region that can later become executable. On macOS/arm64 the -// mapping must be created with MAP_JIT at allocation time; only then can -// mprotect(PROT_READ|PROT_EXEC) succeed after the Hardened Runtime check. -void* AllocJitPages(size_t size) { -#if defined(__APPLE__) - int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT; - void* p = mmap(nullptr, size, PROT_READ | PROT_WRITE, flags, -1, 0); - return (p == MAP_FAILED) ? nullptr : p; -#elif defined(__POSIX__) - void* p = mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - return (p == MAP_FAILED) ? nullptr : p; -#elif defined(_WIN32) - return VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, - PAGE_READWRITE); -#else - return nullptr; -#endif -} - -bool MakePageRX(void* addr, size_t size) { -#if defined(__POSIX__) - if (mprotect(addr, size, PROT_READ | PROT_EXEC) != 0) return false; - // Flush icache for the full range. - __builtin___clear_cache(static_cast(addr), - static_cast(addr) + size); - return true; -#elif defined(_WIN32) - DWORD old; - if (!VirtualProtect(addr, size, PAGE_EXECUTE_READ, &old)) return false; - FlushInstructionCache(GetCurrentProcess(), addr, size); - return true; -#else - return false; -#endif -} - -void FreeJitPages(void* addr, size_t size) { -#if defined(__POSIX__) - munmap(addr, size); -#elif defined(_WIN32) - VirtualFree(addr, 0, MEM_RELEASE); -#endif -} - -size_t SystemPageSize() { -#if defined(__POSIX__) - return static_cast(sysconf(_SC_PAGESIZE)); -#elif defined(_WIN32) - SYSTEM_INFO si; - GetSystemInfo(&si); - return static_cast(si.dwPageSize); -#else - return 4096; -#endif -} - -} // namespace - -JitMemory* JitMemory::Get(v8::Isolate* isolate) { - static JitMemory instance; - // The isolate parameter is retained for API symmetry; the implementation - // uses direct mmap/VirtualAlloc so no per-platform lookup is needed. - std::lock_guard lock(instance.mu_); - if (instance.page_size_ == 0) { - instance.page_size_ = SystemPageSize(); - } - return &instance; -} - -JitMemory::JitMemory() - : page_size_(0), - slot_align_(64), - live_bytes_(0) {} - -void* JitMemory::Allocate(size_t size, size_t* out_alloc_size) { - if (out_alloc_size) *out_alloc_size = 0; - if (size == 0) return nullptr; - - std::lock_guard lock(mu_); - if (page_size_ == 0) return nullptr; // Not initialised. - - size_t need = RoundUp(size, slot_align_); - - if (!pages_.empty()) { - auto& page = pages_.back(); - if (!page.executable && page.bump + need <= page.size) { - void* p = static_cast(page.base) + page.bump; - page.bump += need; - live_bytes_ += need; - if (out_alloc_size) *out_alloc_size = need; - return p; - } - } - - size_t page_bytes = RoundUp(std::max(page_size_, need), page_size_); - void* base = AllocJitPages(page_bytes); - if (base == nullptr) return nullptr; - - pages_.push_back(Page{base, page_bytes, need, false}); - live_bytes_ += need; - if (out_alloc_size) *out_alloc_size = need; - return base; -} - -bool JitMemory::MakeExecutable(void* ptr, size_t alloc_size) { - std::lock_guard lock(mu_); - if (page_size_ == 0) return false; - if (ptr == nullptr || alloc_size == 0) return false; - - uintptr_t addr = reinterpret_cast(ptr); - uintptr_t page_start = addr & ~(page_size_ - 1); - uintptr_t page_end = RoundUp(addr + alloc_size, page_size_); - - if (!MakePageRX(reinterpret_cast(page_start), - page_end - page_start)) { - return false; - } - - for (auto& page : pages_) { - uintptr_t base = reinterpret_cast(page.base); - if (addr >= base && addr < base + page.size) { - page.executable = true; - break; - } - } - return true; -} - -std::optional JitMemory::EmitStub(const uint8_t* code, - size_t size) { - if (code == nullptr || size == 0) return std::nullopt; - - std::lock_guard lock(mu_); - if (page_size_ == 0) return std::nullopt; - - size_t need = RoundUp(size, slot_align_); - - // Find or allocate a writable page. - void* dst = nullptr; - size_t alloc_size = need; - - if (!pages_.empty()) { - auto& page = pages_.back(); - if (!page.executable && page.bump + need <= page.size) { - dst = static_cast(page.base) + page.bump; - page.bump += need; - live_bytes_ += need; - } - } - - if (dst == nullptr) { - size_t page_bytes = RoundUp(std::max(page_size_, need), page_size_); - void* base = AllocJitPages(page_bytes); - if (base == nullptr) return std::nullopt; - pages_.push_back(Page{base, page_bytes, need, false}); - live_bytes_ += need; - dst = base; - } - - std::memcpy(dst, code, size); - - // Make executable — inline the mprotect/flush without re-locking. - uintptr_t addr = reinterpret_cast(dst); - uintptr_t page_start = addr & ~(page_size_ - 1); - uintptr_t page_end = RoundUp(addr + alloc_size, page_size_); - if (!MakePageRX(reinterpret_cast(page_start), - page_end - page_start)) { - // MakePageRX failed (e.g., mprotect rejected by hardened runtime). - // Roll back the live-bytes counter so the leak doesn't show up in - // MemoryTracker. We do NOT roll back page.bump — the failed slot is - // permanently wasted within the page, but the page itself stays in - // pages_ in RW state and the next EmitStub may successfully use the - // rest of it. The mapping is not returned to the OS. - live_bytes_ -= alloc_size; - return std::nullopt; - } - - for (auto& page : pages_) { - uintptr_t base = reinterpret_cast(page.base); - if (addr >= base && addr < base + page.size) { - page.executable = true; - break; - } - } - - return EmittedStub{dst, alloc_size}; -} - -void JitMemory::Free(void* ptr, size_t alloc_size) { - if (ptr == nullptr || alloc_size == 0) return; - std::lock_guard lock(mu_); - CHECK_GE(live_bytes_, alloc_size); - live_bytes_ -= alloc_size; -} - -size_t JitMemory::TotalLiveBytes() const { - std::lock_guard lock(mu_); - return live_bytes_; -} - -void JitMemory::ResetForTesting() { - std::lock_guard lock(mu_); - CHECK_EQ(live_bytes_, 0u); - for (auto& page : pages_) { - FreeJitPages(page.base, page.size); - } - pages_.clear(); -} - -bool JitMemory::SelfTest(v8::Isolate* isolate) { - static std::once_flag once; - static std::atomic result{-1}; - - std::call_once(once, [this, isolate]() { - Get(isolate); - - size_t alloc_size = 0; - void* p = Allocate(16, &alloc_size); - if (p == nullptr) { - result.store(0, std::memory_order_release); - return; - } - - uint8_t* code = static_cast(p); -#if defined(__x86_64__) || defined(_M_X64) - code[0] = 0xb8; code[1] = 0x2a; code[2] = 0x00; - code[3] = 0x00; code[4] = 0x00; - code[5] = 0xc3; -#elif defined(__aarch64__) || defined(_M_ARM64) - uint32_t* w = reinterpret_cast(code); - w[0] = 0x52800540; - w[1] = 0xd65f03c0; -#elif defined(__arm__) || defined(_M_ARM) - uint32_t* w = reinterpret_cast(code); - w[0] = 0xe3a0002a; - w[1] = 0xe12fff1e; -#else - Free(p, alloc_size); - result.store(1, std::memory_order_release); - return; -#endif - - if (!MakeExecutable(p, alloc_size)) { - Free(p, alloc_size); - result.store(0, std::memory_order_release); - return; - } - - using FnPtr = int (*)(); - int got = reinterpret_cast(p)(); - bool ok = (got == 42); - // The page stays executable, but the slot's bytes are no longer - // counted as live. Future allocations skip this page (it's marked - // executable), so the slot just sits unused — that's fine. - Free(p, alloc_size); - result.store(ok ? 1 : 0, std::memory_order_release); - }); - - return result.load(std::memory_order_acquire) == 1; -} - -} // namespace node::ffi::fastcall - -#endif // HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/jit_memory.h b/src/ffi/fastcall/jit_memory.h deleted file mode 100644 index 204ce268a8854f..00000000000000 --- a/src/ffi/fastcall/jit_memory.h +++ /dev/null @@ -1,87 +0,0 @@ -#pragma once - -#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS && \ - defined(HAVE_FFI_FASTCALL) - -#include -#include -#include -#include -#include - -namespace v8 { class Isolate; } - -namespace node::ffi::fastcall { - -// Handle returned from EmitStub (and from the lower-level Allocate path). -struct EmittedStub { - void* entry; // CFunction.address - size_t alloc_size; -}; - -class JitMemory { - public: - // Returns the singleton. On first call, initialises internal state (page - // size detection). The isolate parameter is retained for API symmetry with - // future callers; the implementation uses direct mmap/VirtualAlloc rather - // than the v8::PageAllocator interface to ensure MAP_JIT on Apple Silicon. - // Subsequent calls may pass nullptr. - static JitMemory* Get(v8::Isolate* isolate); - - // Allocate `size` bytes (rounded up to slot alignment, currently 64). - // Returns nullptr on failure (zero size, allocator unavailable, OOM). - // The returned pointer is in RW state — write the stub bytes, then call - // MakeExecutable. - void* Allocate(size_t size, size_t* out_alloc_size); - - // Transition the page containing `ptr` (extent `alloc_size`) from RW to - // RX, flushing icache as needed. Returns true on success. After this - // returns, the page is full as far as Allocate is concerned — subsequent - // allocations go to a fresh page. - bool MakeExecutable(void* ptr, size_t alloc_size); - - // Decrement the live-byte counter. Memory is not returned to the OS. - void Free(void* ptr, size_t alloc_size); - - // Convenience overload for callers that hold an EmittedStub value. - // Documents the pairing with EmitStub at the type level. - void Free(const EmittedStub& stub) { - Free(stub.entry, stub.alloc_size); - } - - // Total bytes currently allocated to live stubs. - size_t TotalLiveBytes() const; - - // All-in-one: allocate, write code, transition to RX, return handle. - // Holds the mutex across the entire operation; safe under multi-isolate - // concurrent emit. Returns std::nullopt on any failure (OOM, MakeExecutable). - std::optional EmitStub(const uint8_t* code, size_t size); - - // For testing only: free all pages back to the OS. Asserts no live bytes. - void ResetForTesting(); - - // One-time process-wide self-test: allocate, write platform-native - // "return 42", make RX, call. Cached. On failure, sets a flag so future - // Allocate calls return nullptr; callers fall back to libffi. - bool SelfTest(v8::Isolate* isolate); - - private: - JitMemory(); - - struct Page { - void* base; - size_t size; - size_t bump; - bool executable; - }; - - size_t page_size_; // 0 means not yet initialised - size_t slot_align_; - std::vector pages_; - size_t live_bytes_; - mutable std::mutex mu_; -}; - -} // namespace node::ffi::fastcall - -#endif // NODE_WANT_INTERNALS && HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/stub_emitter.h b/src/ffi/fastcall/stub_emitter.h deleted file mode 100644 index e4b636d72bfefb..00000000000000 --- a/src/ffi/fastcall/stub_emitter.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS && \ - defined(HAVE_FFI_FASTCALL) - -#include -#include - -#include "ffi/fastcall/jit_memory.h" - -namespace v8 { class Isolate; } - -namespace node::ffi::fastcall { - -// One slot per native arg of the target. Does not include kVoid — that is -// only meaningful as a return type (see ResultClass). -enum class ArgClass : uint8_t { - kGP, // any 32/64-bit integer or pointer (single GP slot) - kFP, // float or double (single FP slot) - kGPPair, // 64-bit integer on AArch32 (two GP slots; not used on - // AArch64/x86_64 — unsupported in v1) -}; - -// Classification of the return type. Separate from ArgClass to make it -// impossible to pass kVoid as an argument or kGPPair as a result. -enum class ResultClass : uint8_t { - kGP, // integer or pointer return - kFP, // float or double return - kVoid, // void return -}; - -// Build a stub for the target. The receiver is implicit and consumes -// one GP slot from V8 ahead of the listed args. Returns std::nullopt on -// failure (out of JIT memory, signature exceeds ABI cap, unsupported -// platform, kGPPair on non-AArch32). -// -// `target` is the native function pointer to tail-call. -// `args` lists the target's args in order. The receiver is implicit. -// `result` is informational on most ABIs; the stub does not synthesize -// a return value, only forwards. -std::optional EmitForwarder( - v8::Isolate* isolate, - void* target, - const std::vector& args, - ResultClass result); - -} // namespace node::ffi::fastcall - -#endif // NODE_WANT_INTERNALS && HAVE_FFI_FASTCALL diff --git a/src/ffi/fastcall/stub_emitter_aarch64.cc b/src/ffi/fastcall/stub_emitter_aarch64.cc deleted file mode 100644 index 4985f3b2869da4..00000000000000 --- a/src/ffi/fastcall/stub_emitter_aarch64.cc +++ /dev/null @@ -1,95 +0,0 @@ -#include "ffi/fastcall/stub_emitter.h" - -#if defined(HAVE_FFI_FASTCALL) && (defined(__aarch64__) || defined(_M_ARM64)) - -#include - -#include "ffi/fastcall/jit_memory.h" - -namespace node::ffi::fastcall { - -namespace { - -constexpr unsigned kMaxGPArgs = 7; // x0..x7 minus receiver - -// AArch64 instruction helpers. All instructions are 32 bits, little-endian. - -// MOV Xd, Xm (encoded as ORR Xd, XZR, Xm) -uint32_t MovReg(unsigned dst, unsigned src) { - return 0xAA0003E0u | (src << 16) | dst; -} - -// MOVZ Xd, #imm16, LSL #shift (shift in {0,16,32,48}) -uint32_t Movz(unsigned dst, uint16_t imm, unsigned shift) { - unsigned hw = shift / 16; - return 0xD2800000u | (hw << 21) | (uint32_t(imm) << 5) | dst; -} - -// MOVK Xd, #imm16, LSL #shift -uint32_t Movk(unsigned dst, uint16_t imm, unsigned shift) { - unsigned hw = shift / 16; - return 0xF2800000u | (hw << 21) | (uint32_t(imm) << 5) | dst; -} - -// BR Xn (unconditional branch to register; tail-call) -uint32_t BrReg(unsigned reg) { - return 0xD61F0000u | (reg << 5); -} - -} // namespace - -std::optional EmitForwarder( - v8::Isolate* isolate, - void* target, - const std::vector& args, - ResultClass /*result*/) { - unsigned gp = 0, fp = 0; - for (auto c : args) { - switch (c) { - case ArgClass::kGP: ++gp; break; - case ArgClass::kFP: ++fp; break; - case ArgClass::kGPPair: return std::nullopt; // unsupported on this ABI - } - } - if (gp > kMaxGPArgs) return std::nullopt; - if (fp > 8) return std::nullopt; - - uint32_t code[16] = {0}; - size_t off = 0; - - // Shift x1..xN -> x0..x(N-1). Forward order is safe: at iteration i - // we emit `MOV xi, x(i+1)`, writing only xi. At iteration i+1 we - // read x(i+2), which has not been written yet. Each register is - // written exactly once, and the read of register k always happens - // before any write to k. - for (unsigned i = 0; i < gp; ++i) { - code[off++] = MovReg(/*dst=*/i, /*src=*/i + 1); - } - - // Load target into x16 (intra-procedure-call scratch reg). Build the - // 64-bit address with MOVZ + up to 3 MOVKs. Always emit MOVZ for the - // bottom 16 bits even if zero; only emit higher MOVKs when their - // bits are non-zero, to keep stubs tight. - uint64_t addr = reinterpret_cast(target); - code[off++] = Movz(16, static_cast(addr & 0xFFFF), 0); - if (((addr >> 16) & 0xFFFF) != 0) { - code[off++] = Movk(16, static_cast((addr >> 16) & 0xFFFF), 16); - } - if (((addr >> 32) & 0xFFFF) != 0) { - code[off++] = Movk(16, static_cast((addr >> 32) & 0xFFFF), 32); - } - if (((addr >> 48) & 0xFFFF) != 0) { - code[off++] = Movk(16, static_cast((addr >> 48) & 0xFFFF), 48); - } - - // BR x16: tail-call. - code[off++] = BrReg(16); - - size_t bytes = off * 4; - return JitMemory::Get(isolate)->EmitStub( - reinterpret_cast(code), bytes); -} - -} // namespace node::ffi::fastcall - -#endif diff --git a/src/ffi/fastcall/stub_emitter_arm.cc b/src/ffi/fastcall/stub_emitter_arm.cc deleted file mode 100644 index 77391bab1a2f16..00000000000000 --- a/src/ffi/fastcall/stub_emitter_arm.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "ffi/fastcall/stub_emitter.h" - -#if defined(HAVE_FFI_FASTCALL) && defined(__arm__) - -#include -#include "ffi/fastcall/jit_memory.h" - -namespace node::ffi::fastcall { - -namespace { - -constexpr unsigned kMaxGPArgs = 3; // r1..r3 after stripping receiver in r0 - -uint32_t MovReg(unsigned dst, unsigned src) { - // mov rd, rm -> 0xE1A00000 | (dst<<12) | src - return 0xE1A00000u | (dst << 12) | src; -} - -uint32_t LdrLitR12() { - // ldr r12, [pc, #0] -> 0xE59FC000 - // pc-relative offset 0 means: literal at pc, where pc = ldr_pos + 8. - return 0xE59FC000u; -} - -uint32_t BxReg(unsigned reg) { - // bx rn -> 0xE12FFF10 | rn - return 0xE12FFF10u | reg; -} - -} // namespace - -std::optional EmitForwarder( - v8::Isolate* isolate, - void* target, - const std::vector& args, - ResultClass /*result*/) { - unsigned gp = 0, fp = 0; - for (auto c : args) { - switch (c) { - case ArgClass::kGP: ++gp; break; - case ArgClass::kFP: ++fp; break; - case ArgClass::kGPPair: return std::nullopt; // i64/u64 unsupported - } - } - if (gp > kMaxGPArgs) return std::nullopt; - if (fp > 16) return std::nullopt; - - uint32_t code[8] = {0}; - size_t off = 0; - - // Shift r1..rN -> r0..r(N-1). - for (unsigned i = 0; i < gp; ++i) { - code[off++] = MovReg(/*dst=*/i, /*src=*/i + 1); - } - - // Layout: ldr r12, [pc, #0] ; bx r12 ; .word target ; .word 0 - // After ldr executes, pc = ldr_addr + 8. With imm12 = 0, the ldr loads - // from pc + 0 = ldr_addr + 8. That's the address of the .word target. - code[off++] = LdrLitR12(); - code[off++] = BxReg(12); - code[off++] = static_cast(reinterpret_cast(target)); - code[off++] = 0; // padding to keep stub size a multiple of 8 - - size_t bytes = off * 4; - return JitMemory::Get(isolate)->EmitStub( - reinterpret_cast(code), bytes); -} - -} // namespace node::ffi::fastcall - -#endif diff --git a/src/ffi/fastcall/stub_emitter_x64_sysv.cc b/src/ffi/fastcall/stub_emitter_x64_sysv.cc deleted file mode 100644 index 20bc82975b8ed1..00000000000000 --- a/src/ffi/fastcall/stub_emitter_x64_sysv.cc +++ /dev/null @@ -1,87 +0,0 @@ -#include "ffi/fastcall/stub_emitter.h" - -#if defined(HAVE_FFI_FASTCALL) && defined(__x86_64__) && \ - (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)) - -#include -#include "ffi/fastcall/jit_memory.h" - -namespace node::ffi::fastcall { - -namespace { -// SysV x86_64 GP arg regs: RDI, RSI, RDX, RCX, R8, R9. -// FP arg regs: XMM0..XMM7. Receiver in RDI. -// -// Strip = shift GP regs down by one. FP regs are NOT shifted — receiver -// is GP. Up to 5 GP args: tail-call. 6 GP args: 7th GP slot is on the -// stack, use call+ret with stack rewrite. - -constexpr uint8_t kMoves[5][3] = { - {0x48, 0x89, 0xF7}, // mov rdi, rsi - {0x48, 0x89, 0xD6}, // mov rsi, rdx - {0x48, 0x89, 0xCA}, // mov rdx, rcx - {0x4C, 0x89, 0xC1}, // mov rcx, r8 - {0x4D, 0x89, 0xC8}, // mov r8, r9 -}; - -} // namespace - -std::optional EmitForwarder( - v8::Isolate* isolate, - void* target, - const std::vector& args, - ResultClass /*result*/) { - unsigned gp_count = 0; - unsigned fp_count = 0; - for (auto c : args) { - switch (c) { - case ArgClass::kGP: ++gp_count; break; - case ArgClass::kFP: ++fp_count; break; - case ArgClass::kGPPair: return std::nullopt; // unsupported on this ABI - } - } - if (gp_count > 6) return std::nullopt; - if (fp_count > 8) return std::nullopt; - - uint8_t buf[64] = {0}; - size_t off = 0; - - if (gp_count <= 5) { - // Tail-call path. - for (unsigned i = 0; i < gp_count; ++i) { - std::memcpy(buf + off, kMoves[i], 3); - off += 3; - } - // mov r10, imm64; jmp r10 - buf[off++] = 0x49; buf[off++] = 0xBA; - uint64_t addr = reinterpret_cast(target); - std::memcpy(buf + off, &addr, 8); off += 8; - buf[off++] = 0x41; buf[off++] = 0xFF; buf[off++] = 0xE2; - } else { - // gp_count == 6: 7th GP slot on stack. Use call+ret with stack rewrite. - // sub rsp, 8 -> 48 83 EC 08 - buf[off++] = 0x48; buf[off++] = 0x83; buf[off++] = 0xEC; buf[off++] = 0x08; - // shift first 5 GP regs - for (unsigned i = 0; i < 5; ++i) { - std::memcpy(buf + off, kMoves[i], 3); - off += 3; - } - // mov r9, [rsp+16] -> 4C 8B 4C 24 10 - buf[off++] = 0x4C; buf[off++] = 0x8B; buf[off++] = 0x4C; - buf[off++] = 0x24; buf[off++] = 0x10; - // mov r10, imm64; call r10 - buf[off++] = 0x49; buf[off++] = 0xBA; - uint64_t addr = reinterpret_cast(target); - std::memcpy(buf + off, &addr, 8); off += 8; - buf[off++] = 0x41; buf[off++] = 0xFF; buf[off++] = 0xD2; - // add rsp, 8; ret - buf[off++] = 0x48; buf[off++] = 0x83; buf[off++] = 0xC4; buf[off++] = 0x08; - buf[off++] = 0xC3; - } - - return JitMemory::Get(isolate)->EmitStub(buf, off); -} - -} // namespace node::ffi::fastcall - -#endif diff --git a/src/ffi/fastcall/stub_emitter_x64_win.cc b/src/ffi/fastcall/stub_emitter_x64_win.cc deleted file mode 100644 index d78367ffe03bc0..00000000000000 --- a/src/ffi/fastcall/stub_emitter_x64_win.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "ffi/fastcall/stub_emitter.h" - -#if defined(HAVE_FFI_FASTCALL) && defined(__x86_64__) && defined(_WIN32) - -#include -#include "ffi/fastcall/jit_memory.h" - -namespace node::ffi::fastcall { - -namespace { -// Win64 ABI: GP regs rcx, rdx, r8, r9 (4); FP regs xmm0-3 (4). -// FP/GP slots are POSITIONAL — slot N is one of {rcx,rdx,r8,r9} or one of -// {xmm0..xmm3} depending on the type at position N. Strip = shift the Nth -// arg from slot N+1 to slot N, where slot K maps to gp_reg[K] for kGP and -// xmm[K] for kFP. -// -// Encoding tables, indexed by destination slot 0..2: -// GP from slot+1 to slot: -// slot 0: mov rcx, rdx 48 89 D1 -// slot 1: mov rdx, r8 4C 89 C2 -// slot 2: mov r8, r9 4D 89 C8 -// FP from slot+1 to slot: -// slot 0: movaps xmm0, xmm1 0F 28 C1 -// slot 1: movaps xmm1, xmm2 0F 28 CA -// slot 2: movaps xmm2, xmm3 0F 28 D3 -// -// Cap: args.size() <= 3 (4 register slots minus the receiver). Stack-arg -// shuffling is not implemented; signatures with more args fall back to libffi. - -const uint8_t kGPMov[3][3] = { - {0x48, 0x89, 0xD1}, // mov rcx, rdx - {0x4C, 0x89, 0xC2}, // mov rdx, r8 - {0x4D, 0x89, 0xC8}, // mov r8, r9 -}; - -const uint8_t kFPMov[3][3] = { - {0x0F, 0x28, 0xC1}, // movaps xmm0, xmm1 - {0x0F, 0x28, 0xCA}, // movaps xmm1, xmm2 - {0x0F, 0x28, 0xD3}, // movaps xmm2, xmm3 -}; - -} // namespace - -std::optional EmitForwarder( - v8::Isolate* isolate, - void* target, - const std::vector& args, - ResultClass /*result*/) { - if (args.size() > 3) return std::nullopt; // 4 reg slots minus receiver - for (auto c : args) { - switch (c) { - case ArgClass::kGP: - case ArgClass::kFP: - break; - case ArgClass::kGPPair: - return std::nullopt; // unsupported on Win64 - } - } - - uint8_t buf[64] = {0}; - size_t off = 0; - - for (size_t i = 0; i < args.size(); ++i) { - const uint8_t* enc = (args[i] == ArgClass::kGP) ? kGPMov[i] : kFPMov[i]; - std::memcpy(buf + off, enc, 3); - off += 3; - } - - // mov r10, imm64; jmp r10 - buf[off++] = 0x49; buf[off++] = 0xBA; - uint64_t addr = reinterpret_cast(target); - std::memcpy(buf + off, &addr, 8); off += 8; - buf[off++] = 0x41; buf[off++] = 0xFF; buf[off++] = 0xE2; - - return JitMemory::Get(isolate)->EmitStub(buf, off); -} - -} // namespace node::ffi::fastcall - -#endif diff --git a/src/ffi/types.cc b/src/ffi/types.cc index 52c93193cfe9f9..01a8fed9d60e98 100644 --- a/src/ffi/types.cc +++ b/src/ffi/types.cc @@ -661,23 +661,6 @@ bool IsFunctionTypeName(const std::string& s) { return s == "function"; } -// Per-platform emitter GP caps (excluding the implicit receiver slot). -// Note: Win64 uses a combined gp+fp <= 3 positional-slot cap checked below; -// it does not use kFastcallMaxGPArgs. -#if defined(__aarch64__) || defined(_M_ARM64) -constexpr unsigned kFastcallMaxGPArgs = 7; -#elif defined(__x86_64__) && !defined(_WIN32) -constexpr unsigned kFastcallMaxGPArgs = 6; -#elif defined(__arm__) -constexpr unsigned kFastcallMaxGPArgs = 3; -#else -constexpr unsigned kFastcallMaxGPArgs = 0; // no emitter on this platform -#endif - -// Every supported emitter allows up to 8 FP args. Win64 is further -// constrained by the gp+fp<=3 positional-slot cap checked below. -constexpr unsigned kFastcallMaxFPArgs = 8; - } // namespace bool IsFastCallEligible(const FFIFunction& fn, const char** out_reason) { @@ -693,7 +676,14 @@ bool IsFastCallEligible(const FFIFunction& fn, const char** out_reason) { return false; } - unsigned gp = 0, fp = 0; + // V8's fast-call lowering caps the C-side arg count. With HasReceiver=kNo + // there's no implicit receiver in the count, so this is the user-arg cap. + // V8's hard limit is 8 args; signatures over that fall back to libffi. + if (fn.args.size() > 8) { + *out_reason = "argument count exceeds V8 fast-call cap"; + return false; + } + for (size_t i = 0; i < fn.args.size(); ++i) { ffi_type* t = fn.args[i]; const std::string& name = fn.arg_type_names[i]; @@ -703,10 +693,8 @@ bool IsFastCallEligible(const FFIFunction& fn, const char** out_reason) { return false; } // `void` is fine as a return type but has no register slot, so it cannot - // appear in `args`. `IsFastCallEligibleFFIType` accepts it for the - // return-type check above; reject it explicitly here so the GP/FP - // counting below stays consistent and `MapArgType` is never called with - // it (where it would hit UNREACHABLE). + // appear in `args`. Reject it explicitly so MapArgType is never called + // with it (where it would hit UNREACHABLE). if (t == &ffi_type_void) { *out_reason = "void cannot be an argument type"; return false; @@ -715,48 +703,7 @@ bool IsFastCallEligible(const FFIFunction& fn, const char** out_reason) { *out_reason = "arg is function"; return false; } - -#if defined(__arm__) - // AArch32: i64/u64 consume two GP regs with alignment rules. Not - // supported in the v1 stub emitter. - if (t == &ffi_type_sint64 || t == &ffi_type_uint64) { - *out_reason = "i64/u64 arg unsupported on aarch32"; - return false; - } -#endif - - if (t == &ffi_type_float || t == &ffi_type_double) { - ++fp; - } else { - ++gp; - } - } - -#if defined(__arm__) - if (fn.return_type == &ffi_type_sint64 || - fn.return_type == &ffi_type_uint64) { - *out_reason = "i64/u64 return unsupported on aarch32"; - return false; - } -#endif - -#if defined(__x86_64__) && defined(_WIN32) - // Win64: positional slots — only 4 total (one taken by receiver), so - // gp + fp must not exceed 3. - if (gp + fp > 3) { - *out_reason = "GP + FP arg count exceeds Win64 cap"; - return false; - } -#else - if (gp > kFastcallMaxGPArgs) { - *out_reason = "GP arg count exceeds ABI cap"; - return false; - } - if (fp > kFastcallMaxFPArgs) { - *out_reason = "FP arg count exceeds ABI cap"; - return false; } -#endif *out_reason = ""; return true; diff --git a/src/node_ffi.cc b/src/node_ffi.cc index 9935d532ad32e7..9c949c0e1449f2 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -10,15 +10,9 @@ #include "ffi/data.h" #include "ffi/types.h" #include "node_errors.h" -#include "node_process-inl.h" - -#include -#include #ifdef HAVE_FFI_FASTCALL #include "ffi/fastcall/cfunction_info.h" -#include "ffi/fastcall/jit_memory.h" -#include "ffi/fastcall/stub_emitter.h" #endif namespace node { @@ -58,22 +52,13 @@ namespace ffi { FastCallState::FastCallState( v8::Isolate* isolate, v8::Local slow_fn, - void* stub, - size_t alloc_size, std::unique_ptr bndl) : slow_invoke(isolate, slow_fn), - stub_entry(stub), - stub_alloc_size(alloc_size), cfun_bundle(std::move(bndl)) {} FastCallState::~FastCallState() { slow_invoke.Reset(); cfun_bundle.reset(); - if (stub_entry != nullptr) { - node::ffi::fastcall::JitMemory::Get(/*isolate=*/nullptr) - ->Free(stub_entry, stub_alloc_size); - stub_entry = nullptr; - } } #endif @@ -106,18 +91,6 @@ void DynamicLibrary::MemoryInfo(MemoryTracker* tracker) const { // FFIFunctionInfo instances are owned by V8 function wrappers and reachable // only via weak references, so they are deliberately not counted here. - -#ifdef HAVE_FFI_FASTCALL - // Process-wide JIT bytes from the singleton allocator. Per-library - // accounting would require plumbing per-stub bookkeeping through - // CreateFunction; for v1 we report the singleton total under each - // library's MemoryInfo, with a name that makes the scope clear. - size_t jit_bytes = - node::ffi::fastcall::JitMemory::Get(env()->isolate())->TotalLiveBytes(); - tracker->TrackFieldWithSize( - "jit_stubs_process_global", jit_bytes, - "ffi::FastCallStubs (process-wide singleton)"); -#endif } #ifdef HAVE_FFI_FASTCALL @@ -288,34 +261,7 @@ MaybeLocal DynamicLibrary::CreateFunction( #ifdef HAVE_FFI_FASTCALL const char* fc_reason = ""; - // One-time process-wide self-test. On failure (entitlement missing, - // hardened runtime, SELinux execmem denial, etc.), every subsequent - // registration falls back to libffi. - static std::atomic selftest_result{-1}; - static std::once_flag selftest_warn_once; - int s = selftest_result.load(std::memory_order_acquire); - if (s == -1) { - bool ok = node::ffi::fastcall::JitMemory::Get(isolate)->SelfTest(isolate); - s = ok ? 1 : 0; - selftest_result.store(s, std::memory_order_release); - if (!ok) { - // The atomic above only dedupes the SelfTest *call*; two threads can - // both observe `s == -1` and both reach this branch. `call_once` - // ensures at most one warning is emitted per process across all - // racing first-callers. - bool warn_failed = false; - std::call_once(selftest_warn_once, [&]() { - if (ProcessEmitWarning(env, - "FFI fast-call self-test failed; falling back to libffi for " - "all FFI invocations") - .IsNothing()) { - warn_failed = true; - } - }); - if (warn_failed) return MaybeLocal(); - } - } - bool fc_ok = (s == 1) && node::ffi::IsFastCallEligible(*fn, &fc_reason); + bool fc_ok = node::ffi::IsFastCallEligible(*fn, &fc_reason); if (fc_ok) { auto bundle = node::ffi::fastcall::BuildCFunctionInfo(*fn); { @@ -392,7 +338,7 @@ MaybeLocal DynamicLibrary::CreateFunction( } info->fast = std::make_unique( - isolate, slow_fn, /*stub=*/nullptr, /*alloc_size=*/0, + isolate, slow_fn, std::make_unique( std::move(bundle))); ret = fc_fn; diff --git a/src/node_ffi.h b/src/node_ffi.h index 8efbb7a1a026d4..2dcae1d1527c96 100644 --- a/src/node_ffi.h +++ b/src/node_ffi.h @@ -14,11 +14,8 @@ #include #ifdef HAVE_FFI_FASTCALL -// jit_memory.h only uses standard-library headers so it is safe to include -// here. cfunction_info.h includes node_ffi.h, so it cannot be included here -// (cycle), but a forward declaration suffices for the unique_ptr member. -#include "ffi/fastcall/jit_memory.h" // for JitMemory, EmittedStub - +// cfunction_info.h includes node_ffi.h, so it cannot be included here +// (cycle); a forward declaration suffices for the unique_ptr member. namespace node::ffi::fastcall { struct CFunctionInfoBundle; } // namespace node::ffi::fastcall @@ -42,40 +39,21 @@ struct FFIFunction { #ifdef HAVE_FFI_FASTCALL // Owns the per-function fast-call state for a single FFI registration. -// Three resources are owned and freed by the destructor: -// 1. The JIT stub bytes — released via -// `JitMemory::Free(stub_entry, stub_alloc_size)`. -// 2. The slow-path libffi `v8::Global` — reset. -// 3. The `CFunctionInfoBundle` (which itself owns heap-allocated -// `v8::CFunctionInfo` and `v8::CTypeInfo[]`) — destroyed via the -// bundle's destructor. +// Two resources are owned: the slow-path libffi `v8::Global`, +// and the `CFunctionInfoBundle` (which itself owns the heap-allocated +// `v8::CFunctionInfo` + `v8::CTypeInfo[]`). // // `FFIFunctionInfo::fast` is null for ineligible signatures (libffi-only // path); a non-null `fast` always represents a fully-constructed // fast-call state. struct FastCallState { v8::Global slow_invoke; - void* stub_entry = nullptr; - size_t stub_alloc_size = 0; std::unique_ptr cfun_bundle; - // Default-construction is forbidden — every FastCallState must hold a - // valid stub/bundle/slow-invoke triple. Construct via the value form. FastCallState() = delete; - // Construct a fully-armed FastCallState. All resources move into the - // new object atomically (under -fno-exceptions, member init runs - // top-to-bottom; an OOM in any V8 handle allocation aborts the - // process, which is the project contract). - // Preconditions: - // - `slow_fn` is a non-empty Local. - // - `stub` is a non-null pointer returned from JitMemory::EmitStub. - // - `alloc_size` is the matching allocation size from EmittedStub. - // - `bndl` is a non-null unique_ptr. FastCallState(v8::Isolate* isolate, v8::Local slow_fn, - void* stub, - size_t alloc_size, std::unique_ptr bndl); ~FastCallState(); FastCallState(const FastCallState&) = delete; diff --git a/test/cctest/test_ffi_fastcall_cfunction.cc b/test/cctest/test_ffi_fastcall_cfunction.cc index d5b2e063db7347..0af06c816cb614 100644 --- a/test/cctest/test_ffi_fastcall_cfunction.cc +++ b/test/cctest/test_ffi_fastcall_cfunction.cc @@ -9,9 +9,7 @@ #include "node_ffi.h" using node::ffi::FFIFunction; -using node::ffi::fastcall::ArgClass; using node::ffi::fastcall::BuildCFunctionInfo; -using node::ffi::fastcall::ResultClass; namespace { FFIFunction MakeFn(ffi_type* ret, @@ -32,14 +30,10 @@ TEST(FFIFastCallCFunction, NumericSignature) { {&ffi_type_sint32, &ffi_type_sint32}, {"i32", "i32"}); auto bundle = BuildCFunctionInfo(fn); - // Receiver + 2 args = 3 CTypeInfo entries. - // No-receiver mode: ArgumentCount counts user args only (no leading + // HasReceiver=kNo: ArgumentCount counts user args only (no leading // v8::Value receiver slot). EXPECT_EQ(bundle.info->ArgumentCount(), 2u); - EXPECT_EQ(bundle.arg_classes.size(), 2u); - EXPECT_EQ(bundle.arg_classes[0], ArgClass::kGP); - EXPECT_EQ(bundle.arg_classes[1], ArgClass::kGP); - EXPECT_EQ(bundle.result_class, ResultClass::kGP); + EXPECT_FALSE(bundle.info->HasReceiverArg()); } TEST(FFIFastCallCFunction, FloatSignature) { @@ -47,36 +41,30 @@ TEST(FFIFastCallCFunction, FloatSignature) { {&ffi_type_float, &ffi_type_double}, {"float", "double"}); auto bundle = BuildCFunctionInfo(fn); - EXPECT_EQ(bundle.arg_classes[0], ArgClass::kFP); - EXPECT_EQ(bundle.arg_classes[1], ArgClass::kFP); - EXPECT_EQ(bundle.result_class, ResultClass::kFP); + EXPECT_EQ(bundle.info->ArgumentCount(), 2u); } TEST(FFIFastCallCFunction, VoidReturn) { auto fn = MakeFn(&ffi_type_void, "void", {&ffi_type_sint32}, {"i32"}); auto bundle = BuildCFunctionInfo(fn); - EXPECT_EQ(bundle.result_class, ResultClass::kVoid); + EXPECT_EQ(bundle.info->ArgumentCount(), 1u); } TEST(FFIFastCallCFunction, PointerSignature) { auto fn = MakeFn(&ffi_type_pointer, "pointer", {&ffi_type_pointer}, {"pointer"}); auto bundle = BuildCFunctionInfo(fn); - EXPECT_EQ(bundle.arg_classes[0], ArgClass::kGP); - EXPECT_EQ(bundle.result_class, ResultClass::kGP); + EXPECT_EQ(bundle.info->ArgumentCount(), 1u); } TEST(FFIFastCallCFunction, MoveCleanupSafe) { auto fn = MakeFn(&ffi_type_sint32, "i32", {&ffi_type_sint32}, {"i32"}); auto a = BuildCFunctionInfo(fn); - // Move into a new bundle and let it drop. The moved-from bundle's - // destructor must not double-free. auto b = std::move(a); EXPECT_EQ(a.info, nullptr); EXPECT_EQ(a.arg_types, nullptr); - // b's destructor runs at end of scope; no crash expected. } TEST(FFIFastCallCFunction, MoveAssignmentSelfSafe) { @@ -85,19 +73,13 @@ TEST(FFIFastCallCFunction, MoveAssignmentSelfSafe) { auto bundle = BuildCFunctionInfo(fn); ASSERT_NE(bundle.info, nullptr); ASSERT_NE(bundle.arg_types, nullptr); - ASSERT_EQ(bundle.arg_classes.size(), 1u); - ASSERT_EQ(bundle.result_class, node::ffi::fastcall::ResultClass::kGP); - // Self-move-assign must be a no-op (or at least not double-free). - // The self-assignment guard should keep all four bundle fields valid. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wself-move" bundle = std::move(bundle); #pragma GCC diagnostic pop EXPECT_NE(bundle.info, nullptr); EXPECT_NE(bundle.arg_types, nullptr); - EXPECT_EQ(bundle.arg_classes.size(), 1u); - EXPECT_EQ(bundle.result_class, node::ffi::fastcall::ResultClass::kGP); } TEST(FFIFastCallCFunction, MoveAssignmentReplaces) { @@ -109,7 +91,6 @@ TEST(FFIFastCallCFunction, MoveAssignmentReplaces) { auto bundle_b = BuildCFunctionInfo(fn_b); const void* old_a_info = bundle_a.info; bundle_a = std::move(bundle_b); - // bundle_a now holds what bundle_b had; bundle_b is nulled out. EXPECT_EQ(bundle_b.info, nullptr); EXPECT_NE(bundle_a.info, nullptr); EXPECT_NE(bundle_a.info, old_a_info); diff --git a/test/cctest/test_ffi_fastcall_eligibility.cc b/test/cctest/test_ffi_fastcall_eligibility.cc index 7b523e0503b254..1f7e274b96bfc0 100644 --- a/test/cctest/test_ffi_fastcall_eligibility.cc +++ b/test/cctest/test_ffi_fastcall_eligibility.cc @@ -72,34 +72,24 @@ TEST(FFIFastCallEligibility, FunctionReturnRejected) { EXPECT_FALSE(IsFastCallEligible(fn, &reason)); } -TEST(FFIFastCallEligibility, TooManyGPArgsRejected) { - // 8 GP args is one over AArch64's cap of 7 (the largest GP cap of any - // supported platform), so this is rejected on every supported emitter. +// V8 fast-call's hard cap is 8 C-side args. With HasReceiver=kNo there's no +// implicit receiver in the count, so user functions can have up to 8 args. +TEST(FFIFastCallEligibility, EightArgsAccepted) { std::vector args(8, &ffi_type_sint32); std::vector names(8, "i32"); auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); const char* reason = nullptr; - EXPECT_FALSE(IsFastCallEligible(fn, &reason)); - ASSERT_NE(reason, nullptr); - // On non-Win64 platforms the reason mentions "GP arg"; on Win64 the combined - // gp+fp cap fires with "Win64 cap". Accept either. - const std::string r(reason); - EXPECT_TRUE(r.find("GP arg") != std::string::npos || - r.find("Win64 cap") != std::string::npos); + EXPECT_TRUE(IsFastCallEligible(fn, &reason)); } -TEST(FFIFastCallEligibility, TooManyFPArgsRejected) { - // 9 FP args is one over the FP cap of 8. - std::vector args(9, &ffi_type_double); - std::vector names(9, "double"); +TEST(FFIFastCallEligibility, NineArgsRejected) { + std::vector args(9, &ffi_type_sint32); + std::vector names(9, "i32"); auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); const char* reason = nullptr; EXPECT_FALSE(IsFastCallEligible(fn, &reason)); ASSERT_NE(reason, nullptr); - // On non-Win64 the reason mentions "FP arg"; on Win64 the combined cap fires. - const std::string r(reason); - EXPECT_TRUE(r.find("FP arg") != std::string::npos || - r.find("Win64 cap") != std::string::npos); + EXPECT_NE(std::string(reason).find("argument count"), std::string::npos); } TEST(FFIFastCallEligibility, VoidArgRejected) { @@ -117,60 +107,6 @@ TEST(FFIFastCallEligibility, NullOutReasonOk) { EXPECT_TRUE(IsFastCallEligible(fn, nullptr)); } -// Per-platform GP cap boundary tests. - -#if defined(__aarch64__) || defined(_M_ARM64) -// AArch64 allows up to 7 GP args — the largest GP cap of any supported -// platform. Verify that 7 is accepted and 8 is rejected. -TEST(FFIFastCallEligibility, AArch64GPCapBoundary) { - std::vector args(7, &ffi_type_sint32); - std::vector names(7, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_TRUE(IsFastCallEligible(fn, &reason)); -} -#endif // AArch64 - -#if defined(__x86_64__) && \ - (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)) -// SysV (Linux/macOS/FreeBSD x86_64) allows up to 6 GP args. Verify that 6 -// is accepted and 7 is rejected. -TEST(FFIFastCallEligibility, SysVGPCapBoundary) { - std::vector args(6, &ffi_type_sint32); - std::vector names(6, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_TRUE(IsFastCallEligible(fn, &reason)); -} - -TEST(FFIFastCallEligibility, SysVGPCapExceeded) { - std::vector args(7, &ffi_type_sint32); - std::vector names(7, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_FALSE(IsFastCallEligible(fn, &reason)); -} -#endif // SysV x86_64 - -#if defined(_WIN32) && (defined(__x86_64__) || defined(_M_X64)) -// Win64 allows at most 3 combined GP+FP args. Verify the boundary. -TEST(FFIFastCallEligibility, Win64CombinedCapBoundary) { - std::vector args(3, &ffi_type_sint32); - std::vector names(3, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_TRUE(IsFastCallEligible(fn, &reason)); -} - -TEST(FFIFastCallEligibility, Win64CombinedCapExceeded) { - std::vector args(4, &ffi_type_sint32); - std::vector names(4, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_FALSE(IsFastCallEligible(fn, &reason)); -} -#endif // Win64 - TEST(FFIFastCallEligibility, StringTypeNameOk) { auto fn = MakeFn(&ffi_type_pointer, "pointer", {&ffi_type_pointer}, {"string"}); @@ -192,42 +128,4 @@ TEST(FFIFastCallEligibility, ArraybufferTypeNameOk) { EXPECT_TRUE(IsFastCallEligible(fn, &reason)); } -#if defined(__arm__) -TEST(FFIFastCallEligibility, ARM32GPCapBoundary) { - std::vector args(3, &ffi_type_sint32); - std::vector names(3, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", - std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_TRUE(IsFastCallEligible(fn, &reason)); -} - -TEST(FFIFastCallEligibility, ARM32GPCapExceeded) { - std::vector args(4, &ffi_type_sint32); - std::vector names(4, "i32"); - auto fn = MakeFn(&ffi_type_void, "void", - std::move(args), std::move(names)); - const char* reason = nullptr; - EXPECT_FALSE(IsFastCallEligible(fn, &reason)); -} - -TEST(FFIFastCallEligibility, ARM32I64ArgRejected) { - auto fn = MakeFn(&ffi_type_void, "void", - {&ffi_type_sint64}, {"i64"}); - const char* reason = nullptr; - EXPECT_FALSE(IsFastCallEligible(fn, &reason)); - ASSERT_NE(reason, nullptr); - EXPECT_NE(std::string(reason).find("aarch32"), std::string::npos); -} - -TEST(FFIFastCallEligibility, ARM32I64ReturnRejected) { - auto fn = MakeFn(&ffi_type_sint64, "i64", - {&ffi_type_sint32}, {"i32"}); - const char* reason = nullptr; - EXPECT_FALSE(IsFastCallEligible(fn, &reason)); - ASSERT_NE(reason, nullptr); - EXPECT_NE(std::string(reason).find("aarch32"), std::string::npos); -} -#endif // __arm__ - #endif // HAVE_FFI_FASTCALL diff --git a/test/cctest/test_ffi_fastcall_emitter.cc b/test/cctest/test_ffi_fastcall_emitter.cc deleted file mode 100644 index 561e396facb6b4..00000000000000 --- a/test/cctest/test_ffi_fastcall_emitter.cc +++ /dev/null @@ -1,233 +0,0 @@ -#include "gtest/gtest.h" - -#ifdef HAVE_FFI_FASTCALL - -#include - -#include "ffi/fastcall/jit_memory.h" -#include "ffi/fastcall/stub_emitter.h" -#include "node_test_fixture.h" - -using node::ffi::fastcall::ArgClass; -using node::ffi::fastcall::EmitForwarder; -using node::ffi::fastcall::JitMemory; -using node::ffi::fastcall::ResultClass; - -class StubEmitterTest : public NodeTestFixture { - protected: - void TearDown() override { - JitMemory::Get(isolate_)->ResetForTesting(); - NodeTestFixture::TearDown(); - } -}; - -// Test target functions. Plain static so the optimizer doesn't merge -// them with anything else. -namespace { -int target_zero_args() { return 1234; } -int target_one_int(int a) { return a + 100; } -int target_two_int(int a, int b) { return a - b; } -int target_six_int(int a, int b, int c, int d, int e, int f) { - return a + b + c + d + e + f; -} -int target_seven_int(int a, int b, int c, int d, int e, int f, int g) { - return a + b + c + d + e + f + g; -} -double target_one_double(double a) { return a * 2.0; } -int target_int_double(int a, double b) { - return a + static_cast(b); -} -[[maybe_unused]] int target_fp_int(double a, int b) { return static_cast(a) + b; } -[[maybe_unused]] int target_three_int(int a, int b, int c) { return a + b + c; } -[[maybe_unused]] int target_four_int(int a, int b, int c, int d) { - return a + b + c + d; -} -} // namespace - -#if defined(__aarch64__) || defined(_M_ARM64) || \ - defined(__x86_64__) || defined(_M_X64) - -TEST_F(StubEmitterTest, ZeroArgs) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_zero_args), - {}, - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(1234, fn(reinterpret_cast(0xdeadbeefULL))); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, OneIntArg) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_one_int), - {ArgClass::kGP}, - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(142, fn(nullptr, 42)); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, TwoIntArgs) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_two_int), - {ArgClass::kGP, ArgClass::kGP}, - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int, int); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(7, fn(nullptr, 10, 3)); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, OneDoubleArg) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_one_double), - {ArgClass::kFP}, - ResultClass::kFP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = double (*)(void* receiver, double); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_DOUBLE_EQ(6.28, fn(nullptr, 3.14)); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, MixedIntDouble) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_int_double), - {ArgClass::kGP, ArgClass::kFP}, - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int, double); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(13, fn(nullptr, 10, 3.7)); - JitMemory::Get(isolate_)->Free(*stub); -} - -// SixIntArgs: passes on AArch64 (cap=7) and SysV x86_64 (cap=6), but fails -// on Win64 (combined GP+FP cap=3). Gate it out on Win64. -#if defined(__aarch64__) || defined(_M_ARM64) || \ - (defined(__x86_64__) && !defined(_WIN32)) -TEST_F(StubEmitterTest, SixIntArgs) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_six_int), - std::vector(6, ArgClass::kGP), - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int, int, int, int, int, int); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(21, fn(nullptr, 1, 2, 3, 4, 5, 6)); - JitMemory::Get(isolate_)->Free(*stub); -} -#endif // AArch64 || (x86_64 && !Win64) - -// SevenIntArgs: only AArch64 supports 7 GP args (cap=7). SysV cap is 6, -// Win64 cap is 3 combined — both would reject 7 GP args. -#if defined(__aarch64__) || defined(_M_ARM64) -TEST_F(StubEmitterTest, SevenIntArgs) { - // AArch64 supports up to 7 GP args after the receiver (kMaxGPArgs=7). - auto stub = EmitForwarder( - isolate_, reinterpret_cast(target_seven_int), - std::vector(7, ArgClass::kGP), ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int, int, int, int, int, int, int); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(28, fn(nullptr, 1, 2, 3, 4, 5, 6, 7)); - JitMemory::Get(isolate_)->Free(*stub); -} -#endif // AArch64 - -// SysV (Linux/macOS/FreeBSD x86_64) cap is exactly 6 GP args. 7 must fail. -#if defined(__x86_64__) && \ - (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)) -TEST_F(StubEmitterTest, SevenGPArgsRejectedSysV) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_seven_int), - std::vector(7, ArgClass::kGP), - ResultClass::kGP); - EXPECT_FALSE(stub.has_value()); -} -#endif // SysV x86_64 - -TEST_F(StubEmitterTest, EightIntArgsRejected) { - // 8 GP args would require stack handling not supported in v1. - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_seven_int), - std::vector(8, ArgClass::kGP), - ResultClass::kGP); - EXPECT_FALSE(stub.has_value()); -} - -TEST_F(StubEmitterTest, GPPairRejected) { - // kGPPair is for AArch32 only; reject on AArch64/x86_64. - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_one_int), - {ArgClass::kGPPair}, - ResultClass::kGP); - EXPECT_FALSE(stub.has_value()); -} - -#if defined(__aarch64__) || defined(_M_ARM64) -TEST_F(StubEmitterTest, NineFPArgsRejected) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_one_double), - std::vector(9, ArgClass::kFP), - ResultClass::kGP); - EXPECT_FALSE(stub.has_value()); -} -#endif // __aarch64__ - -#if defined(_WIN32) && (defined(__x86_64__) || defined(_M_X64)) -TEST_F(StubEmitterTest, Win64MixedGPThenFP) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_int_double), - {ArgClass::kGP, ArgClass::kFP}, - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int, double); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(13, fn(nullptr, 10, 3.7)); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, Win64FPThenGP) { - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_fp_int), - {ArgClass::kFP, ArgClass::kGP}, - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, double, int); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(13, fn(nullptr, 3.7, 10)); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, Win64ThreeArgsAccepted) { - // 3 args is the Win64 cap (4 register slots minus the receiver). - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_three_int), - std::vector(3, ArgClass::kGP), - ResultClass::kGP); - ASSERT_TRUE(stub.has_value()); - using FnPtr = int (*)(void* receiver, int, int, int); - FnPtr fn = reinterpret_cast(stub->entry); - EXPECT_EQ(6, fn(nullptr, 1, 2, 3)); - JitMemory::Get(isolate_)->Free(*stub); -} - -TEST_F(StubEmitterTest, Win64FourArgsRejected) { - // 4 args exceeds the Win64 register-slot cap. - auto stub = EmitForwarder(isolate_, - reinterpret_cast(target_four_int), - std::vector(4, ArgClass::kGP), - ResultClass::kGP); - EXPECT_FALSE(stub.has_value()); -} -#endif // _WIN32 && x86_64 - -#endif // __aarch64__ || x86_64 - -#endif // HAVE_FFI_FASTCALL diff --git a/test/cctest/test_ffi_fastcall_jit.cc b/test/cctest/test_ffi_fastcall_jit.cc deleted file mode 100644 index 473be6abb5e7dd..00000000000000 --- a/test/cctest/test_ffi_fastcall_jit.cc +++ /dev/null @@ -1,159 +0,0 @@ -#include "gtest/gtest.h" - -#ifdef HAVE_FFI_FASTCALL - -#include -#include - -#if defined(__POSIX__) -#include -#endif - -#include "ffi/fastcall/jit_memory.h" -#include "node_test_fixture.h" - -namespace { -size_t SysPageSize() { -#if defined(__POSIX__) - return static_cast(sysconf(_SC_PAGESIZE)); -#else - return 4096; -#endif -} -} // namespace - -using node::ffi::fastcall::JitMemory; - -class JitMemoryTest : public NodeTestFixture { - protected: - void TearDown() override { - JitMemory::Get(isolate_)->ResetForTesting(); - NodeTestFixture::TearDown(); - } -}; - -TEST_F(JitMemoryTest, AllocateZeroFails) { - auto* jit = JitMemory::Get(isolate_); - size_t alloc_size = 0; - EXPECT_EQ(nullptr, jit->Allocate(0, &alloc_size)); -} - -TEST_F(JitMemoryTest, AllocateSmallReturnsAligned) { - auto* jit = JitMemory::Get(isolate_); - size_t alloc_size = 0; - void* p = jit->Allocate(32, &alloc_size); - ASSERT_NE(nullptr, p); - EXPECT_GE(alloc_size, 32u); - EXPECT_EQ(0u, reinterpret_cast(p) % 64); - jit->Free(p, alloc_size); -} - -TEST_F(JitMemoryTest, FreeIsNoOp) { - auto* jit = JitMemory::Get(isolate_); - size_t a = 0, b = 0; - void* p1 = jit->Allocate(48, &a); - ASSERT_NE(nullptr, p1); - size_t live_before = jit->TotalLiveBytes(); - jit->Free(p1, a); - EXPECT_LT(jit->TotalLiveBytes(), live_before); - void* p2 = jit->Allocate(48, &b); - ASSERT_NE(nullptr, p2); - EXPECT_NE(p1, p2); - jit->Free(p2, b); -} - -TEST_F(JitMemoryTest, ManySmallAllocsFit) { - auto* jit = JitMemory::Get(isolate_); - std::vector> ptrs; - for (int i = 0; i < 200; ++i) { - size_t a = 0; - void* p = jit->Allocate(40, &a); - ASSERT_NE(nullptr, p) << "alloc " << i; - ptrs.push_back({p, a}); - } - for (auto& [p, a] : ptrs) jit->Free(p, a); -} - -#if defined(__x86_64__) || defined(_M_X64) || \ - defined(__aarch64__) || defined(_M_ARM64) -TEST_F(JitMemoryTest, ExecutableStubReturnsValue) { - auto* jit = JitMemory::Get(isolate_); - size_t alloc_size = 0; - void* p = jit->Allocate(64, &alloc_size); - ASSERT_NE(nullptr, p); - - uint8_t* code = static_cast(p); -#if defined(__x86_64__) || defined(_M_X64) - // mov eax, 42; ret - code[0] = 0xb8; code[1] = 0x2a; code[2] = 0x00; - code[3] = 0x00; code[4] = 0x00; - code[5] = 0xc3; -#elif defined(__aarch64__) || defined(_M_ARM64) - uint32_t* w = reinterpret_cast(code); - w[0] = 0x52800540; // mov w0, #42 - w[1] = 0xd65f03c0; // ret -#endif - - ASSERT_TRUE(jit->MakeExecutable(p, alloc_size)); - using FnPtr = int (*)(); - EXPECT_EQ(42, reinterpret_cast(p)()); - jit->Free(p, alloc_size); -} -#endif // x86_64 || aarch64 - -TEST_F(JitMemoryTest, SelfTestPasses) { - EXPECT_TRUE(JitMemory::Get(isolate_)->SelfTest(isolate_)); -} - -TEST_F(JitMemoryTest, MultiplePagesAllocate) { - auto* jit = JitMemory::Get(isolate_); - // Force at least 2 pages by allocating many large chunks. - // 1000 × 1024 bytes = ~1 MB, guaranteed to span pages on any platform. - std::vector> ptrs; - for (int i = 0; i < 1000; ++i) { - size_t a = 0; - void* p = jit->Allocate(1024, &a); - ASSERT_NE(nullptr, p) << "alloc " << i; - ptrs.push_back({p, a}); - } - // Sanity: at least two distinct page bases observed. - const size_t page_mask = ~(SysPageSize() - 1); - std::set page_bases; - for (auto& [p, ignored] : ptrs) { - page_bases.insert(reinterpret_cast(p) & page_mask); - } - EXPECT_GE(page_bases.size(), 2u); - for (auto& [p, a] : ptrs) jit->Free(p, a); -} - -TEST_F(JitMemoryTest, MakeExecutableNullArgsSafe) { - auto* jit = JitMemory::Get(isolate_); - // Verify graceful handling of nullptr / zero size. The page_size_==0 - // uninitialized branch is unreachable here because Get(isolate_) already - // initialized the singleton; that branch is defense-in-depth only. - EXPECT_FALSE(jit->MakeExecutable(nullptr, 64)); - EXPECT_FALSE(jit->MakeExecutable(reinterpret_cast(0x1), 0)); -} - -TEST_F(JitMemoryTest, AllocateAfterMakeExecutableSkipsSealedPage) { - auto* jit = JitMemory::Get(isolate_); - size_t a1 = 0; - void* p1 = jit->Allocate(64, &a1); - ASSERT_NE(nullptr, p1); - ASSERT_TRUE(jit->MakeExecutable(p1, a1)); - - // After sealing, a new alloc should NOT be in the same page. - size_t a2 = 0; - void* p2 = jit->Allocate(64, &a2); - ASSERT_NE(nullptr, p2); - const uintptr_t page_mask = ~(SysPageSize() - 1); - uintptr_t page1 = reinterpret_cast(p1) & page_mask; - uintptr_t page2 = reinterpret_cast(p2) & page_mask; - EXPECT_NE(page1, page2); - - jit->Free(p2, a2); - // p1 was already made executable; freeing is fine (counter only). - jit->Free(p1, a1); -} - -#endif // HAVE_FFI_FASTCALL