diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 1351ba966..b8639545d 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -16,7 +16,6 @@ #include "incbin.h" #include "jfrMetadata.h" #include "jniHelper.h" -#include "jvm.h" #include "os.h" #include "profiler.h" #include "rustDemangler.h" diff --git a/ddprof-lib/src/main/cpp/frames.h b/ddprof-lib/src/main/cpp/frames.h new file mode 100644 index 000000000..15549e6e8 --- /dev/null +++ b/ddprof-lib/src/main/cpp/frames.h @@ -0,0 +1,18 @@ +#ifndef _FRAMES_H +#define _FRAMES_H + +#include +#include "vmEntry.h" + +inline int makeFrame(ASGCT_CallFrame *frames, jint type, jmethodID id) { + frames[0].bci = type; + frames[0].method_id = id; + return 1; +} + +inline int makeFrame(ASGCT_CallFrame *frames, jint type, + const char *id) { + return makeFrame(frames, type, (jmethodID)id); +} + +#endif // _FRAMES_H diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp new file mode 100644 index 000000000..ae342efd2 --- /dev/null +++ b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp @@ -0,0 +1,987 @@ +/* + * Copyright The async-profiler authors + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "asyncSampleMutex.h" +#include "hotspot/hotspotSupport.h" +#include "hotspot/jitCodeCache.h" +#include "hotspot/vmStructs.inline.h" +#include "jvmSupport.h" +#include "profiler.h" +#include "stackFrame.h" +#include "stackWalker.inline.h" +#include "frames.h" + +using StackWalkValidation::inDeadZone; +using StackWalkValidation::aligned; +using StackWalkValidation::MAX_FRAME_SIZE; +using StackWalkValidation::sameStack; + +static bool isAddressInCode(const void *pc, bool include_stubs = true) { + if (CodeHeap::contains(pc)) { + return CodeHeap::findNMethod(pc) != NULL && + (include_stubs || !JitCodeCache::isCallStub(pc)); + } else { + return Profiler::instance()->libraries()->findLibraryByAddress(pc) != NULL; + } +} + +/** + * Converts a BCI_* frame type value to the corresponding EventType enum value. + * + * This conversion is necessary because Datadog's implementation uses BCI_* values + * (from ASGCT_CallFrameType) directly as event type identifiers, while upstream + * HotspotSupport::walkVM() expects EventType enum values for its logic. + * + * BCI_* values are special frame types with negative values (except BCI_CPU=0) + * that indicate non-standard frame information in call traces. EventType values + * are positive enum indices used for event categorization in the upstream code. + * + * @param bci_type A BCI_* value (e.g., BCI_CPU, BCI_WALL, BCI_ALLOC) + * @return The corresponding EventType enum value + */ +inline EventType eventTypeFromBCI(jint bci_type) { + switch (bci_type) { + case BCI_CPU: + return EXECUTION_SAMPLE; // CPU samples map to execution samples + case BCI_WALL: + return WALL_CLOCK_SAMPLE; + case BCI_ALLOC: + return ALLOC_SAMPLE; + case BCI_ALLOC_OUTSIDE_TLAB: + return ALLOC_OUTSIDE_TLAB; + case BCI_LIVENESS: + return LIVE_OBJECT; + case BCI_LOCK: + return LOCK_SAMPLE; + case BCI_PARK: + return PARK_SAMPLE; + default: + // For unknown or invalid BCI types, default to EXECUTION_SAMPLE + // This maintains backward compatibility and prevents undefined behavior + return EXECUTION_SAMPLE; + } +} + +static void fillFrameTypes(ASGCT_CallFrame *frames, int num_frames, VMNMethod *nmethod) { + if (nmethod->isNMethod() && nmethod->isAlive()) { + VMMethod *method = nmethod->method(); + if (method == NULL) { + return; + } + + jmethodID current_method_id = method->id(); + if (current_method_id == NULL) { + return; + } + + // Mark current_method as COMPILED and frames above current_method as + // INLINED + for (int i = 0; i < num_frames; i++) { + if (frames[i].method_id == NULL || frames[i].bci <= BCI_NATIVE_FRAME) { + break; + } + if (frames[i].method_id == current_method_id) { + int level = nmethod->level(); + frames[i].bci = FrameType::encode( + level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED, + frames[i].bci); + for (int j = 0; j < i; j++) { + frames[j].bci = FrameType::encode(FRAME_INLINED, frames[j].bci); + } + break; + } + } + } else if (nmethod->isInterpreter()) { + // Mark the first Java frame as INTERPRETED + for (int i = 0; i < num_frames; i++) { + if (frames[i].bci > BCI_NATIVE_FRAME) { + frames[i].bci = FrameType::encode(FRAME_INTERPRETED, frames[i].bci); + break; + } + } + } +} + +static ucontext_t empty_ucontext{}; + +__attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, + StackWalkFeatures features, EventType event_type, int lock_index, bool* truncated) { + if (ucontext == NULL) { + return walkVM(&empty_ucontext, frames, max_depth, features, event_type, + callerPC(), (uintptr_t)callerSP(), (uintptr_t)callerFP(), lock_index, truncated); + } else { + StackFrame frame(ucontext); + return walkVM(ucontext, frames, max_depth, features, event_type, + (const void*)frame.pc(), frame.sp(), frame.fp(), lock_index, truncated); + } +} + +__attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, + StackWalkFeatures features, EventType event_type, + const void* pc, uintptr_t sp, uintptr_t fp, int lock_index, bool* truncated) { + // VMStructs is only available for hotspot JVM + assert(VM::isHotspot()); + StackFrame frame(ucontext); + uintptr_t bottom = (uintptr_t)&frame + MAX_WALK_SIZE; + + Profiler* profiler = Profiler::instance(); + int bcp_offset = InterpreterFrame::bcp_offset(); + + jmp_buf crash_protection_ctx; + VMThread* vm_thread = VMThread::current(); + if (vm_thread != NULL && !vm_thread->isThreadAccessible()) { + Counters::increment(WALKVM_THREAD_INACCESSIBLE); + vm_thread = NULL; + } + if (vm_thread == NULL) { + Counters::increment(WALKVM_NO_VMTHREAD); + } else { + Counters::increment(WALKVM_VMTHREAD_OK); + } + void* saved_exception = vm_thread != NULL ? vm_thread->exception() : NULL; + + // Should be preserved across setjmp/longjmp + volatile int depth = 0; + int actual_max_depth = truncated ? max_depth + 1 : max_depth; + bool fp_chain_fallback = false; + int fp_chain_depth = 0; + + ProfiledThread* profiled_thread = ProfiledThread::currentSignalSafe(); + + VMJavaFrameAnchor* anchor = NULL; + if (vm_thread != NULL) { + anchor = vm_thread->anchor(); + if (anchor == NULL) { + Counters::increment(WALKVM_ANCHOR_NULL); + } + vm_thread->exception() = &crash_protection_ctx; + if (profiled_thread != nullptr) { + profiled_thread->setCrashProtectionActive(true); + } + if (setjmp(crash_protection_ctx) != 0) { + if (profiled_thread != nullptr) { + profiled_thread->setCrashProtectionActive(false); + } + vm_thread->exception() = saved_exception; + if (depth < max_depth) { + fillFrame(frames[depth++], BCI_ERROR, "break_not_walkable"); + } + return depth; + } + } + + const void* prev_native_pc = NULL; + + // Saved anchor data — preserved across anchor consumption so inline + // recovery can redirect even after the anchor pointer has been set to NULL. + // Recovery is one-shot: once attempted, we do not retry to avoid + // ping-ponging between CodeHeap and unmapped native regions. + const void* saved_anchor_pc = NULL; + uintptr_t saved_anchor_sp = 0; + uintptr_t saved_anchor_fp = 0; + bool anchor_recovery_used = false; + + // Show extended frame types and stub frames for execution-type events + bool details = event_type <= MALLOC_SAMPLE || features.mixed; + + if (details && vm_thread != NULL && vm_thread->cachedIsJavaThread()) { + anchor = vm_thread->anchor(); + } + + unwind_loop: + + // Walk until the bottom of the stack or until the first Java frame + while (depth < actual_max_depth) { + if (CodeHeap::contains(pc)) { + Counters::increment(WALKVM_HIT_CODEHEAP); + if (fp_chain_fallback) { + Counters::increment(WALKVM_FP_CHAIN_REACHED_CODEHEAP); + fp_chain_fallback = false; + fp_chain_depth = 0; + } + // If we're in JVM-generated code but don't have a VMThread, we cannot safely + // walk the Java stack because crash protection is not set up. + // + // This can occur during JNI attach/detach transitions: when a thread detaches, + // pthread_setspecific() clears the VMThread TLS, but if a profiling signal arrives + // while PC is still in JVM stubs (JavaCalls, method entry/exit), we see CodeHeap + // code without VMThread context. + // + // Without vm_thread, crash protection via setjmp/longjmp cannot work + // (checkFault() needs vm_thread->exception() to longjmp). Any memory dereference in interpreter + // frame handling or NMethod validation would crash the process with unrecoverable SEGV. + // + // The missing VMThread is a timing issue during thread lifecycle. + if (vm_thread == NULL) { + Counters::increment(WALKVM_CODEH_NO_VM); + fillFrame(frames[depth++], BCI_ERROR, "break_no_vmthread"); + break; + } + prev_native_pc = NULL; // we are in JVM code, no previous 'native' PC + VMNMethod* nm = CodeHeap::findNMethod(pc); + if (nm == NULL) { + if (anchor == NULL) { + // Add an error frame only if we cannot recover + fillFrame(frames[depth++], BCI_ERROR, "unknown_nmethod"); + } + break; + } + + // Always prefer JavaFrameAnchor when it is available, + // since it provides reliable SP and FP. + // Do not treat the topmost stub as Java frame. + if (anchor != NULL && (depth > 0 || !nm->isStub())) { + Counters::increment(WALKVM_ANCHOR_CONSUMED); + // Preserve anchor data before consumption — getFrame() is read-only + // but we set anchor=NULL below, losing the pointer for later recovery. + if (saved_anchor_sp == 0) { + saved_anchor_pc = anchor->lastJavaPC(); + saved_anchor_sp = anchor->lastJavaSP(); + saved_anchor_fp = anchor->lastJavaFP(); + } + if (anchor->getFrame(pc, sp, fp) && !nm->contains(pc)) { + anchor = NULL; + continue; // NMethod has changed as a result of correction + } + anchor = NULL; + } + + if (nm->isInterpreter()) { + if (vm_thread != NULL && vm_thread->inDeopt()) { + fillFrame(frames[depth++], BCI_ERROR, "break_deopt"); + break; + } + + bool is_plausible_interpreter_frame = StackWalkValidation::isPlausibleInterpreterFrame(fp, sp, bcp_offset); + if (is_plausible_interpreter_frame) { + VMMethod* method = ((VMMethod**)fp)[InterpreterFrame::method_offset]; + jmethodID method_id = getMethodId(method); + if (method_id != NULL) { + Counters::increment(WALKVM_JAVA_FRAME_OK); + const char* bytecode_start = method->bytecode(); + const char* bcp = ((const char**)fp)[bcp_offset]; + int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; + fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); + + sp = ((uintptr_t*)fp)[InterpreterFrame::sender_sp_offset]; + pc = stripPointer(((void**)fp)[FRAME_PC_SLOT]); + fp = *(uintptr_t*)fp; + continue; + } + } + + if (depth == 0) { + VMMethod* method = (VMMethod*)frame.method(); + jmethodID method_id = getMethodId(method); + if (method_id != NULL) { + Counters::increment(WALKVM_JAVA_FRAME_OK); + fillFrame(frames[depth++], FRAME_INTERPRETED, 0, method_id); + + if (is_plausible_interpreter_frame) { + pc = stripPointer(((void**)fp)[FRAME_PC_SLOT]); + sp = frame.senderSP(); + fp = *(uintptr_t*)fp; + } else { + pc = stripPointer(SafeAccess::load((void**)sp)); + sp = frame.senderSP(); + } + continue; + } + } + + Counters::increment(WALKVM_BREAK_INTERPRETED); + fillFrame(frames[depth++], BCI_ERROR, "break_interpreted"); + break; + } else if (nm->isNMethod()) { + // Check if deoptimization is in progress before walking compiled frames + if (vm_thread != NULL && vm_thread->inDeopt()) { + fillFrame(frames[depth++], BCI_ERROR, "break_deopt_compiled"); + break; + } + + Counters::increment(WALKVM_JAVA_FRAME_OK); + int level = nm->level(); + FrameTypeId type = details && level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; + fillFrame(frames[depth++], type, 0, nm->method()->id()); + + if (nm->isFrameCompleteAt(pc)) { + if (depth == 1 && frame.unwindEpilogue(nm, (uintptr_t&)pc, sp, fp)) { + continue; + } + + int scope_offset = nm->findScopeOffset(pc); + if (scope_offset > 0) { + depth--; + ScopeDesc scope(nm); + do { + scope_offset = scope.decode(scope_offset); + if (details) { + type = scope_offset > 0 ? FRAME_INLINED : + level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; + } + fillFrame(frames[depth++], type, scope.bci(), scope.method()->id()); + } while (scope_offset > 0 && depth < max_depth); + } + + // Handle situations when sp is temporarily changed in the compiled code + frame.adjustSP(nm->entry(), pc, sp); + + // Validate NMethod metadata before using frameSize() + int frame_size = nm->frameSize(); + if (frame_size <= 0 || frame_size > MAX_FRAME_SIZE_WORDS) { + fillFrame(frames[depth++], BCI_ERROR, "break_invalid_framesize"); + break; + } + + sp += frame_size * sizeof(void*); + + // Verify alignment before dereferencing sp as pointer (secondary defense) + if (!aligned(sp)) { + fillFrame(frames[depth++], BCI_ERROR, "break_misaligned_sp"); + break; + } + + fp = ((uintptr_t*)sp)[-FRAME_PC_SLOT - 1]; + pc = ((const void**)sp)[-FRAME_PC_SLOT]; + continue; + } else if (frame.unwindPrologue(nm, (uintptr_t&)pc, sp, fp)) { + continue; + } + + Counters::increment(WALKVM_BREAK_COMPILED); + fillFrame(frames[depth++], BCI_ERROR, "break_compiled"); + break; + } else if (nm->isEntryFrame(pc) && !features.mixed) { + VMJavaFrameAnchor* next_anchor = VMJavaFrameAnchor::fromEntryFrame(fp); + if (next_anchor == NULL) { + fillFrame(frames[depth++], BCI_ERROR, "break_entry_frame"); + break; + } + uintptr_t prev_sp = sp; + if (!next_anchor->getFrame(pc, sp, fp)) { + // End of Java stack + break; + } + if (sp < prev_sp || sp >= bottom || !aligned(sp)) { + fillFrame(frames[depth++], BCI_ERROR, "break_entry_frame"); + break; + } + continue; + } else { + if (features.vtable_target && nm->isVTableStub() && depth == 0) { + uintptr_t receiver = frame.jarg0(); + if (receiver != 0) { + VMSymbol* symbol = VMKlass::fromOop(receiver)->name(); + u32 class_id = profiler->classMap()->lookup(symbol->body(), symbol->length()); + fillFrame(frames[depth++], BCI_ALLOC, class_id); + } + } + + CodeBlob* stub = JitCodeCache::findRuntimeStub(pc); + const void* start = stub != NULL ? stub->_start : nm->code(); + const char* name = stub != NULL ? stub->_name : nm->name(); + + if (details) { + fillFrame(frames[depth++], BCI_NATIVE_FRAME, name); + } + + if (frame.unwindStub((instruction_t*)start, name, (uintptr_t&)pc, sp, fp)) { + continue; + } + + if (depth > 0 && nm->frameSize() > 0) { + Counters::increment(WALKVM_STUB_FRAMESIZE_FALLBACK); + // Validate NMethod metadata before using frameSize() + int frame_size = nm->frameSize(); + if (frame_size <= 0 || frame_size > MAX_FRAME_SIZE_WORDS) { + fillFrame(frames[depth++], BCI_ERROR, "break_invalid_framesize"); + break; + } + + sp += frame_size * sizeof(void*); + + // Verify alignment before dereferencing sp as pointer (secondary defense) + if (!aligned(sp)) { + fillFrame(frames[depth++], BCI_ERROR, "break_misaligned_sp"); + break; + } + + fp = ((uintptr_t*)sp)[-FRAME_PC_SLOT - 1]; + pc = ((const void**)sp)[-FRAME_PC_SLOT]; + continue; + } + } + } else { + // Resolve native frame (may use remote symbolication if enabled) + Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index); + if (resolution.is_marked) { + // This is a marked C++ interpreter frame, terminate scan + break; + } + const char* method_name = resolution.method_name; + int frame_bci = resolution.bci; + char mark; + if (frame_bci != BCI_NATIVE_FRAME_REMOTE && method_name != NULL && (mark = NativeFunc::read_mark(method_name)) != 0) { + if (mark == MARK_ASYNC_PROFILER && event_type == MALLOC_SAMPLE) { + // Skip all internal frames above malloc_hook functions, leave the hook itself + depth = 0; + } else if (mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) { + // Insert current compile task as a pseudo Java frame + VMMethod* method = vm_thread->compiledMethod(); + jmethodID method_id = method != NULL ? method->id() : NULL; + if (method_id != NULL) { + fillFrame(frames[depth++], FRAME_JIT_COMPILED, 0, method_id); + } + } else if (mark == MARK_THREAD_ENTRY) { + // Thread entry point detected via pre-computed mark - this is the root frame + // No need for expensive symbol resolution, just stop unwinding + Counters::increment(THREAD_ENTRY_MARK_DETECTIONS); + break; + } + } else if (method_name == NULL && details && !anchor_recovery_used + && profiler->findLibraryByAddress(pc) == NULL) { + // Try anchor recovery — prefer live anchor, fall back to saved data + anchor_recovery_used = true; + const void* recovery_pc = NULL; + uintptr_t recovery_sp = 0; + uintptr_t recovery_fp = 0; + bool have_anchor_data = false; + + if (anchor) { + Counters::increment(WALKVM_ANCHOR_USED_INLINE); + recovery_fp = anchor->lastJavaFP(); + recovery_sp = anchor->lastJavaSP(); + recovery_pc = anchor->lastJavaPC(); + have_anchor_data = true; + } else if (saved_anchor_sp != 0) { + Counters::increment(WALKVM_SAVED_ANCHOR_USED); + recovery_fp = saved_anchor_fp; + recovery_sp = saved_anchor_sp; + recovery_pc = saved_anchor_pc; + have_anchor_data = true; + // Clear saved data after use — one-shot recovery + saved_anchor_sp = 0; + } else { + Counters::increment(WALKVM_ANCHOR_INLINE_NO_ANCHOR); + } + + if (have_anchor_data) { + // Try to read the Java method directly from the anchor's FP, + // treating it as an interpreter frame. + // In HotSpot, lastJavaFP is non-zero only for interpreter frames; + // compiled frames record FP=0 in the anchor. + if (StackWalkValidation::isPlausibleInterpreterFrame(recovery_fp, recovery_sp, bcp_offset)) { + VMMethod* method = ((VMMethod**)recovery_fp)[InterpreterFrame::method_offset]; + jmethodID method_id = getMethodId(method); + if (method_id != NULL) { + anchor = NULL; + prev_native_pc = NULL; + if (depth > 0 && depth + 1 < actual_max_depth) { + fillFrame(frames[depth++], BCI_ERROR, "[skipped frames]"); + } + Counters::increment(WALKVM_JAVA_FRAME_OK); + const char* bytecode_start = method->bytecode(); + const char* bcp = ((const char**)recovery_fp)[bcp_offset]; + int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; + fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); + + sp = ((uintptr_t*)recovery_fp)[InterpreterFrame::sender_sp_offset]; + pc = stripPointer(((void**)recovery_fp)[FRAME_PC_SLOT]); + fp = *(uintptr_t*)recovery_fp; + continue; + } + } + + // Fallback: redirect via recovery SP/FP/PC + sp = recovery_sp; + fp = recovery_fp; + pc = recovery_pc; + if (pc != NULL && !CodeHeap::contains(pc) && sp != 0 && aligned(sp) && sp < bottom) { + pc = ((const void**)sp)[-1]; + } + if (sp != 0 && pc != NULL) { + anchor = NULL; + if (sp >= bottom || !aligned(sp)) { + Counters::increment(WALKVM_ANCHOR_INLINE_BAD_SP); + fillFrame(frames[depth++], BCI_ERROR, "break_no_anchor"); + break; + } + prev_native_pc = NULL; + if (depth > 0) { + fillFrame(frames[depth++], BCI_ERROR, "[skipped frames]"); + } + continue; + } + Counters::increment(WALKVM_ANCHOR_INLINE_NO_SP); + } + // Check previous frame for thread entry points (Rust, libc/pthread) + // Only check marks for traditionally-resolved frames; packed remote + // frames store an integer in the method_name union, not a valid pointer. + if (prev_native_pc != NULL) { + Profiler::NativeFrameResolution prev_resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)prev_native_pc, lock_index); + if (prev_resolution.bci != BCI_NATIVE_FRAME_REMOTE) { + const char* prev_method_name = prev_resolution.method_name; + if (prev_method_name != NULL) { + char prev_mark = NativeFunc::read_mark(prev_method_name); + if (prev_mark == MARK_THREAD_ENTRY) { + Counters::increment(THREAD_ENTRY_MARK_DETECTIONS); + break; + } + } + } + } + // Fall through to DWARF section — when findLibraryByAddress(pc) + // returns NULL, default_frame uses FP-chain walking (DW_REG_FP) + // which can bridge symbol-less gaps in libjvm.so. + Counters::increment(WALKVM_FP_CHAIN_ATTEMPT); + fp_chain_fallback = true; + if (++fp_chain_depth > actual_max_depth) { + break; + } + goto dwarf_unwind; + } + fillFrame(frames[depth++], frame_bci, (void*)method_name); + } + + dwarf_unwind: + uintptr_t prev_sp = sp; + CodeCache* cc = profiler->findLibraryByAddress(pc); + FrameDesc f = cc != NULL ? cc->findFrameDesc(pc) : FrameDesc::fallback_default_frame(); + + u8 cfa_reg = (u8)f.cfa; + int cfa_off = f.cfa >> 8; + + // If DWARF is invalid, we cannot continue unwinding reliably + // Thread entry points are detected earlier via MARK_THREAD_ENTRY + if (cfa_reg == DW_REG_INVALID || cfa_reg > DW_REG_PLT) { + break; + } + + if (cfa_reg == DW_REG_SP) { + sp = sp + cfa_off; + } else if (cfa_reg == DW_REG_FP) { + // Sanity-check FP before deriving CFA from it. A corrupted FP can produce a + // phantom CFA and cause the walk to record spurious frames before breaking. + // We cannot check fp < sp here because on aarch64 the frame pointer is set + // to SP at function entry, which is typically less than the previous CFA. + if (fp >= bottom || !aligned(fp)) { + break; + } + sp = fp + cfa_off; + } else if (cfa_reg == DW_REG_PLT) { + sp += ((uintptr_t)pc & 15) >= 11 ? cfa_off * 2 : cfa_off; + } + + // Check if the next frame is below on the current stack + if (sp < prev_sp || sp >= prev_sp + MAX_FRAME_SIZE || sp >= bottom) { + break; + } + + // Stack pointer must be word aligned + if (!aligned(sp)) { + break; + } + + // store the previous pc before unwinding + prev_native_pc = pc; + if (f.fp_off & DW_PC_OFFSET) { + pc = (const char*)pc + (f.fp_off >> 1); + } else { + if (f.fp_off != DW_SAME_FP && f.fp_off < MAX_FRAME_SIZE && f.fp_off > -MAX_FRAME_SIZE) { + fp = (uintptr_t)SafeAccess::load((void**)(sp + f.fp_off)); + } + + if (EMPTY_FRAME_SIZE > 0 || f.pc_off != DW_LINK_REGISTER) { + // Verify alignment before dereferencing sp + offset + uintptr_t pc_addr = sp + f.pc_off; + if (!aligned(pc_addr)) { + break; + } + pc = stripPointer(SafeAccess::load((void**)pc_addr)); + } else if (depth == 1) { + pc = (const void*)frame.link(); + } else { + break; + } + + if (EMPTY_FRAME_SIZE == 0 && cfa_off == 0 && f.fp_off != DW_SAME_FP) { + // AArch64 default_frame + sp = defaultSenderSP(sp, fp); + if (sp < prev_sp || sp >= bottom || !aligned(sp)) { + break; + } + } + } + + if (inDeadZone(pc) || (pc == prev_native_pc && sp == prev_sp)) { + break; + } + } + + // If we did not meet Java frame but current thread has JavaFrameAnchor set, + // try to read the interpreter frame directly from the anchor's FP. + // In HotSpot, lastJavaFP != 0 reliably indicates an interpreter frame. + if (anchor != NULL) { + uintptr_t anchor_fp = anchor->lastJavaFP(); + uintptr_t anchor_sp = anchor->lastJavaSP(); + if (anchor_sp == 0) { + Counters::increment(WALKVM_ANCHOR_NOT_IN_JAVA); + goto done; + } + if (StackWalkValidation::isPlausibleInterpreterFrame(anchor_fp, anchor_sp, bcp_offset)) { + VMMethod* method = ((VMMethod**)anchor_fp)[InterpreterFrame::method_offset]; + jmethodID method_id = getMethodId(method); + if (method_id != NULL) { + Counters::increment(WALKVM_ANCHOR_FALLBACK); + Counters::increment(WALKVM_JAVA_FRAME_OK); + anchor = NULL; + while (depth > 0 && frames[depth - 1].method_id == NULL) depth--; + if (depth < actual_max_depth) { + const char* bytecode_start = method->bytecode(); + const char* bcp = ((const char**)anchor_fp)[bcp_offset]; + int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; + fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); + sp = ((uintptr_t*)anchor_fp)[InterpreterFrame::sender_sp_offset]; + pc = stripPointer(((void**)anchor_fp)[FRAME_PC_SLOT]); + fp = *(uintptr_t*)anchor_fp; + if (sp != 0 && sp < bottom && aligned(sp)) { + goto unwind_loop; + } + } + } + } + // Fallback: redirect via anchor frame and sp[-1] + if (anchor != NULL && anchor->getFrame(pc, sp, fp)) { + if (!CodeHeap::contains(pc) && sp != 0 && aligned(sp) && sp < bottom) { + pc = ((const void**)sp)[-1]; + } + Counters::increment(WALKVM_ANCHOR_FALLBACK); + anchor = NULL; + while (depth > 0 && frames[depth - 1].method_id == NULL) depth--; + if (sp != 0 && sp < bottom && aligned(sp)) { + goto unwind_loop; + } + } else if (anchor != NULL) { + Counters::increment(WALKVM_ANCHOR_FALLBACK_FAIL); + } + } + + done: + if (profiled_thread != nullptr) { + profiled_thread->setCrashProtectionActive(false); + } + if (vm_thread != NULL) { + vm_thread->exception() = saved_exception; + } + + // Drop unknown leaf frame - it provides no useful information and breaks + // aggregation by lumping unrelated samples under a single "unknown" entry + depth = StackWalkValidation::dropUnknownLeaf(frames, depth); + + if (depth == 0) { + Counters::increment(WALKVM_DEPTH_ZERO); + } + + if (truncated) { + if (depth > max_depth) { + *truncated = true; + depth = max_depth; + } else if (depth > 0) { + if (frames[depth - 1].bci == BCI_ERROR) { + // root frame is error; best guess is that the trace is truncated + *truncated = true; + } + } + } + + return depth; +} + +void HotspotSupport::checkFault(ProfiledThread* thrd) { + if (!JVMThread::isInitialized()) { + // JVM has not been loaded or has not been initialized yet + return; + } + + VMThread* vm_thread = VMThread::current(); + if (vm_thread == NULL || !vm_thread->isThreadAccessible()) { + return; + } + + // Prefer the semantic crash protection flag (reliable regardless of stack frame sizes). + // Fall back to sameStack heuristic when ProfiledThread TLS is unavailable (e.g. during + // early init or in crash recovery tests). sameStack uses a fixed 8KB threshold which + // can fail with ASAN-inflated frames, but the crashProtectionActive path handles that. + bool protected_walk = (thrd != nullptr && thrd->isCrashProtectionActive()) + || sameStack(vm_thread->exception(), &vm_thread); + if (!protected_walk) { + return; + } + + if (thrd != nullptr) { + thrd->resetCrashHandler(); + } + longjmp(*(jmp_buf*)vm_thread->exception(), 1); +} + + +int HotspotSupport::getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, + int max_depth, StackContext *java_ctx, + bool *truncated) { + // Workaround for JDK-8132510: it's not safe to call GetEnv() inside a signal + // handler since JDK 9, so we do it only for threads already registered in + // ThreadLocalStorage + VMThread *vm_thread = VMThread::current(); + if (vm_thread == NULL || !vm_thread->isThreadAccessible()) { + Counters::increment(AGCT_NOT_REGISTERED_IN_TLS); + return 0; + } + + JNIEnv *jni = VM::jni(); + if (jni == NULL) { + // Not a Java thread + Counters::increment(AGCT_NOT_JAVA); + return 0; + } + + StackFrame frame(ucontext); + uintptr_t saved_pc, saved_sp, saved_fp; + if (ucontext != NULL) { + saved_pc = frame.pc(); + saved_sp = frame.sp(); + saved_fp = frame.fp(); + + if (JitCodeCache::isCallStub((const void *)saved_pc)) { + // call_stub is unsafe to walk + frames->bci = BCI_ERROR; + frames->method_id = (jmethodID) "call_stub"; + return 1; + } + + if (!VMStructs::isSafeToWalk(saved_pc)) { + frames->bci = BCI_NATIVE_FRAME; + CodeBlob *codeBlob = + VMStructs::libjvm()->findBlobByAddress((const void *)saved_pc); + if (codeBlob) { + frames->method_id = (jmethodID)codeBlob->_name; + } else { + frames->method_id = (jmethodID) "unknown_unwalkable"; + } + return 1; + } + } else { + return 0; + } + + int state = vm_thread->state(); + /** from OpenJDK + https://github.com/openjdk/jdk/blob/7455bb23c1d18224e48e91aae4f11fe114d04fab/src/hotspot/share/utilities/globalDefinitions.hpp#L1030 + enum JavaThreadState { + _thread_uninitialized = 0, // should never happen (missing initialization) + _thread_new = 2, // just starting up, i.e., in process of being initialized + _thread_new_trans = 3, // corresponding transition state (not used, included for completeness) + _thread_in_native = 4, // running in native code + _thread_in_native_trans = 5, // corresponding transition state + _thread_in_vm = 6, // running in VM + _thread_in_vm_trans = 7, // corresponding transition state + _thread_in_Java = 8, // running in Java or in stub code + _thread_in_Java_trans = 9, // corresponding transition state (not used, included for completeness) + _thread_blocked = 10, // blocked in vm + _thread_blocked_trans = 11, // corresponding transition state + _thread_max_state = 12 // maximum thread state+1 - used forstatistics allocation + }; + **/ + + bool in_java = (state == 8 || state == 9); + if (in_java && java_ctx->sp != 0) { + // skip ahead to the Java frames before calling AGCT + frame.restore((uintptr_t)java_ctx->pc, java_ctx->sp, java_ctx->fp); + } else if (state != 0) { + VMJavaFrameAnchor* a = vm_thread->anchor(); + if (a == nullptr || a->lastJavaSP() == 0) { + // we haven't found the top Java frame ourselves, and the lastJavaSP wasn't + // recorded either when not in the Java state, lastJava ucontext will be + // used by AGCT + Counters::increment(AGCT_NATIVE_NO_JAVA_CONTEXT); + return 0; + } + } + bool blocked_in_vm = (state == 10 || state == 11); + // avoid unwinding during deoptimization + if (blocked_in_vm && vm_thread->osThreadState() == OSThreadState::RUNNABLE) { + Counters::increment(AGCT_BLOCKED_IN_VM); + return 0; + } + + JitWriteProtection jit(false); + // AsyncGetCallTrace writes to ASGCT_CallFrame array + ASGCT_CallTrace trace = {jni, 0, frames}; + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + + if (trace.num_frames > 0) { + frame.restore(saved_pc, saved_sp, saved_fp); + return trace.num_frames; + } + + int safe_mode = Profiler::instance()->safe_mode(); + CStack cstack = Profiler::instance()->cstackMode(); + + if ((trace.num_frames == ticks_unknown_Java || + trace.num_frames == ticks_not_walkable_Java) && + !(safe_mode & UNKNOWN_JAVA) && ucontext != NULL) { + CodeBlob *stub = JitCodeCache::findRuntimeStub((const void *)frame.pc()); + if (stub != NULL) { + if (cstack != CSTACK_NO) { + max_depth -= makeFrame(trace.frames++, BCI_NATIVE_FRAME, stub->_name); + } + if (!(safe_mode & POP_STUB) && + frame.unwindStub((instruction_t *)stub->_start, stub->_name) && + isAddressInCode((const void *)frame.pc())) { + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } + } else if (VMStructs::hasMethodStructs()) { + VMNMethod *nmethod = CodeHeap::findNMethod((const void *)frame.pc()); + if (nmethod != NULL && nmethod->isNMethod() && nmethod->isAlive()) { + VMMethod *method = nmethod->method(); + if (method != NULL) { + jmethodID method_id = method->id(); + if (method_id != NULL) { + max_depth -= makeFrame(trace.frames++, 0, method_id); + } + if (!(safe_mode & POP_METHOD) && frame.unwindCompiled(nmethod) && + isAddressInCode((const void *)frame.pc())) { + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } + if ((safe_mode & PROBE_SP) && trace.num_frames < 0) { + if (method_id != NULL) { + trace.frames--; + } + for (int i = 0; trace.num_frames < 0 && i < PROBE_SP_LIMIT; i++) { + frame.sp() += sizeof(void*); + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } + } + } + } else if (nmethod != NULL) { + if (cstack != CSTACK_NO) { + max_depth -= + makeFrame(trace.frames++, BCI_NATIVE_FRAME, nmethod->name()); + } + if (!(safe_mode & POP_STUB) && + frame.unwindStub(NULL, nmethod->name()) && + isAddressInCode((const void *)frame.pc())) { + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } + } + } + } else if (trace.num_frames == ticks_unknown_not_Java && + !(safe_mode & LAST_JAVA_PC)) { + VMJavaFrameAnchor* anchor = vm_thread->anchor(); + if (anchor == NULL) return 0; + uintptr_t sp = anchor->lastJavaSP(); + const void* pc = anchor->lastJavaPC(); + if (sp != 0 && pc == NULL) { + // We have the last Java frame anchor, but it is not marked as walkable. + // Make it walkable here + pc = ((const void**)sp)[-1]; + anchor->setLastJavaPC(pc); + + VMNMethod *m = CodeHeap::findNMethod(pc); + const Libraries* libs = Profiler::instance()->libraries(); + + if (m != NULL) { + // AGCT fails if the last Java frame is a Runtime Stub with an invalid + // _frame_complete_offset. In this case we patch _frame_complete_offset + // manually + if (!m->isNMethod() && m->frameSize() > 0 && + m->frameCompleteOffset() == -1) { + m->setFrameCompleteOffset(0); + } + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } else if (libs->findLibraryByAddress(pc) != NULL) { + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } + + anchor->setLastJavaPC(nullptr); + } + } else if (trace.num_frames == ticks_not_walkable_not_Java && + !(safe_mode & LAST_JAVA_PC)) { + VMJavaFrameAnchor* anchor = vm_thread->anchor(); + if (anchor == NULL) return 0; + uintptr_t sp = anchor->lastJavaSP(); + const void* pc = anchor->lastJavaPC(); + if (sp != 0 && pc != NULL) { + // Similar to the above: last Java frame is set, + // but points to a Runtime Stub with an invalid _frame_complete_offset + VMNMethod *m = CodeHeap::findNMethod(pc); + if (m != NULL && !m->isNMethod() && m->frameSize() > 0 && + m->frameCompleteOffset() == -1) { + m->setFrameCompleteOffset(0); + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + } + } + } else if (trace.num_frames == ticks_GC_active && !(safe_mode & GC_TRACES)) { + VMJavaFrameAnchor* anchor = vm_thread->anchor(); + if (anchor == NULL || anchor->lastJavaSP() == 0) { + // Do not add 'GC_active' for threads with no Java frames, e.g. Compiler + // threads + frame.restore(saved_pc, saved_sp, saved_fp); + return 0; + } + } + + frame.restore(saved_pc, saved_sp, saved_fp); + + if (trace.num_frames > 0) { + return trace.num_frames + (trace.frames - frames); + } + + const char *err_string = Profiler::asgctError(trace.num_frames); + if (err_string == NULL) { + // No Java stack, because thread is not in Java context + return 0; + } + + Profiler::instance()->incFailure(-trace.num_frames); + trace.frames->bci = BCI_ERROR; + trace.frames->method_id = (jmethodID)err_string; + return trace.frames - frames + 1; +} + + +int HotspotSupport::walkJavaStack(StackWalkRequest& request) { + CStack cstack = Profiler::instance()->cstackMode(); + StackWalkFeatures features = Profiler::instance()->stackWalkFeatures(); + void* ucontext = request.ucontext; + ASGCT_CallFrame* frames = request.frames; + int max_depth = request.max_depth; + StackContext* java_ctx = request.java_ctx; + bool* truncated = request.truncated; + u32 lock_index = request.lock_index; + + int java_frames = 0; + if (features.mixed) { + java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); + } else if (request.event_type == BCI_CPU || request.event_type == BCI_WALL) { + if (cstack >= CSTACK_VM) { + java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); + } else { + // Async events + AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); + if (mutex.acquired()) { + java_frames = getJavaTraceAsync(ucontext, frames, max_depth, java_ctx, truncated); + if (java_frames > 0 && java_ctx->pc != NULL && VMStructs::hasMethodStructs()) { + VMNMethod* nmethod = CodeHeap::findNMethod(java_ctx->pc); + if (nmethod != NULL) { + fillFrameTypes(frames, java_frames, nmethod); + } + } + } + } + } + return java_frames; +} diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.h b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.h new file mode 100644 index 000000000..591d61dcb --- /dev/null +++ b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.h @@ -0,0 +1,32 @@ +/* + * Copyright The async-profiler authors + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _HOTSPOT_HOTSPOTSUPPORT_H +#define _HOTSPOT_HOTSPOTSUPPORT_H + +#include "stackWalker.h" + +class ProfiledThread; + +class HotspotSupport { +private: + static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, + StackWalkFeatures features, EventType event_type, + const void* pc, uintptr_t sp, uintptr_t fp, int lock_index, bool* truncated); + static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, + StackWalkFeatures features, EventType event_type, + int lock_index, bool* truncated = nullptr); + + static int getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, + int max_depth, StackContext *java_ctx, + bool *truncated); + +public: + static void checkFault(ProfiledThread* thrd = nullptr); + static int walkJavaStack(StackWalkRequest& request); +}; + +#endif // _HOTSPOT_HOTSPOTSUPPORT_H diff --git a/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.cpp b/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.cpp new file mode 100644 index 000000000..85af3d19f --- /dev/null +++ b/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.cpp @@ -0,0 +1,51 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jitCodeCache.h" + +#include "hotspot/vmStructs.h" + +SpinLock JitCodeCache::_stubs_lock; +CodeCache JitCodeCache::_runtime_stubs("[stubs]"); +std::atomic JitCodeCache::_call_stub_begin = { nullptr }; +std::atomic JitCodeCache::_call_stub_end = { nullptr }; + +// CompiledMethodLoad is also needed to enable DebugNonSafepoints info by +// default +void JNICALL JitCodeCache::CompiledMethodLoad(jvmtiEnv *jvmti, jmethodID method, + jint code_size, const void *code_addr, + jint map_length, + const jvmtiAddrLocationMap *map, + const void *compile_info) { + CodeHeap::updateBounds(code_addr, (const char *)code_addr + code_size); +} + +void JNICALL JitCodeCache::DynamicCodeGenerated(jvmtiEnv *jvmti, const char *name, + const void *address, jint length) { + _stubs_lock.lock(); + _runtime_stubs.add(address, length, name, true); + _stubs_lock.unlock(); + + if (name[0] == 'I' && strcmp(name, "Interpreter") == 0) { + CodeHeap::setInterpreterStart(address); + } else if (strcmp(name, "call_stub") == 0) { + _call_stub_begin.store(address, std::memory_order_relaxed); + // This fence ensures that _call_stub_begin is visible before _call_stub_end, so that isCallStub() works correctly + std::atomic_thread_fence(std::memory_order_release); + _call_stub_end.store((const char *)address + length, std::memory_order_relaxed); + } + + CodeHeap::updateBounds(address, (const char *)address + length); +} + +CodeBlob* JitCodeCache::findRuntimeStub(const void *address) { + CodeBlob *stub = nullptr; + _stubs_lock.lockShared(); + if (_runtime_stubs.contains(address)) { + stub = _runtime_stubs.findBlobByAddress(address); + } + _stubs_lock.unlockShared(); + return stub; +} diff --git a/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.h b/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.h new file mode 100644 index 000000000..64f5f5792 --- /dev/null +++ b/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.h @@ -0,0 +1,41 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _HOTSPOT_JITCODECACHE_H +#define _HOTSPOT_JITCODECACHE_H + +#include + +#include "codeCache.h" +#include "spinLock.h" + +// The class tracks JIT-compiled code and ranges +class JitCodeCache { +private: + static SpinLock _stubs_lock; + static CodeCache _runtime_stubs; + static std::atomic _call_stub_begin; + static std::atomic _call_stub_end; + +public: + static void JNICALL CompiledMethodLoad(jvmtiEnv *jvmti, jmethodID method, + jint code_size, const void *code_addr, + jint map_length, + const jvmtiAddrLocationMap *map, + const void *compile_info); + static void JNICALL DynamicCodeGenerated(jvmtiEnv *jvmti, const char *name, + const void *address, jint length); + + static inline bool isCallStub(const void *address) { + const void* stub_end = _call_stub_end.load(std::memory_order_acquire); + return stub_end != nullptr && + address >= _call_stub_begin.load(std::memory_order_relaxed) && + address < stub_end; + } + + static CodeBlob* findRuntimeStub(const void *address); +}; + +#endif // _HOTSPOT_JITCODECACHE_H diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.h b/ddprof-lib/src/main/cpp/hotspot/vmStructs.h index 418907a98..7356827c9 100644 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.h +++ b/ddprof-lib/src/main/cpp/hotspot/vmStructs.h @@ -41,6 +41,7 @@ inline bool crashProtectionActive(); template inline T* cast_to(const void* ptr) { + assert(VM::isHotspot()); // This should only be used in HotSpot-specific code assert(T::type_size() > 0); // Ensure type size has been initialized assert(crashProtectionActive() || ptr == nullptr || SafeAccess::isReadableRange(ptr, T::type_size())); return reinterpret_cast(const_cast(ptr)); diff --git a/ddprof-lib/src/main/cpp/j9/j9Ext.cpp b/ddprof-lib/src/main/cpp/j9/j9Support.cpp similarity index 78% rename from ddprof-lib/src/main/cpp/j9/j9Ext.cpp rename to ddprof-lib/src/main/cpp/j9/j9Support.cpp index 6ae57cf40..87afb3ce2 100644 --- a/ddprof-lib/src/main/cpp/j9/j9Ext.cpp +++ b/ddprof-lib/src/main/cpp/j9/j9Support.cpp @@ -15,23 +15,23 @@ * limitations under the License. */ -#include "j9Ext.h" +#include "j9/j9Support.h" #include "os.h" #include -jvmtiEnv *J9Ext::_jvmti; +jvmtiEnv *J9Support::_jvmti; -void *(*J9Ext::_j9thread_self)() = NULL; +void *(*J9Support::_j9thread_self)() = NULL; -jvmtiExtensionFunction J9Ext::_GetOSThreadID = NULL; -jvmtiExtensionFunction J9Ext::_GetJ9vmThread = NULL; -jvmtiExtensionFunction J9Ext::_GetStackTraceExtended = NULL; -jvmtiExtensionFunction J9Ext::_GetAllStackTracesExtended = NULL; +jvmtiExtensionFunction J9Support::_GetOSThreadID = NULL; +jvmtiExtensionFunction J9Support::_GetJ9vmThread = NULL; +jvmtiExtensionFunction J9Support::_GetStackTraceExtended = NULL; +jvmtiExtensionFunction J9Support::_GetAllStackTracesExtended = NULL; -int J9Ext::InstrumentableObjectAlloc_id = -1; +int J9Support::InstrumentableObjectAlloc_id = -1; // Look for OpenJ9-specific JVM TI extension -bool J9Ext::initialize(jvmtiEnv *jvmti, const void *j9thread_self) { +bool J9Support::initialize(jvmtiEnv *jvmti, const void *j9thread_self) { _jvmti = jvmti; _j9thread_self = (void *(*)())j9thread_self; diff --git a/ddprof-lib/src/main/cpp/j9/j9Ext.h b/ddprof-lib/src/main/cpp/j9/j9Support.h similarity index 98% rename from ddprof-lib/src/main/cpp/j9/j9Ext.h rename to ddprof-lib/src/main/cpp/j9/j9Support.h index 0618bd676..79bf3e534 100644 --- a/ddprof-lib/src/main/cpp/j9/j9Ext.h +++ b/ddprof-lib/src/main/cpp/j9/j9Support.h @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef _J9_J9EXT_H -#define _J9_J9EXT_H +#ifndef _J9_J9SUPPORT_H +#define _J9_J9SUPPORT_H #include @@ -70,7 +70,7 @@ static inline int sanitizeJ9FrameType(jint j9_type) { return FRAME_JIT_COMPILED; } -class J9Ext { +class J9Support { friend class JVMThread; friend class J9WallClock; private: @@ -181,4 +181,4 @@ class J9Ext { } }; -#endif // _J9_J9EXT_H +#endif // _J9_J9SUPPORT_H diff --git a/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp b/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp index 413d10892..a094cd663 100644 --- a/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp +++ b/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp @@ -16,7 +16,7 @@ */ #include "j9WallClock.h" -#include "j9Ext.h" +#include "j9/j9Support.h" #include "profiler.h" #include "threadState.h" #include @@ -74,7 +74,7 @@ void J9WallClock::timerLoop() { jvmtiStackInfoExtended *stack_infos; jint thread_count; - if (J9Ext::GetAllStackTracesExtended( + if (J9Support::GetAllStackTracesExtended( _max_stack_depth, (void **)&stack_infos, &thread_count) == 0) { for (int i = 0; i < thread_count; i++) { jvmtiStackInfoExtended *si = &stack_infos[i]; @@ -95,7 +95,7 @@ void J9WallClock::timerLoop() { frames[j].bci = FrameType::encode(sanitizeJ9FrameType(fi->type), fi->location); } - int tid = J9Ext::GetOSThreadID(si->thread); + int tid = J9Support::GetOSThreadID(si->thread); if (tid == -1) { // clearly an invalid TID; skip the thread continue; diff --git a/ddprof-lib/src/main/cpp/jvm.cpp b/ddprof-lib/src/main/cpp/jvm.cpp deleted file mode 100644 index 34412e52a..000000000 --- a/ddprof-lib/src/main/cpp/jvm.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - - -#include "jvm.h" -#include "log.h" -#include "hotspot/vmStructs.h" - -bool JVM::_is_readable_pointer_resolved = false; -is_readable_pointer_fn JVM::_is_readable_pointer = NULL; - -bool JVM::is_readable_pointer(void *ptr) { - if (!_is_readable_pointer_resolved) { - const char *sym_name = "_ZN2os19is_readable_pointerEPKv"; - _is_readable_pointer_resolved = true; - _is_readable_pointer = - (is_readable_pointer_fn)VMStructs::libjvm()->findSymbol(sym_name); - if (_is_readable_pointer == NULL) { - Log::error("Can not resolve symbol %s in JVM lib\n", sym_name); - } - } - // if we have no access to the symbol we can not really attempt the pointer - // sanitation - return _is_readable_pointer ? _is_readable_pointer(ptr) : true; -} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/jvm.h b/ddprof-lib/src/main/cpp/jvm.h deleted file mode 100644 index d3d1fbe47..000000000 --- a/ddprof-lib/src/main/cpp/jvm.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _JVM_H -#define _JVM_H - -typedef bool (*is_readable_pointer_fn)(void *); - -class JVM { -private: - static bool _is_readable_pointer_resolved; - static is_readable_pointer_fn _is_readable_pointer; - -public: - static bool is_readable_pointer(void *ptr); -}; -#endif // _JVM_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/jvmSupport.cpp b/ddprof-lib/src/main/cpp/jvmSupport.cpp new file mode 100644 index 000000000..2e9b71d6d --- /dev/null +++ b/ddprof-lib/src/main/cpp/jvmSupport.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jvmSupport.h" + +#include "asyncSampleMutex.h" +#include "frames.h" +#include "os.h" +#include "profiler.h" +#include "thread.h" + +#include "hotspot/hotspotSupport.h" + +#include + +int JVMSupport::walkJavaStack(StackWalkRequest& request) { + if (VM::isHotspot()) { + return HotspotSupport::walkJavaStack(request); + } else if (VM::isOpenJ9() || VM::isZing()) { + assert(request.event_type == BCI_CPU || request.event_type == BCI_WALL); + return asyncGetCallTrace(request.frames, request.max_depth, request.ucontext); + } + assert(false && "Unsupported JVM"); + return 0; +} + +int JVMSupport::asyncGetCallTrace(ASGCT_CallFrame *frames, int max_depth, void* ucontext) { + JNIEnv *jni = VM::jni(); + if (jni == nullptr) { + return 0; + } + + AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); + if (!mutex.acquired()) { + return 0; + } + + JitWriteProtection jit(false); + // AsyncGetCallTrace writes to ASGCT_CallFrame array + ASGCT_CallTrace trace = {jni, 0, frames}; + VM::_asyncGetCallTrace(&trace, max_depth, ucontext); + if (trace.num_frames > 0) { + return trace.num_frames; + } + + const char* err_string = Profiler::asgctError(trace.num_frames); + if (err_string == NULL) { + // No Java stack, because thread is not in Java context + return 0; + } + + Profiler::instance()->incFailure(-trace.num_frames); + return makeFrame(frames, BCI_ERROR, err_string); +} diff --git a/ddprof-lib/src/main/cpp/jvmSupport.h b/ddprof-lib/src/main/cpp/jvmSupport.h new file mode 100644 index 000000000..9b8a65dc2 --- /dev/null +++ b/ddprof-lib/src/main/cpp/jvmSupport.h @@ -0,0 +1,30 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _JVMSUPPORT_H +#define _JVMSUPPORT_H + +#include "stackWalker.h" +#include "vmEntry.h" + +// Stack recovery techniques used to workaround AsyncGetCallTrace flaws. +// Can be disabled with 'safemode' option. +enum StackRecovery { + UNKNOWN_JAVA = (1 << 0), + POP_STUB = (1 << 1), + POP_METHOD = (1 << 2), + LAST_JAVA_PC = (1 << 4), + GC_TRACES = (1 << 5), + PROBE_SP = 0x100, +}; + + +class JVMSupport { + static int asyncGetCallTrace(ASGCT_CallFrame *frames, int max_depth, void* ucontext); +public: + static int walkJavaStack(StackWalkRequest& request); +}; + +#endif // _JVMSUPPORT_H diff --git a/ddprof-lib/src/main/cpp/jvmThread.cpp b/ddprof-lib/src/main/cpp/jvmThread.cpp index 7fdddf4b4..782ed79e6 100644 --- a/ddprof-lib/src/main/cpp/jvmThread.cpp +++ b/ddprof-lib/src/main/cpp/jvmThread.cpp @@ -5,7 +5,7 @@ #include "jvmThread.h" #include "hotspot/vmStructs.inline.h" -#include "j9/j9Ext.h" +#include "j9/j9Support.h" #include "zing/zingSupport.h" #include "vmEntry.h" @@ -30,7 +30,7 @@ bool JVMThread::initialize() { } int JVMThread::nativeThreadId(JNIEnv* jni, jthread thread) { - return VM::isOpenJ9() ? J9Ext::GetOSThreadID(thread) : VMThread::nativeThreadId(jni, thread); + return VM::isOpenJ9() ? J9Support::GetOSThreadID(thread) : VMThread::nativeThreadId(jni, thread); } void* JVMThread::currentThreadSlow() { @@ -47,7 +47,7 @@ void* JVMThread::currentThreadSlow() { } if (VM::isOpenJ9()) { - return J9Ext::j9thread_self(); + return J9Support::j9thread_self(); } else if (VM::isZing()) { return ZingSupport::initialize(thread); } else { diff --git a/ddprof-lib/src/main/cpp/libraries.cpp b/ddprof-lib/src/main/cpp/libraries.cpp index a0322665f..39919d1ce 100644 --- a/ddprof-lib/src/main/cpp/libraries.cpp +++ b/ddprof-lib/src/main/cpp/libraries.cpp @@ -99,7 +99,7 @@ CodeCache *Libraries::findLibraryByName(const char *lib_name) { return NULL; } -CodeCache *Libraries::findLibraryByAddress(const void *address) { +CodeCache *Libraries::findLibraryByAddress(const void *address) const { const int native_lib_count = _native_libs.count(); for (int i = 0; i < native_lib_count; i++) { CodeCache *lib = _native_libs[i]; diff --git a/ddprof-lib/src/main/cpp/libraries.h b/ddprof-lib/src/main/cpp/libraries.h index 4abd980aa..e58ceaa02 100644 --- a/ddprof-lib/src/main/cpp/libraries.h +++ b/ddprof-lib/src/main/cpp/libraries.h @@ -18,7 +18,7 @@ class Libraries { // This function will return the 'libjvm' on non-J9 VMs and the library with the given name on J9 VMs CodeCache *findJvmLibrary(const char *j9_lib_name); CodeCache *findLibraryByName(const char *lib_name); - CodeCache *findLibraryByAddress(const void *address); + CodeCache *findLibraryByAddress(const void *address) const; // Get library by index (used for remote symbolication unpacking) // Note: Parameter is uint32_t to match lib_index packing (17 bits = max 131K libraries) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index a081ad9ca..f67f32b7f 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -15,8 +15,10 @@ #include "flightRecorder.h" #include "itimer.h" #include "hotspot/vmStructs.h" -#include "j9/j9Ext.h" +#include "hotspot/hotspotSupport.h" +#include "j9/j9Support.h" #include "j9/j9WallClock.h" +#include "jvmSupport.h" #include "jvmThread.h" #include "libraryPatcher.h" #include "objectSampler.h" @@ -28,7 +30,10 @@ #include "symbols.h" #include "thread.h" #include "tsc.h" +#include "utils.h" #include "wallClock.h" +#include "frames.h" + #include #include #include @@ -58,54 +63,6 @@ static J9WallClock j9_engine; static ITimer itimer; static CTimer ctimer; -// Stack recovery techniques used to workaround AsyncGetCallTrace flaws. -// Can be disabled with 'safemode' option. -enum StackRecovery { - UNKNOWN_JAVA = (1 << 0), - POP_STUB = (1 << 1), - POP_METHOD = (1 << 2), - UNWIND_NATIVE = (1 << 3), - LAST_JAVA_PC = (1 << 4), - GC_TRACES = (1 << 5), - PROBE_SP = 0x100, -}; - -static inline int makeFrame(ASGCT_CallFrame *frames, jint type, jmethodID id) { - frames[0].bci = type; - frames[0].method_id = id; - return 1; -} - -static inline int makeFrame(ASGCT_CallFrame *frames, jint type, uintptr_t id) { - return makeFrame(frames, type, (jmethodID)id); -} - -static inline int makeFrame(ASGCT_CallFrame *frames, jint type, - const char *id) { - return makeFrame(frames, type, (jmethodID)id); -} - -void Profiler::addJavaMethod(const void *address, int length, - jmethodID method) { - CodeHeap::updateBounds(address, (const char *)address + length); -} - -void Profiler::addRuntimeStub(const void *address, int length, - const char *name) { - _stubs_lock.lock(); - _runtime_stubs.add(address, length, name, true); - _stubs_lock.unlock(); - - if (name[0] == 'I' && strcmp(name, "Interpreter") == 0) { - CodeHeap::setInterpreterStart(address); - } else if (strcmp(name, "call_stub") == 0) { - _call_stub_begin = address; - _call_stub_end = (const char *)address + length; - } - - CodeHeap::updateBounds(address, (const char *)address + length); -} - void Profiler::onThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { ProfiledThread::initCurrentThread(); ProfiledThread *current = ProfiledThread::current(); @@ -290,19 +247,6 @@ const char *Profiler::findNativeMethod(const void *address) { return name; } -CodeBlob *Profiler::findRuntimeStub(const void *address) { - return _runtime_stubs.findBlobByAddress(address); -} - -bool Profiler::isAddressInCode(const void *pc, bool include_stubs) { - if (CodeHeap::contains(pc)) { - return CodeHeap::findNMethod(pc) != NULL && - (include_stubs || !(pc >= _call_stub_begin && pc < _call_stub_end)); - } else { - return _libs->findLibraryByAddress(pc) != NULL; - } -} - int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, int event_type, int tid, StackContext *java_ctx, bool *truncated, int lock_index) { @@ -491,273 +435,6 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain, return depth; } -int Profiler::getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, - int max_depth, StackContext *java_ctx, - bool *truncated) { - // Workaround for JDK-8132510: it's not safe to call GetEnv() inside a signal - // handler since JDK 9, so we do it only for threads already registered in - // ThreadLocalStorage - VMThread *vm_thread = (VMThread*)JVMThread::current(); - if (vm_thread == NULL || !vm_thread->isThreadAccessible()) { - Counters::increment(AGCT_NOT_REGISTERED_IN_TLS); - return 0; - } - - JNIEnv *jni = VM::jni(); - if (jni == NULL) { - // Not a Java thread - Counters::increment(AGCT_NOT_JAVA); - return 0; - } - - StackFrame frame(ucontext); - uintptr_t saved_pc, saved_sp, saved_fp; - if (ucontext != NULL) { - saved_pc = frame.pc(); - saved_sp = frame.sp(); - saved_fp = frame.fp(); - - if (saved_pc >= (uintptr_t)_call_stub_begin && - saved_pc < (uintptr_t)_call_stub_end) { - // call_stub is unsafe to walk - frames->bci = BCI_ERROR; - frames->method_id = (jmethodID) "call_stub"; - return 1; - } - - if (!VMStructs::isSafeToWalk(saved_pc)) { - frames->bci = BCI_NATIVE_FRAME; - CodeBlob *codeBlob = - VMStructs::libjvm()->findBlobByAddress((const void *)saved_pc); - if (codeBlob) { - frames->method_id = (jmethodID)codeBlob->_name; - } else { - frames->method_id = (jmethodID) "unknown_unwalkable"; - } - return 1; - } - } else { - return 0; - } - - int state = vm_thread->state(); - // from OpenJDK - // https://github.com/openjdk/jdk/blob/7455bb23c1d18224e48e91aae4f11fe114d04fab/src/hotspot/share/utilities/globalDefinitions.hpp#L1030 - /* - enum JavaThreadState { - _thread_uninitialized = 0, // should never happen (missing - initialization) _thread_new = 2, // just starting up, i.e., in - process of being initialized _thread_new_trans = 3, // corresponding - transition state (not used, included for completeness) _thread_in_native = 4, - // running in native code _thread_in_native_trans = 5, // corresponding - transition state _thread_in_vm = 6, // running in VM - _thread_in_vm_trans = 7, // corresponding transition state - _thread_in_Java = 8, // running in Java or in stub code - _thread_in_Java_trans = 9, // corresponding transition state (not - used, included for completeness) _thread_blocked = 10, // blocked in - vm _thread_blocked_trans = 11, // corresponding transition state - _thread_max_state = 12 // maximum thread state+1 - used for - statistics allocation - }; - */ - bool in_java = (state == 8 || state == 9); - if (in_java && java_ctx->sp != 0) { - // skip ahead to the Java frames before calling AGCT - frame.restore((uintptr_t)java_ctx->pc, java_ctx->sp, java_ctx->fp); - } else if (state != 0) { - VMJavaFrameAnchor* a = vm_thread->anchor(); - if (a == nullptr || a->lastJavaSP() == 0) { - // we haven't found the top Java frame ourselves, and the lastJavaSP wasn't - // recorded either when not in the Java state, lastJava ucontext will be - // used by AGCT - Counters::increment(AGCT_NATIVE_NO_JAVA_CONTEXT); - return 0; - } - } - bool blocked_in_vm = (state == 10 || state == 11); - // avoid unwinding during deoptimization - if (blocked_in_vm && vm_thread->osThreadState() == OSThreadState::RUNNABLE) { - Counters::increment(AGCT_BLOCKED_IN_VM); - return 0; - } - - JitWriteProtection jit(false); - // AsyncGetCallTrace writes to ASGCT_CallFrame array - ASGCT_CallTrace trace = {jni, 0, frames}; - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - - if (trace.num_frames > 0) { - frame.restore(saved_pc, saved_sp, saved_fp); - return trace.num_frames; - } - - if ((trace.num_frames == ticks_unknown_Java || - trace.num_frames == ticks_not_walkable_Java) && - !(_safe_mode & UNKNOWN_JAVA) && ucontext != NULL) { - CodeBlob *stub = NULL; - _stubs_lock.lockShared(); - if (_runtime_stubs.contains((const void *)frame.pc())) { - stub = findRuntimeStub((const void *)frame.pc()); - } - _stubs_lock.unlockShared(); - - if (stub != NULL) { - if (_cstack != CSTACK_NO) { - max_depth -= makeFrame(trace.frames++, BCI_NATIVE_FRAME, stub->_name); - } - if (!(_safe_mode & POP_STUB) && - frame.unwindStub((instruction_t *)stub->_start, stub->_name) && - isAddressInCode((const void *)frame.pc())) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } else if (VMStructs::hasMethodStructs()) { - VMNMethod *nmethod = CodeHeap::findNMethod((const void *)frame.pc()); - if (nmethod != NULL && nmethod->isNMethod() && nmethod->isAlive()) { - VMMethod *method = nmethod->method(); - if (method != NULL) { - jmethodID method_id = method->id(); - if (method_id != NULL) { - max_depth -= makeFrame(trace.frames++, 0, method_id); - } - if (!(_safe_mode & POP_METHOD) && frame.unwindCompiled(nmethod) && - isAddressInCode((const void *)frame.pc())) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - if ((_safe_mode & PROBE_SP) && trace.num_frames < 0) { - if (method_id != NULL) { - trace.frames--; - } - for (int i = 0; trace.num_frames < 0 && i < PROBE_SP_LIMIT; i++) { - frame.sp() += sizeof(void*); - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } - } - } else if (nmethod != NULL) { - if (_cstack != CSTACK_NO) { - max_depth -= - makeFrame(trace.frames++, BCI_NATIVE_FRAME, nmethod->name()); - } - if (!(_safe_mode & POP_STUB) && - frame.unwindStub(NULL, nmethod->name()) && - isAddressInCode((const void *)frame.pc())) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } - } - } else if (trace.num_frames == ticks_unknown_not_Java && - !(_safe_mode & LAST_JAVA_PC)) { - VMJavaFrameAnchor* anchor = vm_thread->anchor(); - if (anchor == NULL) return 0; - uintptr_t sp = anchor->lastJavaSP(); - const void* pc = anchor->lastJavaPC(); - if (sp != 0 && pc == NULL) { - // We have the last Java frame anchor, but it is not marked as walkable. - // Make it walkable here - pc = ((const void**)sp)[-1]; - anchor->setLastJavaPC(pc); - - VMNMethod *m = CodeHeap::findNMethod(pc); - if (m != NULL) { - // AGCT fails if the last Java frame is a Runtime Stub with an invalid - // _frame_complete_offset. In this case we patch _frame_complete_offset - // manually - if (!m->isNMethod() && m->frameSize() > 0 && - m->frameCompleteOffset() == -1) { - m->setFrameCompleteOffset(0); - } - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } else if (_libs->findLibraryByAddress(pc) != NULL) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - - anchor->setLastJavaPC(nullptr); - } - } else if (trace.num_frames == ticks_not_walkable_not_Java && - !(_safe_mode & LAST_JAVA_PC)) { - VMJavaFrameAnchor* anchor = vm_thread->anchor(); - if (anchor == NULL) return 0; - uintptr_t sp = anchor->lastJavaSP(); - const void* pc = anchor->lastJavaPC(); - if (sp != 0 && pc != NULL) { - // Similar to the above: last Java frame is set, - // but points to a Runtime Stub with an invalid _frame_complete_offset - VMNMethod *m = CodeHeap::findNMethod(pc); - if (m != NULL && !m->isNMethod() && m->frameSize() > 0 && - m->frameCompleteOffset() == -1) { - m->setFrameCompleteOffset(0); - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } - } else if (trace.num_frames == ticks_GC_active && !(_safe_mode & GC_TRACES)) { - VMJavaFrameAnchor* anchor = vm_thread->anchor(); - if (anchor == NULL || anchor->lastJavaSP() == 0) { - // Do not add 'GC_active' for threads with no Java frames, e.g. Compiler - // threads - frame.restore(saved_pc, saved_sp, saved_fp); - return 0; - } - } - - frame.restore(saved_pc, saved_sp, saved_fp); - - if (trace.num_frames > 0) { - return trace.num_frames + (trace.frames - frames); - } - - const char *err_string = asgctError(trace.num_frames); - if (err_string == NULL) { - // No Java stack, because thread is not in Java context - return 0; - } - - atomicIncRelaxed(_failures[-trace.num_frames]); - trace.frames->bci = BCI_ERROR; - trace.frames->method_id = (jmethodID)err_string; - return trace.frames - frames + 1; -} - -void Profiler::fillFrameTypes(ASGCT_CallFrame *frames, int num_frames, - VMNMethod *nmethod) { - if (nmethod->isNMethod() && nmethod->isAlive()) { - VMMethod *method = nmethod->method(); - if (method == NULL) { - return; - } - - jmethodID current_method_id = method->id(); - if (current_method_id == NULL) { - return; - } - - // Mark current_method as COMPILED and frames above current_method as - // INLINED - for (int i = 0; i < num_frames; i++) { - if (frames[i].method_id == NULL || frames[i].bci <= BCI_NATIVE_FRAME) { - break; - } - if (frames[i].method_id == current_method_id) { - int level = nmethod->level(); - frames[i].bci = FrameType::encode( - level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED, - frames[i].bci); - for (int j = 0; j < i; j++) { - frames[j].bci = FrameType::encode(FRAME_INLINED, frames[j].bci); - } - break; - } - } - } else if (nmethod->isInterpreter()) { - // Mark the first Java frame as INTERPRETED - for (int i = 0; i < num_frames; i++) { - if (frames[i].bci > BCI_NATIVE_FRAME) { - frames[i].bci = FrameType::encode(FRAME_INTERPRETED, frames[i].bci); - break; - } - } - } -} - u64 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event_type, Event *event, bool deferred) { // Protect JVMTI sampling operations to prevent signal handler interference CriticalSection cs; @@ -859,34 +536,16 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid, ASGCT_CallFrame *native_stop = frames + num_frames; num_frames += getNativeTrace(ucontext, native_stop, event_type, tid, &java_ctx, &truncated, lock_index); - if (num_frames < _max_stack_depth) { - int max_remaining = _max_stack_depth - num_frames; - if (_features.mixed) { - int vm_start = num_frames; - int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated); - num_frames += vm_frames; - } else if (event_type == BCI_CPU || event_type == BCI_WALL) { - if (_cstack >= CSTACK_VM) { - int vm_start = num_frames; - int vm_frames = StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), lock_index, &truncated); - num_frames += vm_frames; - } else { - // Async events - AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); - int java_frames = 0; - if (mutex.acquired()) { - java_frames = getJavaTraceAsync(ucontext, frames + num_frames, max_remaining, &java_ctx, &truncated); - if (java_frames > 0 && java_ctx.pc != NULL && VMStructs::hasMethodStructs()) { - VMNMethod* nmethod = CodeHeap::findNMethod(java_ctx.pc); - if (nmethod != NULL) { - fillFrameTypes(frames + num_frames, java_frames, nmethod); - } - } - } - num_frames += java_frames; - } - } + assert(num_frames >= 0); + + int max_remaining = _max_stack_depth - num_frames; + if (max_remaining > 0) { + // Walk Java frames if we have room, but only for mixed mode or CPU/Wall events with cstack enabled. For async events, we want to avoid walking Java frames in the signal handler if possible, since it can lead to deadlocks. Instead, we'll try to get the Java trace asynchronously after the signal handler returns. + StackWalkRequest request = {event_type, lock_index, ucontext, frames + num_frames, max_remaining, &java_ctx, &truncated}; + num_frames += JVMSupport::walkJavaStack(request); } + + assert(num_frames >= 0); if (num_frames == 0) { num_frames += makeFrame(frames + num_frames, BCI_ERROR, "no_Java_frame"); } @@ -1025,6 +684,20 @@ void *Profiler::dlopen_hook(const char *filename, int flags) { return result; } +const char* Profiler::cstack() const { + switch (_cstack) { + case CSTACK_DEFAULT: return "default"; + case CSTACK_NO: return "no"; + case CSTACK_FP: return "fp"; + case CSTACK_DWARF: return "dwarf"; + case CSTACK_LBR: return "lbr"; + case CSTACK_VM: { + return _features.mixed ? "vmx" : "vm"; + } + default: return "default"; + } +} + void Profiler::switchLibraryTrap(bool enable) { if (_dlopen_entry == NULL) { return; // Not initialized yet, nothing to do @@ -1083,7 +756,7 @@ int Profiler::crashHandlerInternal(int signo, siginfo_t *siginfo, void *ucontext // Reentrancy protection: use TLS-based tracking if available. // If TLS is not available, we can only safely handle faults that we can // prove are from our protected code paths (checked via sameStack heuristic - // in StackWalker::checkFault). For anything else, we must chain immediately + // in HotspotSupport::checkFault). For anything else, we must chain immediately // to avoid claiming faults that aren't ours. bool have_tls_protection = false; if (thrd != nullptr) { @@ -1094,7 +767,7 @@ int Profiler::crashHandlerInternal(int signo, siginfo_t *siginfo, void *ucontext have_tls_protection = true; } // If thrd == nullptr, we proceed but with limited handling capability. - // Only StackWalker::checkFault (which has its own sameStack fallback) + // Only HotspotSupport::checkFault (which has its own sameStack fallback) // and the JDK-8313796 workaround can safely handle faults without TLS. StackFrame frame(ucontext); @@ -1119,11 +792,11 @@ int Profiler::crashHandlerInternal(int signo, siginfo_t *siginfo, void *ucontext if (VM::isHotspot()) { // the following checks require vmstructs and therefore HotSpot - // StackWalker::checkFault has its own fallback for when TLS is unavailable: + // HotspotSupport::checkFault has its own fallback for when TLS is unavailable: // it uses sameStack() heuristic to check if we're in a protected stack walk. // If the fault is from our protected walk, it will longjmp and never return. // If it returns, the fault wasn't from our code. - StackWalker::checkFault(thrd); + HotspotSupport::checkFault(thrd); // Workaround for JDK-8313796 if needed. Setting cstack=dwarf also helps if (_need_JDK_8313796_workaround && @@ -1229,8 +902,8 @@ Engine *Profiler::selectCpuEngine(Arguments &args) { return &noop_engine; } else if (args._cpu >= 0 || strcmp(args._event, EVENT_CPU) == 0) { if (VM::isOpenJ9()) { - if (!J9Ext::shouldUseAsgct() || !J9Ext::can_use_ASGCT()) { - if (!J9Ext::is_jvmti_jmethodid_safe()) { + if (!J9Support::shouldUseAsgct() || !J9Support::can_use_ASGCT()) { + if (!J9Support::is_jvmti_jmethodid_safe()) { LOG_WARN("Safe jmethodID access is not available on this JVM. Using " "CPU profiler on your own risk. Use -XX:+KeepJNIIDs=true JVM " "flag to make access to jmethodIDs safe, if your JVM supports it"); @@ -1261,8 +934,8 @@ Engine *Profiler::selectWallEngine(Arguments &args) { return &noop_engine; } if (VM::isOpenJ9()) { - if (args._wallclock_sampler == JVMTI || !J9Ext::shouldUseAsgct() || !J9Ext::can_use_ASGCT()) { - if (!J9Ext::is_jvmti_jmethodid_safe()) { + if (args._wallclock_sampler == JVMTI || !J9Support::shouldUseAsgct() || !J9Support::can_use_ASGCT()) { + if (!J9Support::is_jvmti_jmethodid_safe()) { LOG_WARN("Safe jmethodID access is not available on this JVM. Using " "wallclock profiler on your own risk. Use -XX:+KeepJNIIDs=true JVM " "flag to make access to jmethodIDs safe, if your JVM supports it"); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 31260784f..a592e901f 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -58,46 +58,9 @@ class VM; enum State { NEW, IDLE, RUNNING, TERMINATED }; -/** - * Converts a BCI_* frame type value to the corresponding EventType enum value. - * - * This conversion is necessary because Datadog's implementation uses BCI_* values - * (from ASGCT_CallFrameType) directly as event type identifiers, while upstream - * StackWalker::walkVM() expects EventType enum values for its logic. - * - * BCI_* values are special frame types with negative values (except BCI_CPU=0) - * that indicate non-standard frame information in call traces. EventType values - * are positive enum indices used for event categorization in the upstream code. - * - * @param bci_type A BCI_* value (e.g., BCI_CPU, BCI_WALL, BCI_ALLOC) - * @return The corresponding EventType enum value - */ -inline EventType eventTypeFromBCI(jint bci_type) { - switch (bci_type) { - case BCI_CPU: - return EXECUTION_SAMPLE; // CPU samples map to execution samples - case BCI_WALL: - return WALL_CLOCK_SAMPLE; - case BCI_ALLOC: - return ALLOC_SAMPLE; - case BCI_ALLOC_OUTSIDE_TLAB: - return ALLOC_OUTSIDE_TLAB; - case BCI_LIVENESS: - return LIVE_OBJECT; - case BCI_LOCK: - return LOCK_SAMPLE; - case BCI_PARK: - return PARK_SAMPLE; - default: - // For unknown or invalid BCI types, default to EXECUTION_SAMPLE - // This maintains backward compatibility and prevents undefined behavior - return EXECUTION_SAMPLE; - } -} - // Aligned to satisfy SpinLock member alignment requirement (64 bytes) // Required because this class contains multiple SpinLock members: -// _class_map_lock, _locks[], and _stubs_lock +// _class_map_lock and _locks[] class alignas(alignof(SpinLock)) Profiler { friend VM; @@ -148,10 +111,6 @@ class alignas(alignof(SpinLock)) Profiler { volatile jvmtiEventMode _thread_events_state; Libraries* _libs; - SpinLock _stubs_lock; - CodeCache _runtime_stubs; - const void *_call_stub_begin; - const void *_call_stub_end; u32 _num_context_attributes; bool _omit_stacktraces; bool _remote_symbolication; // Enable remote symbolication for native frames @@ -164,21 +123,12 @@ class alignas(alignof(SpinLock)) Profiler { void enableEngines(); void disableEngines(); - void addJavaMethod(const void *address, int length, jmethodID method); - void addRuntimeStub(const void *address, int length, const char *name); - void onThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); void onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); - const char *asgctError(int code); u32 getLockIndex(int tid); - bool isAddressInCode(uintptr_t addr); int getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, int event_type, int tid, StackContext *java_ctx, bool *truncated, int lock_index); - int getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, int max_depth, - StackContext *java_ctx, bool *truncated); - void fillFrameTypes(ASGCT_CallFrame *frames, int num_frames, - VMNMethod *nmethod); void updateThreadName(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, bool self = false); void updateJavaThreadNames(); @@ -208,36 +158,46 @@ class alignas(alignof(SpinLock)) Profiler { _start_time(0), _stop_time(0), _epoch(0), _timer_id(NULL), _total_samples(0), _failures(), _class_map_lock(), _max_stack_depth(0), _features(), _safe_mode(0), _cstack(CSTACK_NO), - _thread_events_state(JVMTI_DISABLE), _libs(Libraries::instance()), _stubs_lock(), - _runtime_stubs("[stubs]"), _call_stub_begin(NULL), _call_stub_end(NULL), - _num_context_attributes(0), _omit_stacktraces(false), _remote_symbolication(false), - _dlopen_entry(NULL) { + _thread_events_state(JVMTI_DISABLE), _libs(Libraries::instance()), + _num_context_attributes(0), _omit_stacktraces(false), + _remote_symbolication(false), _dlopen_entry(NULL) { for (int i = 0; i < CONCURRENCY_LEVEL; i++) { _calltrace_buffer[i] = NULL; } } - static Profiler *instance() { + static inline Profiler *instance() { return _instance; } - int status(char* status, int max_len); - const char* cstack() { - switch (_cstack) { - case CSTACK_DEFAULT: return "default"; - case CSTACK_NO: return "no"; - case CSTACK_FP: return "fp"; - case CSTACK_DWARF: return "dwarf"; - case CSTACK_LBR: return "lbr"; - case CSTACK_VM: { - return _features.mixed ? "vmx" : "vm"; - } - default: return "default"; + inline void incFailure(int type) { + if (type < ASGCT_FAILURE_TYPES) { + atomicIncRelaxed(_failures[type]); } } + int status(char* status, int max_len); + static const char *asgctError(int code); + + + inline int safe_mode() const { + return _safe_mode; + } + + inline const Libraries* libraries() const { + return _libs; + } + + inline CStack cstackMode() const { + return _cstack; + } + + inline const StackWalkFeatures& stackWalkFeatures() const { + return _features; + } + u64 total_samples() { return _total_samples; } int max_stack_depth() { return _max_stack_depth; } time_t uptime() { return time(NULL) - _start_time; } @@ -250,6 +210,7 @@ class alignas(alignof(SpinLock)) Profiler { u32 numContextAttributes() { return _num_context_attributes; } ThreadFilter *threadFilter() { return &_thread_filter; } + const char* cstack() const; int lookupClass(const char *key, size_t length); void processCallTraces(std::function&)> processor) { if (!_omit_stacktraces) { @@ -280,7 +241,7 @@ class alignas(alignof(SpinLock)) Profiler { Error flushJfr(); Error dump(const char *path, const int length); void logStats(); - void switchThreadEvents(jvmtiEventMode mode); + void switchThreadEvents(jvmtiEventMode mode); /** * Remote symbolication packed data layout (BCI_NATIVE_FRAME_REMOTE): @@ -388,8 +349,6 @@ class alignas(alignof(SpinLock)) Profiler { const void *resolveSymbol(const char *name); const char *getLibraryName(const char *native_symbol); const char *findNativeMethod(const void *address); - CodeBlob *findRuntimeStub(const void *address); - bool isAddressInCode(const void *pc, bool include_stubs = true); static void segvHandler(int signo, siginfo_t *siginfo, void *ucontext); static void busHandler(int signo, siginfo_t *siginfo, void *ucontext); @@ -398,20 +357,6 @@ class alignas(alignof(SpinLock)) Profiler { static int registerThread(int tid); static void unregisterThread(int tid); - // CompiledMethodLoad is also needed to enable DebugNonSafepoints info by - // default - static void JNICALL CompiledMethodLoad(jvmtiEnv *jvmti, jmethodID method, - jint code_size, const void *code_addr, - jint map_length, - const jvmtiAddrLocationMap *map, - const void *compile_info) { - instance()->addJavaMethod(code_addr, code_size, method); - } - - static void JNICALL DynamicCodeGenerated(jvmtiEnv *jvmti, const char *name, - const void *address, jint length) { - instance()->addRuntimeStub(address, length, name); - } static void JNICALL ThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { diff --git a/ddprof-lib/src/main/cpp/stackWalker.cpp b/ddprof-lib/src/main/cpp/stackWalker.cpp index 93448a87f..cc608efe5 100644 --- a/ddprof-lib/src/main/cpp/stackWalker.cpp +++ b/ddprof-lib/src/main/cpp/stackWalker.cpp @@ -5,67 +5,21 @@ */ #include -#include "stackWalker.h" +#include "stackWalker.inline.h" #include "dwarf.h" #include "profiler.h" -#include "safeAccess.h" #include "stackFrame.h" #include "symbols.h" #include "hotspot/vmStructs.inline.h" #include "jvmThread.h" #include "thread.h" - -const uintptr_t MAX_WALK_SIZE = 0x100000; -const intptr_t MAX_FRAME_SIZE_WORDS = StackWalkValidation::MAX_FRAME_SIZE / sizeof(void*); // 0x8000 = 32768 words - -static ucontext_t empty_ucontext{}; - // Use validation helpers from header (shared with tests) using StackWalkValidation::inDeadZone; using StackWalkValidation::aligned; using StackWalkValidation::MAX_FRAME_SIZE; using StackWalkValidation::sameStack; -// AArch64: on Linux, frame link is stored at the top of the frame, -// while on macOS, frame link is at the bottom. -static inline uintptr_t defaultSenderSP(uintptr_t sp, uintptr_t fp) { -#ifdef __APPLE__ - return sp + 2 * sizeof(void*); -#else - return fp; -#endif -} - -static inline void fillFrame(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, const char* name) { - frame.bci = type; - frame.method_id = (jmethodID)name; -} - -// Overload for RemoteFrameInfo* (passed as void* to support both char* and RemoteFrameInfo*) -static inline void fillFrame(ASGCT_CallFrame& frame, int bci, void* method_id_ptr) { - frame.bci = bci; - frame.method_id = (jmethodID)method_id_ptr; -} - -static inline void fillFrame(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, u32 class_id) { - frame.bci = type; - frame.method_id = (jmethodID)(uintptr_t)class_id; -} - -static inline void fillFrame(ASGCT_CallFrame& frame, FrameTypeId type, int bci, jmethodID method) { - frame.bci = FrameType::encode(type, bci); - frame.method_id = method; -} - -static jmethodID getMethodId(VMMethod* method) { - if (!inDeadZone(method) && aligned((uintptr_t)method) - && SafeAccess::isReadableRange(method, VMMethod::type_size())) { - return method->validatedId(); - } - return NULL; -} - int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated) { const void* pc; @@ -228,649 +182,3 @@ int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth return depth; } - -__attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, int lock_index, bool* truncated) { - if (ucontext == NULL) { - return walkVM(&empty_ucontext, frames, max_depth, features, event_type, - callerPC(), (uintptr_t)callerSP(), (uintptr_t)callerFP(), lock_index, truncated); - } else { - StackFrame frame(ucontext); - return walkVM(ucontext, frames, max_depth, features, event_type, - (const void*)frame.pc(), frame.sp(), frame.fp(), lock_index, truncated); - } -} - -__attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, VMJavaFrameAnchor* anchor, EventType event_type, int lock_index, bool* truncated) { - uintptr_t sp = anchor->lastJavaSP(); - if (sp == 0) { - return 0; - } - - uintptr_t fp = anchor->lastJavaFP(); - if (fp == 0) { - fp = sp; - } - - const void* pc = anchor->lastJavaPC(); - if (pc == NULL || !CodeHeap::contains(pc)) { - // lastJavaPC is NULL (thread not in Java→native transition) or points outside - // the tracked CodeHeap range (e.g. interpreter/stub code in a separately mmap'd - // region). Read the actual return address from the stack frame instead. - if (!aligned(sp)) { - return 0; - } - pc = ((const void**)sp)[-1]; - } - - StackWalkFeatures no_features{}; - return walkVM(ucontext, frames, max_depth, no_features, event_type, pc, sp, fp, lock_index, truncated); -} - -__attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, - const void* pc, uintptr_t sp, uintptr_t fp, int lock_index, bool* truncated) { - // VMStructs is only available for hotspot JVM - assert(VM::isHotspot()); - StackFrame frame(ucontext); - uintptr_t bottom = (uintptr_t)&frame + MAX_WALK_SIZE; - - Profiler* profiler = Profiler::instance(); - int bcp_offset = InterpreterFrame::bcp_offset(); - - jmp_buf crash_protection_ctx; - VMThread* vm_thread = VMThread::current(); - if (vm_thread != NULL && !vm_thread->isThreadAccessible()) { - Counters::increment(WALKVM_THREAD_INACCESSIBLE); - vm_thread = NULL; - } - if (vm_thread == NULL) { - Counters::increment(WALKVM_NO_VMTHREAD); - } else { - Counters::increment(WALKVM_VMTHREAD_OK); - } - void* saved_exception = vm_thread != NULL ? vm_thread->exception() : NULL; - - // Should be preserved across setjmp/longjmp - volatile int depth = 0; - int actual_max_depth = truncated ? max_depth + 1 : max_depth; - bool fp_chain_fallback = false; - int fp_chain_depth = 0; - - ProfiledThread* profiled_thread = ProfiledThread::currentSignalSafe(); - - VMJavaFrameAnchor* anchor = NULL; - if (vm_thread != NULL) { - anchor = vm_thread->anchor(); - if (anchor == NULL) { - Counters::increment(WALKVM_ANCHOR_NULL); - } - vm_thread->exception() = &crash_protection_ctx; - if (profiled_thread != nullptr) { - profiled_thread->setCrashProtectionActive(true); - } - if (setjmp(crash_protection_ctx) != 0) { - if (profiled_thread != nullptr) { - profiled_thread->setCrashProtectionActive(false); - } - vm_thread->exception() = saved_exception; - if (depth < max_depth) { - fillFrame(frames[depth++], BCI_ERROR, "break_not_walkable"); - } - return depth; - } - } - - const void* prev_native_pc = NULL; - - // Saved anchor data — preserved across anchor consumption so inline - // recovery can redirect even after the anchor pointer has been set to NULL. - // Recovery is one-shot: once attempted, we do not retry to avoid - // ping-ponging between CodeHeap and unmapped native regions. - const void* saved_anchor_pc = NULL; - uintptr_t saved_anchor_sp = 0; - uintptr_t saved_anchor_fp = 0; - bool anchor_recovery_used = false; - - // Show extended frame types and stub frames for execution-type events - bool details = event_type <= MALLOC_SAMPLE || features.mixed; - - if (details && vm_thread != NULL && vm_thread->cachedIsJavaThread()) { - anchor = vm_thread->anchor(); - } - - unwind_loop: - - // Walk until the bottom of the stack or until the first Java frame - while (depth < actual_max_depth) { - if (CodeHeap::contains(pc)) { - Counters::increment(WALKVM_HIT_CODEHEAP); - if (fp_chain_fallback) { - Counters::increment(WALKVM_FP_CHAIN_REACHED_CODEHEAP); - fp_chain_fallback = false; - fp_chain_depth = 0; - } - // If we're in JVM-generated code but don't have a VMThread, we cannot safely - // walk the Java stack because crash protection is not set up. - // - // This can occur during JNI attach/detach transitions: when a thread detaches, - // pthread_setspecific() clears the VMThread TLS, but if a profiling signal arrives - // while PC is still in JVM stubs (JavaCalls, method entry/exit), we see CodeHeap - // code without VMThread context. - // - // Without vm_thread, crash protection via setjmp/longjmp cannot work - // (checkFault() needs vm_thread->exception() to longjmp). Any memory dereference in interpreter - // frame handling or NMethod validation would crash the process with unrecoverable SEGV. - // - // The missing VMThread is a timing issue during thread lifecycle. - if (vm_thread == NULL) { - Counters::increment(WALKVM_CODEH_NO_VM); - fillFrame(frames[depth++], BCI_ERROR, "break_no_vmthread"); - break; - } - prev_native_pc = NULL; // we are in JVM code, no previous 'native' PC - VMNMethod* nm = CodeHeap::findNMethod(pc); - if (nm == NULL) { - if (anchor == NULL) { - // Add an error frame only if we cannot recover - fillFrame(frames[depth++], BCI_ERROR, "unknown_nmethod"); - } - break; - } - - // Always prefer JavaFrameAnchor when it is available, - // since it provides reliable SP and FP. - // Do not treat the topmost stub as Java frame. - if (anchor != NULL && (depth > 0 || !nm->isStub())) { - Counters::increment(WALKVM_ANCHOR_CONSUMED); - // Preserve anchor data before consumption — getFrame() is read-only - // but we set anchor=NULL below, losing the pointer for later recovery. - if (saved_anchor_sp == 0) { - saved_anchor_pc = anchor->lastJavaPC(); - saved_anchor_sp = anchor->lastJavaSP(); - saved_anchor_fp = anchor->lastJavaFP(); - } - if (anchor->getFrame(pc, sp, fp) && !nm->contains(pc)) { - anchor = NULL; - continue; // NMethod has changed as a result of correction - } - anchor = NULL; - } - - if (nm->isInterpreter()) { - if (vm_thread != NULL && vm_thread->inDeopt()) { - fillFrame(frames[depth++], BCI_ERROR, "break_deopt"); - break; - } - - bool is_plausible_interpreter_frame = StackWalkValidation::isPlausibleInterpreterFrame(fp, sp, bcp_offset); - if (is_plausible_interpreter_frame) { - VMMethod* method = ((VMMethod**)fp)[InterpreterFrame::method_offset]; - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - Counters::increment(WALKVM_JAVA_FRAME_OK); - const char* bytecode_start = method->bytecode(); - const char* bcp = ((const char**)fp)[bcp_offset]; - int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; - fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); - - sp = ((uintptr_t*)fp)[InterpreterFrame::sender_sp_offset]; - pc = stripPointer(((void**)fp)[FRAME_PC_SLOT]); - fp = *(uintptr_t*)fp; - continue; - } - } - - if (depth == 0) { - VMMethod* method = (VMMethod*)frame.method(); - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - Counters::increment(WALKVM_JAVA_FRAME_OK); - fillFrame(frames[depth++], FRAME_INTERPRETED, 0, method_id); - - if (is_plausible_interpreter_frame) { - pc = stripPointer(((void**)fp)[FRAME_PC_SLOT]); - sp = frame.senderSP(); - fp = *(uintptr_t*)fp; - } else { - pc = stripPointer(SafeAccess::load((void**)sp)); - sp = frame.senderSP(); - } - continue; - } - } - - Counters::increment(WALKVM_BREAK_INTERPRETED); - fillFrame(frames[depth++], BCI_ERROR, "break_interpreted"); - break; - } else if (nm->isNMethod()) { - // Check if deoptimization is in progress before walking compiled frames - if (vm_thread != NULL && vm_thread->inDeopt()) { - fillFrame(frames[depth++], BCI_ERROR, "break_deopt_compiled"); - break; - } - - Counters::increment(WALKVM_JAVA_FRAME_OK); - int level = nm->level(); - FrameTypeId type = details && level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; - fillFrame(frames[depth++], type, 0, nm->method()->id()); - - if (nm->isFrameCompleteAt(pc)) { - if (depth == 1 && frame.unwindEpilogue(nm, (uintptr_t&)pc, sp, fp)) { - continue; - } - - int scope_offset = nm->findScopeOffset(pc); - if (scope_offset > 0) { - depth--; - ScopeDesc scope(nm); - do { - scope_offset = scope.decode(scope_offset); - if (details) { - type = scope_offset > 0 ? FRAME_INLINED : - level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; - } - fillFrame(frames[depth++], type, scope.bci(), scope.method()->id()); - } while (scope_offset > 0 && depth < max_depth); - } - - // Handle situations when sp is temporarily changed in the compiled code - frame.adjustSP(nm->entry(), pc, sp); - - // Validate NMethod metadata before using frameSize() - int frame_size = nm->frameSize(); - if (frame_size <= 0 || frame_size > MAX_FRAME_SIZE_WORDS) { - fillFrame(frames[depth++], BCI_ERROR, "break_invalid_framesize"); - break; - } - - sp += frame_size * sizeof(void*); - - // Verify alignment before dereferencing sp as pointer (secondary defense) - if (!aligned(sp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_misaligned_sp"); - break; - } - - fp = ((uintptr_t*)sp)[-FRAME_PC_SLOT - 1]; - pc = ((const void**)sp)[-FRAME_PC_SLOT]; - continue; - } else if (frame.unwindPrologue(nm, (uintptr_t&)pc, sp, fp)) { - continue; - } - - Counters::increment(WALKVM_BREAK_COMPILED); - fillFrame(frames[depth++], BCI_ERROR, "break_compiled"); - break; - } else if (nm->isEntryFrame(pc) && !features.mixed) { - VMJavaFrameAnchor* next_anchor = VMJavaFrameAnchor::fromEntryFrame(fp); - if (next_anchor == NULL) { - fillFrame(frames[depth++], BCI_ERROR, "break_entry_frame"); - break; - } - uintptr_t prev_sp = sp; - if (!next_anchor->getFrame(pc, sp, fp)) { - // End of Java stack - break; - } - if (sp < prev_sp || sp >= bottom || !aligned(sp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_entry_frame"); - break; - } - continue; - } else { - if (features.vtable_target && nm->isVTableStub() && depth == 0) { - uintptr_t receiver = frame.jarg0(); - if (receiver != 0) { - VMSymbol* symbol = VMKlass::fromOop(receiver)->name(); - u32 class_id = profiler->classMap()->lookup(symbol->body(), symbol->length()); - fillFrame(frames[depth++], BCI_ALLOC, class_id); - } - } - - CodeBlob* stub = profiler->findRuntimeStub(pc); - const void* start = stub != NULL ? stub->_start : nm->code(); - const char* name = stub != NULL ? stub->_name : nm->name(); - - if (details) { - fillFrame(frames[depth++], BCI_NATIVE_FRAME, name); - } - - if (frame.unwindStub((instruction_t*)start, name, (uintptr_t&)pc, sp, fp)) { - continue; - } - - if (depth > 0 && nm->frameSize() > 0) { - Counters::increment(WALKVM_STUB_FRAMESIZE_FALLBACK); - // Validate NMethod metadata before using frameSize() - int frame_size = nm->frameSize(); - if (frame_size <= 0 || frame_size > MAX_FRAME_SIZE_WORDS) { - fillFrame(frames[depth++], BCI_ERROR, "break_invalid_framesize"); - break; - } - - sp += frame_size * sizeof(void*); - - // Verify alignment before dereferencing sp as pointer (secondary defense) - if (!aligned(sp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_misaligned_sp"); - break; - } - - fp = ((uintptr_t*)sp)[-FRAME_PC_SLOT - 1]; - pc = ((const void**)sp)[-FRAME_PC_SLOT]; - continue; - } - } - } else { - // Resolve native frame (may use remote symbolication if enabled) - Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index); - if (resolution.is_marked) { - // This is a marked C++ interpreter frame, terminate scan - break; - } - const char* method_name = resolution.method_name; - int frame_bci = resolution.bci; - char mark; - if (frame_bci != BCI_NATIVE_FRAME_REMOTE && method_name != NULL && (mark = NativeFunc::read_mark(method_name)) != 0) { - if (mark == MARK_ASYNC_PROFILER && event_type == MALLOC_SAMPLE) { - // Skip all internal frames above malloc_hook functions, leave the hook itself - depth = 0; - } else if (mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) { - // Insert current compile task as a pseudo Java frame - VMMethod* method = vm_thread->compiledMethod(); - jmethodID method_id = method != NULL ? method->id() : NULL; - if (method_id != NULL) { - fillFrame(frames[depth++], FRAME_JIT_COMPILED, 0, method_id); - } - } else if (mark == MARK_THREAD_ENTRY) { - // Thread entry point detected via pre-computed mark - this is the root frame - // No need for expensive symbol resolution, just stop unwinding - Counters::increment(THREAD_ENTRY_MARK_DETECTIONS); - break; - } - } else if (method_name == NULL && details && !anchor_recovery_used - && profiler->findLibraryByAddress(pc) == NULL) { - // Try anchor recovery — prefer live anchor, fall back to saved data - anchor_recovery_used = true; - const void* recovery_pc = NULL; - uintptr_t recovery_sp = 0; - uintptr_t recovery_fp = 0; - bool have_anchor_data = false; - - if (anchor) { - Counters::increment(WALKVM_ANCHOR_USED_INLINE); - recovery_fp = anchor->lastJavaFP(); - recovery_sp = anchor->lastJavaSP(); - recovery_pc = anchor->lastJavaPC(); - have_anchor_data = true; - } else if (saved_anchor_sp != 0) { - Counters::increment(WALKVM_SAVED_ANCHOR_USED); - recovery_fp = saved_anchor_fp; - recovery_sp = saved_anchor_sp; - recovery_pc = saved_anchor_pc; - have_anchor_data = true; - // Clear saved data after use — one-shot recovery - saved_anchor_sp = 0; - } else { - Counters::increment(WALKVM_ANCHOR_INLINE_NO_ANCHOR); - } - - if (have_anchor_data) { - // Try to read the Java method directly from the anchor's FP, - // treating it as an interpreter frame. - // In HotSpot, lastJavaFP is non-zero only for interpreter frames; - // compiled frames record FP=0 in the anchor. - if (StackWalkValidation::isPlausibleInterpreterFrame(recovery_fp, recovery_sp, bcp_offset)) { - VMMethod* method = ((VMMethod**)recovery_fp)[InterpreterFrame::method_offset]; - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - anchor = NULL; - prev_native_pc = NULL; - if (depth > 0 && depth + 1 < actual_max_depth) { - fillFrame(frames[depth++], BCI_ERROR, "[skipped frames]"); - } - Counters::increment(WALKVM_JAVA_FRAME_OK); - const char* bytecode_start = method->bytecode(); - const char* bcp = ((const char**)recovery_fp)[bcp_offset]; - int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; - fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); - - sp = ((uintptr_t*)recovery_fp)[InterpreterFrame::sender_sp_offset]; - pc = stripPointer(((void**)recovery_fp)[FRAME_PC_SLOT]); - fp = *(uintptr_t*)recovery_fp; - continue; - } - } - - // Fallback: redirect via recovery SP/FP/PC - sp = recovery_sp; - fp = recovery_fp; - pc = recovery_pc; - if (pc != NULL && !CodeHeap::contains(pc) && sp != 0 && aligned(sp) && sp < bottom) { - pc = ((const void**)sp)[-1]; - } - if (sp != 0 && pc != NULL) { - anchor = NULL; - if (sp >= bottom || !aligned(sp)) { - Counters::increment(WALKVM_ANCHOR_INLINE_BAD_SP); - fillFrame(frames[depth++], BCI_ERROR, "break_no_anchor"); - break; - } - prev_native_pc = NULL; - if (depth > 0) { - fillFrame(frames[depth++], BCI_ERROR, "[skipped frames]"); - } - continue; - } - Counters::increment(WALKVM_ANCHOR_INLINE_NO_SP); - } - // Check previous frame for thread entry points (Rust, libc/pthread) - // Only check marks for traditionally-resolved frames; packed remote - // frames store an integer in the method_name union, not a valid pointer. - if (prev_native_pc != NULL) { - Profiler::NativeFrameResolution prev_resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)prev_native_pc, lock_index); - if (prev_resolution.bci != BCI_NATIVE_FRAME_REMOTE) { - const char* prev_method_name = prev_resolution.method_name; - if (prev_method_name != NULL) { - char prev_mark = NativeFunc::read_mark(prev_method_name); - if (prev_mark == MARK_THREAD_ENTRY) { - Counters::increment(THREAD_ENTRY_MARK_DETECTIONS); - break; - } - } - } - } - // Fall through to DWARF section — when findLibraryByAddress(pc) - // returns NULL, default_frame uses FP-chain walking (DW_REG_FP) - // which can bridge symbol-less gaps in libjvm.so. - Counters::increment(WALKVM_FP_CHAIN_ATTEMPT); - fp_chain_fallback = true; - if (++fp_chain_depth > actual_max_depth) { - break; - } - goto dwarf_unwind; - } - fillFrame(frames[depth++], frame_bci, (void*)method_name); - } - - dwarf_unwind: - uintptr_t prev_sp = sp; - CodeCache* cc = profiler->findLibraryByAddress(pc); - FrameDesc f = cc != NULL ? cc->findFrameDesc(pc) : FrameDesc::fallback_default_frame(); - - u8 cfa_reg = (u8)f.cfa; - int cfa_off = f.cfa >> 8; - - // If DWARF is invalid, we cannot continue unwinding reliably - // Thread entry points are detected earlier via MARK_THREAD_ENTRY - if (cfa_reg == DW_REG_INVALID || cfa_reg > DW_REG_PLT) { - break; - } - - if (cfa_reg == DW_REG_SP) { - sp = sp + cfa_off; - } else if (cfa_reg == DW_REG_FP) { - // Sanity-check FP before deriving CFA from it. A corrupted FP can produce a - // phantom CFA and cause the walk to record spurious frames before breaking. - // We cannot check fp < sp here because on aarch64 the frame pointer is set - // to SP at function entry, which is typically less than the previous CFA. - if (fp >= bottom || !aligned(fp)) { - break; - } - sp = fp + cfa_off; - } else if (cfa_reg == DW_REG_PLT) { - sp += ((uintptr_t)pc & 15) >= 11 ? cfa_off * 2 : cfa_off; - } - - // Check if the next frame is below on the current stack - if (sp < prev_sp || sp >= prev_sp + MAX_FRAME_SIZE || sp >= bottom) { - break; - } - - // Stack pointer must be word aligned - if (!aligned(sp)) { - break; - } - - // store the previous pc before unwinding - prev_native_pc = pc; - if (f.fp_off & DW_PC_OFFSET) { - pc = (const char*)pc + (f.fp_off >> 1); - } else { - if (f.fp_off != DW_SAME_FP && f.fp_off < MAX_FRAME_SIZE && f.fp_off > -MAX_FRAME_SIZE) { - fp = (uintptr_t)SafeAccess::load((void**)(sp + f.fp_off)); - } - - if (EMPTY_FRAME_SIZE > 0 || f.pc_off != DW_LINK_REGISTER) { - // Verify alignment before dereferencing sp + offset - uintptr_t pc_addr = sp + f.pc_off; - if (!aligned(pc_addr)) { - break; - } - pc = stripPointer(SafeAccess::load((void**)pc_addr)); - } else if (depth == 1) { - pc = (const void*)frame.link(); - } else { - break; - } - - if (EMPTY_FRAME_SIZE == 0 && cfa_off == 0 && f.fp_off != DW_SAME_FP) { - // AArch64 default_frame - sp = defaultSenderSP(sp, fp); - if (sp < prev_sp || sp >= bottom || !aligned(sp)) { - break; - } - } - } - - if (inDeadZone(pc) || (pc == prev_native_pc && sp == prev_sp)) { - break; - } - } - - // If we did not meet Java frame but current thread has JavaFrameAnchor set, - // try to read the interpreter frame directly from the anchor's FP. - // In HotSpot, lastJavaFP != 0 reliably indicates an interpreter frame. - if (anchor != NULL) { - uintptr_t anchor_fp = anchor->lastJavaFP(); - uintptr_t anchor_sp = anchor->lastJavaSP(); - if (anchor_sp == 0) { - Counters::increment(WALKVM_ANCHOR_NOT_IN_JAVA); - goto done; - } - if (StackWalkValidation::isPlausibleInterpreterFrame(anchor_fp, anchor_sp, bcp_offset)) { - VMMethod* method = ((VMMethod**)anchor_fp)[InterpreterFrame::method_offset]; - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - Counters::increment(WALKVM_ANCHOR_FALLBACK); - Counters::increment(WALKVM_JAVA_FRAME_OK); - anchor = NULL; - while (depth > 0 && frames[depth - 1].method_id == NULL) depth--; - if (depth < actual_max_depth) { - const char* bytecode_start = method->bytecode(); - const char* bcp = ((const char**)anchor_fp)[bcp_offset]; - int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; - fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); - sp = ((uintptr_t*)anchor_fp)[InterpreterFrame::sender_sp_offset]; - pc = stripPointer(((void**)anchor_fp)[FRAME_PC_SLOT]); - fp = *(uintptr_t*)anchor_fp; - if (sp != 0 && sp < bottom && aligned(sp)) { - goto unwind_loop; - } - } - } - } - // Fallback: redirect via anchor frame and sp[-1] - if (anchor != NULL && anchor->getFrame(pc, sp, fp)) { - if (!CodeHeap::contains(pc) && sp != 0 && aligned(sp) && sp < bottom) { - pc = ((const void**)sp)[-1]; - } - Counters::increment(WALKVM_ANCHOR_FALLBACK); - anchor = NULL; - while (depth > 0 && frames[depth - 1].method_id == NULL) depth--; - if (sp != 0 && sp < bottom && aligned(sp)) { - goto unwind_loop; - } - } else if (anchor != NULL) { - Counters::increment(WALKVM_ANCHOR_FALLBACK_FAIL); - } - } - - done: - if (profiled_thread != nullptr) { - profiled_thread->setCrashProtectionActive(false); - } - if (vm_thread != NULL) { - vm_thread->exception() = saved_exception; - } - - // Drop unknown leaf frame - it provides no useful information and breaks - // aggregation by lumping unrelated samples under a single "unknown" entry - depth = StackWalkValidation::dropUnknownLeaf(frames, depth); - - if (depth == 0) { - Counters::increment(WALKVM_DEPTH_ZERO); - } - - if (truncated) { - if (depth > max_depth) { - *truncated = true; - depth = max_depth; - } else if (depth > 0) { - if (frames[depth - 1].bci == BCI_ERROR) { - // root frame is error; best guess is that the trace is truncated - *truncated = true; - } - } - } - - return depth; -} - -void StackWalker::checkFault(ProfiledThread* thrd) { - if (!JVMThread::isInitialized()) { - // JVM has not been loaded or VMStructs have not been initialized yet - return; - } - - VMThread* vm_thread = VMThread::current(); - if (vm_thread == NULL || !vm_thread->isThreadAccessible()) { - return; - } - - // Prefer the semantic crash protection flag (reliable regardless of stack frame sizes). - // Fall back to sameStack heuristic when ProfiledThread TLS is unavailable (e.g. during - // early init or in crash recovery tests). sameStack uses a fixed 8KB threshold which - // can fail with ASAN-inflated frames, but the crashProtectionActive path handles that. - bool protected_walk = (thrd != nullptr && thrd->isCrashProtectionActive()) - || sameStack(vm_thread->exception(), &vm_thread); - if (!protected_walk) { - return; - } - - if (thrd != nullptr) { - thrd->resetCrashHandler(); - } - longjmp(*(jmp_buf*)vm_thread->exception(), 1); -} diff --git a/ddprof-lib/src/main/cpp/stackWalker.h b/ddprof-lib/src/main/cpp/stackWalker.h index 10b91e0c4..a477099f3 100644 --- a/ddprof-lib/src/main/cpp/stackWalker.h +++ b/ddprof-lib/src/main/cpp/stackWalker.h @@ -71,18 +71,20 @@ namespace StackWalkValidation { } } -class StackWalker { - static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, - const void* pc, uintptr_t sp, uintptr_t fp, int lock_index, bool* truncated); +typedef struct { + jint event_type; + u32 lock_index; + void* ucontext; + ASGCT_CallFrame* frames; + int max_depth; + StackContext* java_ctx; + bool* truncated; +} StackWalkRequest; +class StackWalker { public: static int walkFP(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated = nullptr); static int walkDwarf(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated = nullptr); - static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, StackWalkFeatures features, EventType event_type, int lock_index, bool* truncated = nullptr); - static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, VMJavaFrameAnchor* anchor, EventType event_type, int lock_index, bool* truncated = nullptr); - - static void checkFault(ProfiledThread* thrd = nullptr); }; #endif // _STACKWALKER_H diff --git a/ddprof-lib/src/main/cpp/stackWalker.inline.h b/ddprof-lib/src/main/cpp/stackWalker.inline.h new file mode 100644 index 000000000..35877e54d --- /dev/null +++ b/ddprof-lib/src/main/cpp/stackWalker.inline.h @@ -0,0 +1,58 @@ +/* + * Copyright The async-profiler authors + * Copyright 2026 Datadog, Inc + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _STACKWALKER_INLINE_H +#define _STACKWALKER_INLINE_H + +#include "stackWalker.h" +#include "hotspot/vmStructs.h" +#include "safeAccess.h" + +#include + +inline constexpr uintptr_t MAX_WALK_SIZE = 0x100000; +inline constexpr intptr_t MAX_FRAME_SIZE_WORDS = StackWalkValidation::MAX_FRAME_SIZE / sizeof(void*); // 0x8000 = 32768 words + +// AArch64: on Linux, frame link is stored at the top of the frame, +// while on macOS, frame link is at the bottom. +inline uintptr_t defaultSenderSP(uintptr_t sp, uintptr_t fp) { +#ifdef __APPLE__ + return sp + 2 * sizeof(void*); +#else + return fp; +#endif +} + +inline void fillFrame(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, const char* name) { + frame.bci = type; + frame.method_id = (jmethodID)name; +} + +// Overload for RemoteFrameInfo* (passed as void* to support both char* and RemoteFrameInfo*) +inline void fillFrame(ASGCT_CallFrame& frame, int bci, void* method_id_ptr) { + frame.bci = bci; + frame.method_id = (jmethodID)method_id_ptr; +} + +inline void fillFrame(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, u32 class_id) { + frame.bci = type; + frame.method_id = (jmethodID)(uintptr_t)class_id; +} + +inline void fillFrame(ASGCT_CallFrame& frame, FrameTypeId type, int bci, jmethodID method) { + frame.bci = FrameType::encode(type, bci); + frame.method_id = method; +} + +inline jmethodID getMethodId(VMMethod* method) { + if (!StackWalkValidation::inDeadZone(method) && StackWalkValidation::aligned((uintptr_t)method) + && SafeAccess::isReadableRange(method, VMMethod::type_size())) { + return method->validatedId(); + } + return NULL; +} + +#endif // _STACKWALKER_INLINE_H diff --git a/ddprof-lib/src/main/cpp/utils.h b/ddprof-lib/src/main/cpp/utils.h index 1d0473e5c..ecc4d2b00 100644 --- a/ddprof-lib/src/main/cpp/utils.h +++ b/ddprof-lib/src/main/cpp/utils.h @@ -32,4 +32,5 @@ inline size_t align_up(size_t size, size_t alignment) noexcept { return align_down(size + alignment - 1, alignment); } + #endif // _UTILS_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/vmEntry.cpp b/ddprof-lib/src/main/cpp/vmEntry.cpp index b3b4d5159..84acd256e 100644 --- a/ddprof-lib/src/main/cpp/vmEntry.cpp +++ b/ddprof-lib/src/main/cpp/vmEntry.cpp @@ -8,7 +8,7 @@ #include "vmEntry.h" #include "arguments.h" #include "context.h" -#include "j9/j9Ext.h" +#include "j9/j9Support.h" #include "jniHelper.h" #include "jvmThread.h" #include "libraries.h" @@ -17,6 +17,7 @@ #include "profiler.h" #include "safeAccess.h" #include "hotspot/vmStructs.h" +#include "hotspot/jitCodeCache.h" #include #include #include @@ -242,7 +243,7 @@ bool VM::initShared(JavaVM* vm) { Libraries *libraries = Libraries::instance(); libraries->updateSymbols(false); - _openj9 = !_hotspot && J9Ext::initialize( + _openj9 = !_hotspot && J9Support::initialize( _jvmti, libraries->resolveSymbol("j9thread_self*")); if (_openj9) { @@ -428,8 +429,8 @@ bool VM::initProfilerBridge(JavaVM *vm, bool attach) { callbacks.VMDeath = VMDeath; callbacks.ClassLoad = ClassLoad; callbacks.ClassPrepare = ClassPrepare; - callbacks.CompiledMethodLoad = Profiler::CompiledMethodLoad; - callbacks.DynamicCodeGenerated = Profiler::DynamicCodeGenerated; + callbacks.CompiledMethodLoad = JitCodeCache::CompiledMethodLoad; + callbacks.DynamicCodeGenerated = JitCodeCache::DynamicCodeGenerated; callbacks.ThreadStart = Profiler::ThreadStart; callbacks.ThreadEnd = Profiler::ThreadEnd; callbacks.SampledObjectAlloc = ObjectSampler::SampledObjectAlloc; diff --git a/ddprof-lib/src/main/cpp/vmEntry.h b/ddprof-lib/src/main/cpp/vmEntry.h index 6072fcaa5..322436558 100644 --- a/ddprof-lib/src/main/cpp/vmEntry.h +++ b/ddprof-lib/src/main/cpp/vmEntry.h @@ -68,7 +68,7 @@ typedef struct RemoteFrameInfo { #endif } RemoteFrameInfo; -typedef struct { +typedef struct _asgct_callframe { jint bci; LP64_ONLY(jint padding;) union { diff --git a/ddprof-lib/src/main/cpp/zing/zingSupport.cpp b/ddprof-lib/src/main/cpp/zing/zingSupport.cpp index 141758861..bd8459206 100644 --- a/ddprof-lib/src/main/cpp/zing/zingSupport.cpp +++ b/ddprof-lib/src/main/cpp/zing/zingSupport.cpp @@ -26,3 +26,4 @@ void* ZingSupport::initialize(jthread thread) { return (void*)env->GetLongField(thread, eetop); } +