Rework built-in COM->CLR stubs to build on the UnmanagedCallersOnly model (#126002)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Jan Kotas <jkotas@microsoft.com>

Author: 3 people

docs/design/coreclr/botr/guide-for-porting.md

@@ -368,8 +368,7 @@ Here is an annotated list of the stubs implemented for Unix on Arm64.
         FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP can be implemented as
         performance needs require.
 
-    7.  `ComCallPreStub`/ `COMToCLRDispatchHelper` /`GenericComCallStub` - not
-        necessary for non-Windows platforms at this time
+    7.  `ComCallPreStub` - not necessary for non-Windows platforms at this time
 
     8.  `TheUMEntryPrestub`/ `UMThunkStub` - used to enter the runtime from
         non-managed code through entrypoints generated from the

src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs

@@ -638,6 +638,19 @@ internal static void ClearNative(IntPtr pUnk)
                 Marshal.Release(pUnk);
             }
         }
+
+        internal static object GetObjectForComCallableWrapperIUnknown(IntPtr unk)
+        {
+            object? retObject = null;
+            GetObjectForComCallableWrapperIUnknown(unk, ObjectHandleOnStack.Create(ref retObject));
+            return retObject!;
+        }
+
+        [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "InterfaceMarshaler_GetObjectForComCallableWrapperIUnknown")]
+        private static partial void GetObjectForComCallableWrapperIUnknown(IntPtr unk, ObjectHandleOnStack retObject);
+
+        [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "InterfaceMarshaler_ValidateComVisibilityForIUnknown")]
+        internal static partial void ValidateComVisibilityForIUnknown(IntPtr unk);
     }  // class InterfaceMarshaler
 #endif // FEATURE_COMINTEROP
 

src/coreclr/debug/daccess/dacdbiimplstackwalk.cpp

@@ -603,42 +603,6 @@ HRESULT STDMETHODCALLTYPE DacDbiInterfaceImpl::EnumerateInternalFrames(VMPTR_Thr
                 frameData.vmCurrentAppDomainToken.SetHostPtr(pAppDomain);
 
                 MethodDesc * pMD = pFrame->GetFunction();
-    #if defined(FEATURE_COMINTEROP)
-                if (frameData.stubFrame.frameType == STUBFRAME_U2M)
-                {
-                    _ASSERTE(pMD == NULL);
-
-                    // U2M transition frame generally don't store the target MD because we know what the target
-                    // is by looking at the callee stack frame.  However, for reverse COM interop, we can try
-                    // to get the MD for the interface.
-                    //
-                    // Note that some reverse COM interop cases don't have an intermediate interface MD, so
-                    // pMD may still be NULL.
-                    //
-                    // Even if there is an MD on the ComMethodFrame, it could be in a different appdomain than
-                    // the ComMethodFrame itself.  The only known scenario is a cross-appdomain reverse COM
-                    // interop call.  We need to check for this case.  The end result is that GetFunction() and
-                    // GetFunctionToken() on ICDInternalFrame will return NULL.
-
-                    // Minidumps without full memory don't guarantee to capture the CCW since we can do without
-                    // it.  In this case, pMD will remain NULL.
-                    EX_TRY_ALLOW_DATATARGET_MISSING_MEMORY
-                    {
-                        if (pFrame->GetFrameIdentifier() == FrameIdentifier::ComMethodFrame)
-                        {
-                            ComMethodFrame * pCOMFrame = dac_cast<PTR_ComMethodFrame>(pFrame);
-                            PTR_VOID pUnkStackSlot     = pCOMFrame->GetPointerToArguments();
-                            PTR_IUnknown pUnk          = dac_cast<PTR_IUnknown>(*dac_cast<PTR_TADDR>(pUnkStackSlot));
-                            ComCallWrapper * pCCW      = ComCallWrapper::GetWrapperFromIP(pUnk);
-
-                            ComCallMethodDesc * pCMD = NULL;
-                            pCMD = dac_cast<PTR_ComCallMethodDesc>(pCOMFrame->ComMethodFrame::GetDatum());
-                            pMD  = pCMD->GetInterfaceMethodDesc();
-                        }
-                    }
-                    EX_END_CATCH_ALLOW_DATATARGET_MISSING_MEMORY
-                }
-    #endif // FEATURE_COMINTEROP
 
                 Module *     pModule = (pMD ? pMD->GetModule() : NULL);
                 DomainAssembly * pDomainAssembly = (pModule ? pModule->GetDomainAssembly() : NULL);
@@ -902,7 +866,7 @@ void DacDbiInterfaceImpl::InitFrameData(StackFrameIterator *   pIter,
         pFrameData->v.fVarArgs = pMD->IsVarArg();
 
         // Check if this is a NoMetadata method or if the method should be hidden.
-        // These methods should not be visible in the debugger both for convenience and 
+        // These methods should not be visible in the debugger both for convenience and
         // because they don't have backing metadata. For more information see comments in
         // MethodDesc::IsNoMetadata and MethodDesc::IsDiagnosticsHidden.
         pFrameData->v.fNoMetadata = pMD->IsNoMetadata() || pMD->IsDiagnosticsHidden();

src/coreclr/debug/ee/frameinfo.cpp

@@ -801,28 +801,8 @@ void FrameInfo::InitForM2UInternalFrame(CrawlFrame * pCF)
 void FrameInfo::InitForU2MInternalFrame(CrawlFrame * pCF)
 {
     _ASSERTE(pCF != NULL);
-    MethodDesc * pMDHint = NULL;
-
-#ifdef FEATURE_COMINTEROP
-    Frame * pFrame = pCF->GetFrame();
-    _ASSERTE(pFrame != NULL);
-
-
-    // For regular U2M PInvoke cases, we don't care about MD b/c it's just going to
-    // be the next frame.
-    // If we're a COM2CLR call, perhaps we can get the MD for the interface.
-    if (pFrame->GetFrameIdentifier() == FrameIdentifier::ComMethodFrame)
-    {
-        ComMethodFrame* pCOMFrame = dac_cast<PTR_ComMethodFrame> (pFrame);
-        ComCallMethodDesc* pCMD = reinterpret_cast<ComCallMethodDesc *> (pCOMFrame->ComMethodFrame::GetDatum());
-        pMDHint = pCMD->GetInterfaceMethodDesc();
-
-        // Some COM-interop cases don't have an intermediate interface method desc, so
-        // pMDHint may be null.
-    }
-#endif
 
-    InitFromStubHelper(pCF, pMDHint, STUBFRAME_U2M);
+    InitFromStubHelper(pCF, NULL, STUBFRAME_U2M);
     InitForScratchFrameInfo();
 }
 

src/coreclr/inc/CrstTypes.def

@@ -463,7 +463,7 @@ Crst UnresolvedClassLock
 End
 
 Crst WrapperTemplate
-    AcquiredBefore ExecutableAllocatorLock
+    AcquiredBefore LoaderHeap
 End
 
 Crst UMEntryThunkCache
@@ -536,4 +536,4 @@ End
 
 Crst CallStubCache
     AcquiredBefore LoaderHeap
-End
+End

src/coreclr/inc/clrtypes.h

@@ -215,9 +215,6 @@ static_assert(sizeof(CLR_BOOL) == 1);
 #define CLR_BOOL_MAX    BOOL_MAX
 #define CLR_BOOL_MIN    BOOL_MIN
 
-#define CLR_NAN_32 0xFFC00000
-#define CLR_NAN_64 0xFFF8000000000000LL
-
 // ================================================================================
 // Simple utility functions
 // ================================================================================

src/coreclr/inc/crsttypes_generated.h

@@ -235,7 +235,7 @@ int g_rgCrstLevelMap[] =
     2,          // CrstUnwindInfoTablePendingLock
     3,          // CrstUnwindInfoTablePublishLock
     3,          // CrstVSDIndirectionCellLock
-    2,          // CrstWrapperTemplate
+    3,          // CrstWrapperTemplate
 };
 
 // An array mapping CrstType to a stringized name.

src/coreclr/vm/CMakeLists.txt

@@ -662,7 +662,6 @@ elseif(CLR_CMAKE_TARGET_WIN32)
             ${ARCH_SOURCES_DIR}/CachedInterfaceDispatchCoreCLR.asm
             ${ARCH_SOURCES_DIR}/CallDescrWorkerAMD64.asm
             ${ARCH_SOURCES_DIR}/ComCallPreStub.asm
-            ${ARCH_SOURCES_DIR}/GenericComCallStubs.asm
             ${ARCH_SOURCES_DIR}/GenericCLRToCOMCallStubs.asm
             ${ARCH_SOURCES_DIR}/getstate.asm
             ${ARCH_SOURCES_DIR}/JitHelpers_Fast.asm

src/coreclr/vm/FrameTypes.h

@@ -19,9 +19,7 @@ FRAME_TYPE_NAME(SoftwareExceptionFrame)
 FRAME_TYPE_NAME(FuncEvalFrame)
 #endif // DEBUGGING_SUPPORTED
 #ifdef FEATURE_COMINTEROP
-FRAME_TYPE_NAME(ComMethodFrame)
 FRAME_TYPE_NAME(CLRToCOMMethodFrame)
-FRAME_TYPE_NAME(ComPrestubMethodFrame)
 #endif // FEATURE_COMINTEROP
 FRAME_TYPE_NAME(PInvokeCalliFrame)
 #ifdef FEATURE_HIJACK

src/coreclr/vm/amd64/AsmHelpers.asm

@@ -106,41 +106,6 @@ NESTED_ENTRY JIT_RareDisableHelper, _TEXT
 
 NESTED_END JIT_RareDisableHelper, _TEXT
 
-
-; extern "C" void setFPReturn(int fpSize, INT64 retVal);
-LEAF_ENTRY setFPReturn, _TEXT
-        cmp     ecx, 4
-        je      setFPReturn4
-        cmp     ecx, 8
-        jne     setFPReturnNot8
-        mov     [rsp+10h], rdx
-        movsd   xmm0, real8 ptr [rsp+10h]
-setFPReturnNot8:
-        REPRET
-
-setFPReturn4:
-        mov     [rsp+10h], rdx
-        movss   xmm0, real4 ptr [rsp+10h]
-        ret
-LEAF_END setFPReturn, _TEXT
-
-
-; extern "C" void getFPReturn(int fpSize, INT64 *retval);
-LEAF_ENTRY getFPReturn, _TEXT
-        cmp     ecx, 4
-        je      getFPReturn4
-        cmp     ecx, 8
-        jne     getFPReturnNot8
-        movsd   real8 ptr [rdx], xmm0
-getFPReturnNot8:
-        REPRET
-
-getFPReturn4:
-        movss   real4 ptr [rdx], xmm0
-        ret
-LEAF_END getFPReturn, _TEXT
-
-
 ; A JITted method's return address was hijacked to return to us here.
 ; VOID OnHijackTripThread()
 NESTED_ENTRY OnHijackTripThread, _TEXT