Prevent unwinding through stack bottom

When processing unhandled exception on the most recent Alpine 3.17,
the libunwind doesn't stop at the bottom
frame of the main thread (the caller of `main`) and tries to unwind
further. The reason is that the method is missing dwarf unwind
information, so the libunwind falls back to using RBP chain, but the RBP
points to a garbage and so it ends up crashing with SIGSEGV.

While the missing DWARF unwind info seems to be a bug in the Alpine 3.17
(older ones work fine), we can prevent issues like this by stopping at
the hosting API boundary and not trying to unwind past that. This is
what this PR does.

Author: janvorli

src/coreclr/dlls/mscoree/exports.cpp

@@ -23,6 +23,25 @@
 
 #define ASSERTE_ALL_BUILDS(expr) _ASSERTE_ALL_BUILDS((expr))
 
+#ifdef TARGET_UNIX
+#define NO_HOSTING_API_FRAME_ADDRESS ((void*)ULONG_PTR_MAX)
+void* g_hostingApiFrameAddress = NO_HOSTING_API_FRAME_ADDRESS;
+
+class HostingApiFrameHolder
+{
+public:
+    HostingApiFrameHolder(void* frameAddress)
+    {
+        g_hostingApiFrameAddress = frameAddress;
+    }
+
+    ~HostingApiFrameHolder()
+    {
+        g_hostingApiFrameAddress = NO_HOSTING_API_FRAME_ADDRESS;
+    }
+};
+#endif // TARGET_UNIX
+
 // Holder for const wide strings
 typedef NewArrayHolder<const WCHAR> ConstWStringHolder;
 
@@ -212,6 +231,10 @@ int coreclr_initialize(
     bool hostPolicyEmbedded = false;
     PInvokeOverrideFn* pinvokeOverride = nullptr;
 
+#ifdef TARGET_UNIX
+    HostingApiFrameHolder apiFrameHolder(__builtin_frame_address(0));
+#endif
+
     ConvertConfigPropertiesToUnicode(
         propertyKeys,
         propertyValues,
@@ -435,6 +458,10 @@ int coreclr_execute_assembly(
     }
     *exitCode = -1;
 
+#ifdef TARGET_UNIX
+    HostingApiFrameHolder apiFrameHolder(__builtin_frame_address(0));
+#endif
+
     ICLRRuntimeHost4* host = reinterpret_cast<ICLRRuntimeHost4*>(hostHandle);
 
     ConstWStringArrayHolder argvW;

src/coreclr/vm/exceptionhandling.cpp

@@ -4523,6 +4523,8 @@ VOID UnwindManagedExceptionPass2(PAL_SEHException& ex, CONTEXT* unwindStartConte
     EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
 }
 
+extern void* g_hostingApiFrameAddress;
+
 //---------------------------------------------------------------------------------------
 //
 // This functions performs dispatching of a managed exception.
@@ -4724,7 +4726,8 @@ VOID DECLSPEC_NORETURN UnwindManagedExceptionPass1(PAL_SEHException& ex, CONTEXT
 
             STRESS_LOG2(LF_EH, LL_INFO100, "Processing exception at native frame: IP = %p, SP = %p \n", controlPc, sp);
 
-            if (controlPc == 0)
+            // Consider the exception unhandled if the unwinding cannot proceed further or if it went past the coreclr_initialize or coreclr_execute_assembly
+            if ((controlPc == 0) || (sp > (UINT_PTR)g_hostingApiFrameAddress))
             {
                 if (!GetThread()->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException))
                 {