Fix paltest_pal_sxs_test1 on illumos (#105207)

* Fix RPATH for exception handling PAL test

* Don't use alternate stack on illumos or Solaris.

When .NET translates SIGSEV to NullReferenceException, it does not return
from the signal handler. Instead it resumes execution at the catch handler
for the exception. This is not recommend by the manpage for sigaction(2):

> It is not recommended that [the ucontext] arg be used by the handler to
> restore the context from before the signal delivery.

The practical effect of resuming execution without returning from a handler
is that the alternate stack will not be used for subsequent signal delivery.
This is in contrast to the behavior on linux, which will always use the
alternate stack if the stack pointer at the time of fault does not fall on
the alternate stack.

Since the alternate stack is only usable for a single exception, don't
bother using it for any exceptions.

Author: AustinWise

src/coreclr/pal/src/exception/signal.cpp

@@ -190,8 +190,13 @@ BOOL SEHInitializeSignals(CorUnix::CPalThread *pthrCurrent, DWORD flags)
         handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv);
 #else
         handle_signal(SIGTRAP, sigtrap_handler, &g_previous_sigtrap);
+        int additionalFlagsForSigSegv = 0;
+#ifndef TARGET_SUNOS
+        // On platforms that support signal handlers that don't return,
         // SIGSEGV handler runs on a separate stack so that we can handle stack overflow
-        handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv, SA_ONSTACK);
+        additionalFlagsForSigSegv |= SA_ONSTACK;
+#endif
+        handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv, additionalFlagsForSigSegv);
 
         if (!pthrCurrent->EnsureSignalAlternateStack())
         {
@@ -344,7 +349,7 @@ Return :
 --*/
 bool IsRunningOnAlternateStack(void *context)
 {
-#if HAVE_MACH_EXCEPTIONS
+#if HAVE_MACH_EXCEPTIONS || defined(TARGET_SUNOS)
     return false;
 #else
     bool isRunningOnAlternateStack;

src/coreclr/pal/tests/palsuite/exception_handling/pal_sxs/test1/CMakeLists.txt

@@ -5,16 +5,14 @@ endif(CLR_CMAKE_HOST_UNIX)
 # Set the RPATH of paltest_pal_sxs_test1 so that it can find dependencies without needing to set LD_LIBRARY
 # For more information: http://www.cmake.org/Wiki/CMake_RPATH_handling.
 if(CORECLR_SET_RPATH)
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
   if(CLR_CMAKE_HOST_OSX)
     set(CMAKE_BUILD_WITH_INSTALL_NAME_DIR ON)
     set(CMAKE_INSTALL_NAME_DIR "@rpath")
-    set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
     set(CMAKE_INSTALL_RPATH "@loader_path")
-  endif(CLR_CMAKE_HOST_OSX)
-  if(CLR_CMAKE_HOST_LINUX OR CLR_CMAKE_HOST_HAIKU)
-    set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  else()
     set(CMAKE_INSTALL_RPATH "\$ORIGIN")
-  endif(CLR_CMAKE_HOST_LINUX OR CLR_CMAKE_HOST_HAIKU)
+  endif(CLR_CMAKE_HOST_OSX)
 endif(CORECLR_SET_RPATH)
 
 # Test DLL1