[release/8.0] Don't shut down event pipe in DLLs on Windows (#92044)

Trying to enable EventSourceSupport in a shared library project on Native AOT currently results in a build failure since this is actively blocked in the targets. But EventSourceSupport/event pipe mostly work, with some issues (#91762).

The blocking makes it impossible for anyone to run diagnostics on their shared library. This is a regression from .NET 7, where it was at least possible to get GC ETW events in PerfView on Windows.

This changes the blocking to a suppressible warning, and fixes and issue that was causing a shutdown hang.

src/coreclr/nativeaot/Bootstrap/main.cpp

@@ -93,7 +93,7 @@ static char& __unbox_z = __stop___unbox;
 
 #endif // _MSC_VER
 
-extern "C" bool RhInitialize();
+extern "C" bool RhInitialize(bool isDll);
 extern "C" void RhSetRuntimeInitializationCallback(int (*fPtr)());
 
 extern "C" bool RhRegisterOSModule(void * pModule,
@@ -164,7 +164,13 @@ extern "C" void __managed__Startup();
 
 static int InitializeRuntime()
 {
-    if (!RhInitialize())
+    if (!RhInitialize(
+#ifdef NATIVEAOT_DLL
+        /* isDll */ true
+#else
+        /* isDll */ false
+#endif
+        ))
         return -1;
 
     void * osModule = PalGetModuleHandleFromPointer((void*)&NATIVEAOT_ENTRYPOINT);

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Publish.targets

@@ -50,8 +50,7 @@
       Text="RuntimeIdentifier is required for native compilation. Try running dotnet publish with the -r option value specified." />
     <Error Condition="'$(GeneratePackageOnBuild)' == 'true'" Text="GeneratePackageOnBuild is not supported for native compilation." />
     <Error Condition="'$(OutputType)' != 'Library' and '$(NativeLib)' != '' and '$(CustomNativeMain)' != 'true'" Text="NativeLib requires OutputType=Library." />
-    <!-- See https://github.com/dotnet/runtime/issues/89346 for details -->
-    <Error Condition="'$(NativeLib)' != '' and '$(EventSourceSupport)' == 'true'" Text="EventSource is not supported when compiling to a native library. Set EventSourceSupport to false." />
+    <Warning Condition="'$(NativeLib)' != '' and '$(EventSourceSupport)' == 'true' and '$(_SuppressNativeLibEventSourceWarning)' != 'true'" Text="EventSource is not supported or recommended when compiling to a native library. Please go to https://github.com/dotnet/runtime/issues/91762 for more details." />
 
     <Error Condition="'$(PublishTrimmed)' == 'false'" Text="PublishTrimmed is implied by native compilation and cannot be disabled." />
 

src/coreclr/nativeaot/Runtime/startup.cpp

@@ -297,6 +297,10 @@ static void UninitDLL()
 Thread* g_threadPerformingShutdown = NULL;
 #endif
 
+#if defined(_WIN32) && defined(FEATURE_PERFTRACING)
+bool g_safeToShutdownTracing;
+#endif
+
 static void __cdecl OnProcessExit()
 {
 #ifdef _WIN32
@@ -309,8 +313,16 @@ static void __cdecl OnProcessExit()
 #endif
 
 #ifdef FEATURE_PERFTRACING
-    EventPipe_Shutdown();
-    DiagnosticServer_Shutdown();
+#ifdef _WIN32
+    // We forgo shutting down event pipe if it wouldn't be safe and could lead to a hang.
+    // If there was an active trace session, the trace will likely be corrupted without
+    // orderly shutdown. See https://github.com/dotnet/runtime/issues/89346.
+    if (g_safeToShutdownTracing)
+#endif
+    {
+        EventPipe_Shutdown();
+        DiagnosticServer_Shutdown();
+    }
 #endif
 }
 
@@ -348,7 +360,7 @@ void RuntimeThreadShutdown(void* thread)
 #endif
 }
 
-extern "C" bool RhInitialize()
+extern "C" bool RhInitialize(bool isDll)
 {
     if (!PalInit())
         return false;
@@ -357,6 +369,10 @@ extern "C" bool RhInitialize()
     atexit(&OnProcessExit);
 #endif
 
+#if defined(_WIN32) && defined(FEATURE_PERFTRACING)
+    g_safeToShutdownTracing = !isDll;
+#endif
+
     if (!InitDLL(PalGetModuleHandleFromPointer((void*)&RhInitialize)))
         return false;