Add Symbolication to XUnitLogChecker (#83702)

src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs

@@ -363,7 +363,7 @@ static bool RunProcess(string fileName, string arguments, TextWriter outputWrite
         /// <param name="crashReportJsonFile">crash dump path</param>
         /// <param name="outputWriter">Stream for writing logs</param>
         /// <returns>true, if we can print the stack trace, otherwise false.</returns>
-        static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, StreamWriter outputWriter)
+        public static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, TextWriter outputWriter)
         {
             if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
             {
@@ -585,7 +585,7 @@ private static void AppendAddress(StringBuilder sb, string coreRoot, string nati
             }
         }
 
-        static bool TryPrintStackTraceFromDmp(string dmpFile, StreamWriter outputWriter)
+        public static bool TryPrintStackTraceFromDmp(string dmpFile, TextWriter outputWriter)
         {
             string targetArchitecture = Environment.GetEnvironmentVariable(TEST_TARGET_ARCHITECTURE_ENVIRONMENT_VAR);
             if (string.IsNullOrEmpty(targetArchitecture))

src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs

@@ -6,6 +6,9 @@
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Xml;
+using System.Xml.Linq;
+
+using CoreclrTestWrapperLib = CoreclrTestLib.CoreclrTestWrapperLib;
 
 public class XUnitLogChecker
 {
@@ -35,7 +38,6 @@ private enum TagCategory { OPENING, CLOSING }
 
     static int Main(string[] args)
     {
-        // Maybe add a '--help' flag that also gets triggered in this case.
         if (args.Length < 2)
         {
             Console.WriteLine("[XUnitLogChecker]: The path to the log file and"
@@ -77,9 +79,7 @@ static int Main(string[] args)
         }
 
         // If the final results log file is present, then we can assume everything
-        // went fine, and it's ready to go without any further processing. We just
-        // check the stats csv file to know how many tests were run, and display a
-        // brief summary of the work item.
+        // went fine, and it's ready to go without any further processing.
 
         if (File.Exists(finalLogPath))
         {
@@ -100,9 +100,7 @@ static int Main(string[] args)
             return FAILURE;
         }
 
-        // Declaring the enumerable to contain the log lines first because we
-        // might not be able to read on the first try due to locked resources
-        // on Windows. We will retry for up to one minute when this case happens.
+        // Read the tests run stats csv.
         IEnumerable<string>? workItemStats = TryReadFile(statsCsvPath);
 
         if (workItemStats is null)
@@ -142,14 +140,38 @@ static int Main(string[] args)
 
         PrintWorkItemSummary(numExpectedTests, workItemEndStatus);
 
+        // The third command-line argument is an optional path where dumps would
+        // be located. If passed, then search that path accordingly. Otherwise,
+        // just skip and finish running.
+
+        if (args.Length > 2)
+        {
+            string dumpsPath = args[2];
+
+            if (Directory.Exists(dumpsPath))
+            {
+                PrintStackTracesFromDumps(dumpsPath, tempLogPath);
+            }
+            else
+            {
+                Console.WriteLine("[XUnitLogChecker]: The provided dumps path"
+                                + $" '{dumpsPath}' was not able to be read or"
+                                + " found. Skipping stack traces search...");
+            }
+        }
+
         // Rename the temp log to the final log, so that Helix can use it without
         // knowing what transpired here.
         File.Move(tempLogPath, finalLogPath);
+        Console.WriteLine("[XUnitLogChecker]: Finished!");
         return SUCCESS;
     }
 
     static IEnumerable<string> TryReadFile(string filePath)
     {
+        // Declaring the enumerable to contain the log lines first because we
+        // might not be able to read on the first try due to locked resources
+        // on Windows. We will retry for up to one minute when this case happens.
         IEnumerable<string>? fileContents = null;
         Stopwatch fileReadStopwatch = Stopwatch.StartNew();
 
@@ -175,6 +197,18 @@ static IEnumerable<string> TryReadFile(string filePath)
         return fileContents;
     }
 
+    static void PrintMissingCrashPath(string wrapperName,
+                                      string crashFileType,
+                                      string crashFilePath)
+    {
+        Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did not complete"
+                        + $" successfully, but there was no {crashFileType} found."
+                        + " The XML log was fixed successfully though.");
+
+        Console.WriteLine($"[XUnitLogChecker]: Expected {crashFileType} path"
+                        + $" was '{crashFilePath}'");
+    }
+
     static void PrintWorkItemSummary(int numExpectedTests, int[] workItemEndStatus)
     {
         Console.WriteLine($"\n{workItemEndStatus[0]}/{numExpectedTests} tests run.");
@@ -347,5 +381,65 @@ static TagResult[] GetOrderedTagMatches(Match[] openingTags, Match[] closingTags
         }
         return result;
     }
+
+    static void PrintStackTracesFromDumps(string dumpsPath, string tempLogPath)
+    {
+        Console.WriteLine("[XUnitLogChecker]: Checking for dumps...");
+
+        // Read our newly fixed log to retrieve the time and date when the
+        // test was run. This is to exclude potentially existing older dumps
+        // that are not related to this test run.
+        XElement fixedLogTree = XElement.Load(tempLogPath);
+
+        // We know from the XUnitWrapperGenerator that the top element
+        // is the 'assembly' tag we're looking for.
+        var testRunDateTime = DateTime.ParseExact
+        (
+            fixedLogTree.Attribute("run-date-time").Value,
+            "yyyy-MM-dd HH:mm:ss",
+            System.Globalization.CultureInfo.InvariantCulture
+        );
+
+        IEnumerable<string> dumpsFound =
+            Directory.GetFiles(dumpsPath, "*coredump*.dmp")
+                     .Where(dmp => DateTime.Compare(File.GetCreationTime(dmp), testRunDateTime) >= 0);
+
+        if (dumpsFound.Count() == 0)
+        {
+            Console.WriteLine("[XUnitLogChecker]: No crash dumps found. Continuing...");
+            return ;
+        }
+
+        foreach (string dumpPath in dumpsFound)
+        {
+            if (OperatingSystem.IsWindows())
+            {
+                Console.WriteLine("[XUnitLogChecker]: Reading crash dump"
+                                + $" '{dumpPath}'...");
+                Console.WriteLine("[XUnitLogChecker]: Stack Trace Found:\n");
+
+                CoreclrTestWrapperLib.TryPrintStackTraceFromDmp(dumpPath,
+                                                                Console.Out);
+            }
+            else
+            {
+                string crashReportPath = $"{dumpPath}.crashreport.json";
+
+                if (!File.Exists(crashReportPath))
+                {
+                    Console.WriteLine("[XUnitLogChecker]: There was no crash"
+                                    + $" report for dump '{dumpPath}'. Skipping...");
+                    continue;
+                }
+
+                Console.WriteLine("[XUnitLogChecker]: Reading crash report"
+                                + $" '{crashReportPath}'...");
+                Console.WriteLine("[XUnitLogChecker]: Stack Trace Found:\n");
+
+                CoreclrTestWrapperLib.TryPrintStackTraceFromCrashReport(crashReportPath,
+                                                                        Console.Out);
+            }
+        }
+    }
 }
 

src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj

@@ -4,10 +4,15 @@
     <OutputType>Exe</OutputType>
     <TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup>
     <Compile Include="XUnitLogChecker.cs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Compile Include="../Coreclr.TestWrapper/CoreclrTestWrapperLib.cs" Link="CoreclrTestWrapperLib.cs" />
+    <Compile Include="../Coreclr.TestWrapper/MobileAppHandler.cs" Link="MobileAppHandler.cs" />
+  </ItemGroup>
 </Project>

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs

@@ -182,7 +182,6 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
         builder.AppendLine();
 
         builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args, testExclusionList);");
-        // builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new(TestCount.Count);");
         builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();");
         builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();");
         builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);");
@@ -190,14 +189,17 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
         builder.AppendLine();
 
         builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.tempLog.xml""))");
-        builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv""))");
-
+        builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{");
         CodeBuilder testExecutorBuilder = new();
         int totalTestsEmitted = 0;
 
         using (builder.NewBracesScope())
         {
             builder.AppendLine("statsCsvSw.WriteLine($\"{TestCount.Count},0,0,0\");");
+            // CAUTION NOTE: If this ever changes and the 'assembly' tag is no longer
+            // the topmost one in the temp log, XUnitLogChecker must be updated accordingly.
+            // Otherwise, it's going to fail when attempting to find dumps.
+            builder.AppendLine($@"summary.WriteHeaderToTempLog(""{assemblyName}"", tempLogSw);");
 
             ITestReporterWrapper reporter =
                 new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder");
@@ -243,6 +245,9 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
                 testExecutorBuilder.AppendLine("}");
                 testExecutorBuilder.AppendLine();
             }
+
+            testExecutorBuilder.AppendLine("}");
+            builder.AppendLine("tempLogSw.WriteLine(\"</assembly>\");");
         }
         builder.AppendLine();
 

src/tests/Common/XUnitWrapperLibrary/TestSummary.cs

@@ -104,6 +104,16 @@ public string ToXmlString()
     private readonly List<TestResult> _testResults = new();
     private DateTime _testRunStart = DateTime.Now;
 
+    public void WriteHeaderToTempLog(string assemblyName, StreamWriter tempLogSw)
+    {
+        // We are writing down both, date and time, in the same field here because
+        // it's much simpler to parse later on in the XUnitLogChecker.
+        tempLogSw.WriteLine("<assembly\n"
+                        + $"    name=\"{assemblyName}\"\n"
+                        + $"    test-framework=\"XUnitWrapperGenerator-generated-runner\"\n"
+                        + $"    run-date-time=\"{_testRunStart.ToString("yyyy-MM-dd HH:mm:ss")}\">");
+    }
+
     public void ReportPassedTest(string name,
                                  string containingTypeName,
                                  string methodName,
@@ -174,7 +184,7 @@ public string GetTestResultOutput(string assemblyName)
     name=""{assemblyName}""
     test-framework=""XUnitWrapperGenerator-generated-runner""
     run-date=""{_testRunStart.ToString("yyyy-MM-dd")}""
-    run-time=""{_testRunStart.ToString("hh:mm:ss")}""
+    run-time=""{_testRunStart.ToString("HH:mm:ss")}""
     time=""{totalRunSeconds}""
     total=""{_testResults.Count}""
     passed=""{PassedTests}""

src/tests/Common/helixpublishwitharcade.proj

@@ -400,9 +400,14 @@
     </ItemGroup>
 
     <PropertyGroup>
-      <XUnitLogCheckerCommand Condition="'$(TestWrapperTargetsWindows)' != 'true'">dotnet %24HELIX_CORRELATION_PAYLOAD/XUnitLogChecker/</XUnitLogCheckerCommand>
-      <XUnitLogCheckerCommand Condition="'$(TestWrapperTargetsWindows)' == 'true'">dotnet %25HELIX_CORRELATION_PAYLOAD%25/XUnitLogChecker/</XUnitLogCheckerCommand>
-      <XUnitLogCheckerCommand>$(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName)</XUnitLogCheckerCommand>
+      <XUnitLogCheckerHelixPath Condition="'$(TestWrapperTargetsWindows)' != 'true'">%24HELIX_CORRELATION_PAYLOAD/</XUnitLogCheckerHelixPath>
+      <XUnitLogCheckerHelixPath Condition="'$(TestWrapperTargetsWindows)' == 'true'">%25HELIX_CORRELATION_PAYLOAD%25/</XUnitLogCheckerHelixPath>
+      <XUnitLogCheckerHelixPath>$(XUnitLogCheckerHelixPath)XUnitLogChecker/</XUnitLogCheckerHelixPath>
+
+      <XUnitLogCheckerArgs>$(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName)</XUnitLogCheckerArgs>
+      <XUnitLogCheckerArgs Condition="'$(TestWrapperTargetsWindows)' != 'true'">$(XUnitLogCheckerArgs) %24HELIX_DUMP_FOLDER</XUnitLogCheckerArgs>
+      <XUnitLogCheckerArgs Condition="'$(TestWrapperTargetsWindows)' == 'true'">$(XUnitLogCheckerArgs) %25HELIX_DUMP_FOLDER%25</XUnitLogCheckerArgs>
+      <XUnitLogCheckerCommand>dotnet $(XUnitLogCheckerHelixPath)XUnitLogChecker.dll $(XUnitLogCheckerArgs)</XUnitLogCheckerCommand>
     </PropertyGroup>
 
     <ItemGroup>