Log information about crashes during integration tests

src/VisualStudio/IntegrationTest/TestUtilities/EventLogCollector.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics.Eventing.Reader;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace Microsoft.VisualStudio.IntegrationTest.Utilities
+{
+    /// <summary>
+    /// Helper class to read the Application Event Log for Watson and .NetRuntime entries
+    /// </summary>
+    internal static class EventLogCollector
+    {
+        /// <summary>
+        /// The name of the Event Log to query
+        /// </summary>
+        private const string EventLogName = "Application";
+
+        /// <summary>
+        /// We want to get either the entries for the past day or the last 5 (whichever has a greater count)
+        /// </summary>
+        private const int MinimumEntries = 5;
+
+        /// <summary>
+        /// We want to get either the entries for the past day or the last 5 (whichever has a greater count)
+        /// </summary>
+        private const int DaysToGetEventsFor = 1;
+
+        /// <summary>
+        /// We don't want to add events that are older than a week
+        /// </summary>
+        private const int MaxDaysToGetEventsFor = 7;
+
+        /// <summary>
+        /// For Watson, the Provider Name in the Event Log is "Windows Error Reporting"
+        /// </summary>
+        private const string WatsonProviderName = "Windows Error Reporting";
+
+        /// <summary>
+        /// For Watson, the Event Id in the Event Log that we are interested in is 1001
+        /// </summary>
+        private const int WatsonEventId = 1001;
+
+        /// <summary>
+        /// Each entry in the EventLog has 22 Properties: 0-bucketId, 1-eventTypeId, 2-eventName, 3-response, 4-cabId, 5:14-bucketParameters P1:P10,
+        /// 15-attachedFiles, 16-location, 17-analysisSymbol, 18-recheck, 19-reportId, 20-reportStatus, 21-hashedBucket
+        /// </summary>
+        private const int WatsonEventLogEntryPropertyCount = 22;
+
+        /// <summary>
+        /// FaultBucket is the first property on the log entry
+        /// </summary>
+        private const int FaultBucketIndex = 0;
+
+        /// <summary>
+        /// For .DotNetRuntime, the Provider Name in the Event Log
+        /// </summary>
+        private const string DotNetProviderName = ".NET Runtime";
+
+        /// <summary>
+        /// The Event Id in the Event Log for .DotNetRuntime that we want to scope down to
+        /// 1023 - ERT_UnmanagedFailFast, 1025 - ERT_ManagedFailFast, 1026 - ERT_UnhandledException, 1027 - ERT_StackOverflow, 1028 - ERT_CodeContractFailed
+        /// </summary>
+        private static readonly ImmutableArray<int> s_dotNetEventId = ImmutableArray.Create(1023, 1024, 1025, 1026, 1027, 1028);
+
+        /// <summary>
+        /// List of EventNames to exclude from our search in the Event Log
+        /// </summary>
+        internal static HashSet<string> ExcludedEventNames = new HashSet<string>()
+        {
+            "VisualStudioNonFatalErrors",
+            "VisualStudioNonFatalErrors2"
+        };
+
+        /// <summary>
+        /// List of VS EXEs to search in the Event Log for
+        /// </summary>
+        internal static HashSet<string> VsRelatedExes = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+        {
+                "devenv.exe",
+                "csc.exe",
+                "csi.exe",
+                "git.exe",
+                "msbuild.exe",
+                "MSBuildTaskHost.exe",
+                "mspdbsrv.exe",
+                "MStest.exe",
+                "ServiceHub.Host.CLR.exe",
+                "ServiceHub.Host.CLR.x64.exe",
+                "ServiceHub.Host.CLR.x86.exe",
+                "ServiceHub.IdentityHost.exe",
+                "ServiceHub.RoslynCodeAnalysisService32.exe",
+                "ServiceHub.SettingsHost.exe",
+                "ServiceHub.VSDetouredHost.exe",
+                "vbc.exe",
+                "vbc2.exe",
+                "VBCSCompiler.exe",
+                "VStest.Console.Exe",
+                "VSTest.DiscoveryEngine.exe",
+                "VSTest.DiscoveryEngine.x86.exe",
+                "vstest.executionengine.appcontainer.exe",
+                "vstest.executionengine.appcontainer.x86.exe",
+                "vstest.executionengine.clr20.exe",
+                "VSTest.executionEngine.exe",
+                "VSTest.executionEngine.x86.exe",
+            };
+
+        /// <summary>
+        /// Get the WER entries for VS and VS related EXEs from the Event Log and write them to a file
+        /// </summary>
+        internal static void TryWriteWatsonEntriesToFile(string filePath)
+        {
+            try
+            {
+                // Use a HashSet to make sure the entries we add aren't duplicates (calls the Equals override from FeedbackItemWatsonEntry)
+                var watsonEntries = new HashSet<FeedbackItemWatsonEntry>();
+
+                // We need to search in the Application Event Log, since that's where Watson logs entries
+                var eventLogQuery = new EventLogQuery(EventLogName, PathType.LogName)
+                {
+                    // Read events in descending order, so we can get either the last 5 entries or the past day of entries, whichever has a bigger count
+                    ReverseDirection = true
+                };
+
+                var eventLogReader = new EventLogReader(eventLogQuery);
+                EventRecord eventLogRecord;
+                var watsonEntriesCount = 0;
+                while ((eventLogRecord = eventLogReader.ReadEvent()) != null)
+                {
+                    // We only want the last 5 entries or the past day of entries, whichever has a bigger count
+                    if (IsLastDayOrLastFiveRecentEntry(eventLogRecord, watsonEntriesCount))
+                    {
+                        // Filter the entries by Watson specific ones for VS EXEs
+                        if (IsValidWatsonEntry(eventLogRecord))
+                        {
+                            var entry = new FeedbackItemWatsonEntry(eventLogRecord);
+                            watsonEntries.Add(entry);
+
+                            // If the entry doesn't have a valid BucketId, we don't want it to count towards the maxCount we send
+                            if (!string.IsNullOrWhiteSpace(GetEventRecordPropertyToString(eventLogRecord, FaultBucketIndex)))
+                            {
+                                watsonEntriesCount++;
+                            }
+                        }
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+
+                if (watsonEntries.Any())
+                {
+                    var watsonEntriesStringBuilder = new StringBuilder();
+                    foreach (var entry in watsonEntries)
+                    {
+                        watsonEntriesStringBuilder.AppendLine($"Event Time (UTC): {entry.EventTime}");
+                        watsonEntriesStringBuilder.AppendLine($"Application Name: {entry.ApplicationName}");
+                        watsonEntriesStringBuilder.AppendLine($"Application Version: {entry.ApplicationVersion}");
+                        watsonEntriesStringBuilder.AppendLine($"Faulting Module: {entry.FaultingModule}");
+                        watsonEntriesStringBuilder.AppendLine($"Faulting Module Version: {entry.FaultingModuleVersion}");
+                        watsonEntriesStringBuilder.AppendLine($"Event Name: {entry.EventName}");
+                        watsonEntriesStringBuilder.AppendLine($"Cab Id: {entry.CabId}");
+                        watsonEntriesStringBuilder.AppendLine($"Fault Bucket: {entry.FaultBucket}");
+                        watsonEntriesStringBuilder.AppendLine($"Hashed Bucket: {entry.HashedBucket}");
+                        watsonEntriesStringBuilder.AppendLine($"Watson Report Id: {entry.WatsonReportId}");
+                        watsonEntriesStringBuilder.AppendLine();
+                    }
+
+                    File.WriteAllText(filePath, watsonEntriesStringBuilder.ToString());
+                }
+            }
+            catch (Exception ex)
+            {
+                File.WriteAllText(filePath, ex.ToString());
+            }
+        }
+
+        /// <summary>
+        /// Get the .NetRuntime entries from the Event Log and write them to a file
+        /// </summary>
+        internal static void TryWriteDotNetEntriesToFile(string filePath)
+        {
+            try
+            {
+                var dotNetEntries = new HashSet<FeedbackItemDotNetEntry>();
+
+                // We need to search in the Application Event Log, since that's where .NetRuntime logs entries
+                var eventLogQuery = new EventLogQuery(EventLogName, PathType.LogName)
+                {
+                    // Read events in descending order, so we can get either the last 5 entries or the past day of entries, whichever has a bigger count
+                    ReverseDirection = true
+                };
+
+                var eventLogReader = new EventLogReader(eventLogQuery);
+                EventRecord eventLogRecord;
+                while ((eventLogRecord = eventLogReader.ReadEvent()) != null)
+                {
+                    // We only want the last 5 entries or the past day of entries, whichever has a bigger count
+                    if (IsLastDayOrLastFiveRecentEntry(eventLogRecord, dotNetEntries.Count))
+                    {
+                        // Filter the entries by .NetRuntime specific ones
+                        FeedbackItemDotNetEntry entry = null;
+                        if (IsValidDotNetEntry(eventLogRecord, ref entry))
+                        {
+                            dotNetEntries.Add(entry);
+                        }
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+
+                if (dotNetEntries.Any())
+                {
+                    var dotNetEntriesStringBuilder = new StringBuilder();
+                    foreach (var entry in dotNetEntries)
+                    {
+                        dotNetEntriesStringBuilder.AppendLine($"Event Time (UTC): {entry.EventTime}");
+                        dotNetEntriesStringBuilder.AppendLine($"Event ID: {entry.EventId}");
+                        dotNetEntriesStringBuilder.AppendLine($"Data: {entry.Data.Replace("\n", "\r\n")}");
+                        dotNetEntriesStringBuilder.AppendLine();
+                    }
+
+                    File.WriteAllText(filePath, dotNetEntriesStringBuilder.ToString());
+                }
+            }
+            catch (Exception ex)
+            {
+                File.WriteAllText(filePath, ex.ToString());
+            }
+        }
+
+        /// <summary>
+        /// Returns true if this is one of the last 5 entries over the past week or the past day of entries, whichever has a bigger count
+        /// </summary>
+        /// <param name="eventLogRecord">Event entry to be checked</param>
+        /// <param name="entriesCount">List of already valid entries</param>
+        private static bool IsLastDayOrLastFiveRecentEntry(EventRecord eventLogRecord, int entriesCount)
+        {
+            // This is local time (it will be later converted to UTC when we send the feedback)
+            if (eventLogRecord.TimeCreated.HasValue
+                && (eventLogRecord.TimeCreated.Value > DateTime.Now.AddDays(-MaxDaysToGetEventsFor))
+                && ((eventLogRecord.TimeCreated.Value > DateTime.Now.AddDays(-DaysToGetEventsFor)) || (entriesCount < MinimumEntries)))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Verifies if an entry is a valid Watson one by checking:
+        /// the provider, if it's for VS EXEs or the installer EXEs, and it's not a VisualStudioNonFatalErrors or VisualStudioNonFatalErrors2
+        /// </summary>
+        /// <param name="eventLogRecord">Entry to be checked</param>
+        private static bool IsValidWatsonEntry(EventRecord eventLogRecord)
+        {
+            if (StringComparer.InvariantCultureIgnoreCase.Equals(eventLogRecord.ProviderName, WatsonProviderName)
+                && (eventLogRecord.Id == WatsonEventId)
+                && (eventLogRecord.Properties.Count >= WatsonEventLogEntryPropertyCount)
+                && (!ExcludedEventNames.Contains(GetEventRecordPropertyToString(eventLogRecord, FeedbackItemWatsonEntry.EventNameIndex)))
+                && VsRelatedExes.Contains(GetEventRecordPropertyToString(eventLogRecord, FeedbackItemWatsonEntry.ApplicationNameIndex)))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Verifies if an entry is a valid .NET one by checking:
+        /// the provider, if it's for certain event log IDs and for VS related EXEs
+        /// </summary>
+        /// <param name="eventLogRecord">Entry to be checked</param>
+        private static bool IsValidDotNetEntry(EventRecord eventLogRecord, ref FeedbackItemDotNetEntry dotNetEntry)
+        {
+            if (StringComparer.InvariantCultureIgnoreCase.Equals(eventLogRecord.ProviderName, DotNetProviderName)
+                 && s_dotNetEventId.Contains(eventLogRecord.Id))
+            {
+                dotNetEntry = new FeedbackItemDotNetEntry(eventLogRecord);
+                foreach (var app in VsRelatedExes)
+                {
+                    if (dotNetEntry.Data.IndexOf(app, StringComparison.InvariantCultureIgnoreCase) >= 0)
+                    {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Given the EventRecord and the index in it, get its value as a string (empty if it's null)
+        /// </summary>
+        /// <param name="eventLogRecord">EventRecord</param>
+        /// <param name="index">Index in the EventRecord</param>
+        /// <returns>string if not null or string.Empty</returns>
+        internal static string GetEventRecordPropertyToString(EventRecord eventLogRecord, int index)
+        {
+            if (eventLogRecord.Properties[index].Value == null)
+            {
+                return string.Empty;
+            }
+            else
+            {
+                return eventLogRecord.Properties[index].Value.ToString();
+            }
+        }
+    }
+}

src/VisualStudio/IntegrationTest/TestUtilities/FeedbackItemDotNetEntry.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics.Eventing.Reader;
+using System.Linq;
+using System.Runtime.Serialization;
+
+namespace Microsoft.VisualStudio.IntegrationTest.Utilities
+{
+    /// <summary>
+    /// Mapper for the .NetRuntime entry in the Event Log
+    /// </summary>
+    [DataContract]
+    internal class FeedbackItemDotNetEntry
+    {
+        /// <summary>
+        /// The time the event happend (UTC)
+        /// </summary>
+        [DataMember(Name = "eventTime")]
+        public DateTime EventTime { get; set; }
+
+        /// <summary>
+        /// The .NET Runtime event id (this is set by .NET and we get it from the Event Log, so we can better differenciate between them)
+        /// As defined in CLR code:  ndp\clr\src\vm\eventreporter.cpp, these IDs are:
+        /// 1023 - ERT_UnmanagedFailFast, 1025 - ERT_ManagedFailFast, 1026 - ERT_UnhandledException, 1027 - ERT_StackOverflow, 1028 - ERT_CodeContractFailed
+        /// </summary>
+        [DataMember(Name = "eventId")]
+        public int EventId { get; set; }
+
+        /// <summary>
+        /// The event log properties to be passed as one string. E.g.
+        /// Application: CSAv.exe, Framework version: v4.0.30319,
+        /// Description: The application requested termination through System.Environment.FailFast(string message)
+        /// Stack: at CSAv.Program.GetModuleFileName(IntPtr, Int32, Int32)
+        /// </summary>
+        [DataMember(Name = "data")]
+        public string Data { get; set; }
+
+        /// <summary>
+        /// Constructor for the FeedbackItemDotNetEntry based on an EventRecord from the EventLog
+        /// </summary>
+        public FeedbackItemDotNetEntry(EventRecord eventLogRecord)
+        {
+            EventTime = eventLogRecord.TimeCreated.Value.ToUniversalTime();
+            EventId = eventLogRecord.Id;
+            Data = string.Join(";", eventLogRecord.Properties.Select(pr => pr.Value ?? string.Empty));
+        }
+
+        /// <summary>
+        /// Used to make sure we aren't adding dupe entries to the list of Watson entries
+        /// </summary>
+        public override bool Equals(object obj)
+        {
+            if ((obj is FeedbackItemDotNetEntry dotNetEntry)
+                && (EventId == dotNetEntry.EventId)
+                && (Data == dotNetEntry.Data))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return EventId.GetHashCode() ^ Data.GetHashCode();
+        }
+    }
+}

src/VisualStudio/IntegrationTest/TestUtilities/FeedbackItemWatsonEntry.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics.Eventing.Reader;
+using System.Runtime.Serialization;
+
+namespace Microsoft.VisualStudio.IntegrationTest.Utilities
+{
+    /// <summary>
+    /// Mapper for the Watson entry in the Event Log
+    /// </summary>
+    [DataContract]
+    internal class FeedbackItemWatsonEntry
+    {
+        /// <summary>
+        /// The time the event happend (UTC)
+        /// </summary>
+        [DataMember(Name = "eventTime")]
+        public DateTime EventTime { get; }
+
+        /// <summary>
+        /// Bucket Id
+        /// </summary>
+        [DataMember(Name = "faultBucket")]
+        public string FaultBucket { get; }
+
+        /// <summary>
+        /// Bucket Hash (might replace ID which would be deprecated, so sending for future proofing)
+        /// </summary>
+        [DataMember(Name = "hashedBucket")]
+        public string HashedBucket { get; }
+
+        /// <summary>
+        /// Watson Report ID
+        /// </summary>
+        [DataMember(Name = "watsonReportId")]
+        public string WatsonReportId { get; }
+
+        /// <summary>
+        /// The name of the event (some possible ones: "AppHangB1", "AppHangXProcB1", "MoAppHang","MoAppHangXProc","AppCrash","Crash32","Crash64","MoAppCrash","BEX","BEX64","clr20r3","MoBEX"
+        /// </summary>
+        [DataMember(Name = "eventName")]
+        public string EventName { get; }
+
+        /// <summary>
+        /// The CAB unique ID (can be empty - 0)
+        /// </summary>
+        [DataMember(Name = "cabId")]
+        public string CabId { get; }
+
+        /// <summary>
+        /// The name of the application causing the event (we have a list of VS EXEs that we grab for), e.g. "devenv.exe"
+        /// </summary>
+        [DataMember(Name = "applicationName")]
+        public string ApplicationName { get; }
+
+        /// <summary>
+        /// The version of the application causing the event, e.g. "14.0.23107.0"
+        /// </summary>
+        [DataMember(Name = "applicationVersion")]
+        public string ApplicationVersion { get; }
+
+        /// <summary>
+        /// The faulting module (what inside the app is causing the event), e.g. "ntdll.dll"
+        /// </summary>
+        [DataMember(Name = "faultingModule")]
+        public string FaultingModule { get; }
+
+        /// <summary>
+        /// The faulting module version
+        /// </summary>
+        [DataMember(Name = "faultModuleVersion")]
+        public string FaultingModuleVersion { get; }
+
+        /// <summary>
+        /// FaultBucket is the first property on the log entry
+        /// </summary>
+        private const int FaultBucketIndex = 0;
+
+        /// <summary>
+        /// EventName index in the log entry properties (2)
+        /// </summary>
+        internal const int EventNameIndex = 2;
+
+        /// <summary>
+        /// CabId index in the log entry properties
+        /// </summary>
+        private const int CabIdIndex = 4;
+
+        /// <summary>
+        /// Application name is contained in the P1 bucket parameter
+        /// </summary>
+        internal const int ApplicationNameIndex = 5;
+
+        /// <summary>
+        /// Application version is the P2 bucket parameter
+        /// </summary>
+        private const int ApplicationVersionIndex = 6;
+
+        /// <summary>
+        /// Faulting module is the P4 bucket parameter
+        /// </summary>
+        private const int FaultingModuleIndex = 8;
+
+        /// <summary>
+        /// Faulting module version is the P5 bucket parameter
+        /// </summary>
+        private const int FaultingModuleVersionindex = 9;
+
+        /// <summary>
+        /// WatsonReportId index in the log entry properties
+        /// </summary>
+        private const int WatsonReportIdIndex = 19;
+
+        /// <summary>
+        /// HashedBucket index in the log entry properties
+        /// </summary>
+        private const int HashedBucketIndex = 21;
+
+        /// <summary>
+        /// Constructor for a FeedbackItemWatsonEntry based on an EventRecord for future easiness of reading and modifying
+        /// </summary>
+        public FeedbackItemWatsonEntry(EventRecord eventLogRecord)
+        {
+            EventTime = eventLogRecord.TimeCreated.Value.ToUniversalTime();
+            FaultBucket = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, FaultBucketIndex);
+            HashedBucket = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, HashedBucketIndex);
+            WatsonReportId = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, WatsonReportIdIndex);
+            EventName = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, EventNameIndex);
+            CabId = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, CabIdIndex);
+            ApplicationName = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, ApplicationNameIndex);
+            ApplicationVersion = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, ApplicationVersionIndex);
+            FaultingModule = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, FaultingModuleIndex);
+            FaultingModuleVersion = EventLogCollector.GetEventRecordPropertyToString(eventLogRecord, FaultingModuleVersionindex);
+        }
+
+        /// <summary>
+        /// Used to make sure we aren't adding dupe entries to the list of Watson entries
+        /// </summary>
+        public override bool Equals(object obj)
+        {
+            if ((obj is FeedbackItemWatsonEntry watsonEntry)
+                && (EventName == watsonEntry.EventName)
+                && (ApplicationName == watsonEntry.ApplicationName)
+                && (ApplicationVersion == watsonEntry.ApplicationVersion)
+                && (FaultingModule == watsonEntry.FaultingModule)
+                && (FaultingModuleVersion == watsonEntry.FaultingModuleVersion))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return EventName.GetHashCode() ^ ApplicationName.GetHashCode() ^ ApplicationVersion.GetHashCode() ^ FaultingModule.GetHashCode() ^ FaultingModuleVersion.GetHashCode();
+        }
+    }
+}

src/VisualStudio/IntegrationTest/TestUtilities/VisualStudioInstanceFactory.cs

@@ -75,6 +75,8 @@ private static void FirstChanceExceptionHandler(object sender, FirstChanceExcept
                     Path.Combine(logDir, $"{baseFileName}.log"),
                     $"{exception}.GetType().Name{Environment.NewLine}{exception.StackTrace}");
 
+                EventLogCollector.TryWriteDotNetEntriesToFile(Path.Combine(logDir, $"{baseFileName}.DotNet.log"));
+                EventLogCollector.TryWriteWatsonEntriesToFile(Path.Combine(logDir, $"{baseFileName}.Watson.log"));
             }
             finally
             {