Bring /errorlog up to SARIF v0.4 draft

* "issues" -> "results"
* "isSuppressedInSource" from custom property to first class
* deduce "kind" from severity

docs/compilers/Error Log Format.md

@@ -15,9 +15,9 @@ rather adds information that is specific to the implementation provided
 by the C# and Visual Basic Compilers.
 
 
-Issue Properties
+Result Properties
 ================
-The SARIF standard allows the `properties` property of `issue` objects
+The SARIF standard allows the `properties` property of `result` objects
 to contain arbitrary (string, string) key-value pairs.
 
 The keys and values used by the C# and VB compilers are serialized from
@@ -33,6 +33,4 @@ Key                      | Value
 "category"               | `Diagnostic.Category`
 "helpLink"               | `DiagnosticDescriptor.HelpLink` (omitted if null or empty)
 "isEnabledByDefault"     | `Diagnostic.IsEnabledByDefault` ("True" or "False")
-"isSuppressedInSource"   | `Diagnostic.IsSuppressedInSource` ("True" or "False")
-"customTags"             | `Diagnostic.CustomTags` (joined together in a `;`-delimted list)
 "customProperties.[key]" | `Diagnostic.Properties[key]` (for each key in the dictionary)

src/Compilers/CSharp/Test/CommandLine/ErrorLoggerTests.cs

@@ -46,7 +46,7 @@ public static void Main(string[] args)
 
             var expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd);
             var expectedIssues = @"
-      ""issues"": [
+      ""results"": [
       ]
     }
   ]
@@ -57,7 +57,7 @@ public static void Main(string[] args)
             CleanupAllGeneratedFiles(hello);
             CleanupAllGeneratedFiles(errorLogFile);
         }
-
+        
         [Fact]
         public void SimpleCompilerDiagnostics()
         {
@@ -85,9 +85,10 @@ public class C
 
             var expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd);
             var expectedIssues = string.Format(@"
-      ""issues"": [
+      ""results"": [
         {{
           ""ruleId"": ""CS0169"",
+          ""kind"": ""warning"",
           ""locations"": [
             {{
               ""analysisTarget"": [
@@ -104,29 +105,37 @@ public class C
             }}
           ],
           ""fullMessage"": ""The field 'C.x' is never used"",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry""
+          ],
           ""properties"": {{
             ""severity"": ""Warning"",
             ""warningLevel"": ""3"",
             ""defaultSeverity"": ""Warning"",
             ""title"": ""Field is never used"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": ""Compiler;Telemetry""
+            ""isEnabledByDefault"": ""True""
           }}
         }},
         {{
           ""ruleId"": ""CS5001"",
+          ""kind"": ""error"",
           ""locations"": [
           ],
           ""fullMessage"": ""Program does not contain a static 'Main' method suitable for an entry point"",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry"",
+            ""NotConfigurable""
+          ],
           ""properties"": {{
             ""severity"": ""Error"",
             ""defaultSeverity"": ""Error"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": ""Compiler;Telemetry;NotConfigurable""
+            ""isEnabledByDefault"": ""True""
           }}
         }}
       ]
@@ -171,9 +180,10 @@ public class C
 
             var expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd);
             var expectedIssues = string.Format(@"
-      ""issues"": [
+      ""results"": [
         {{
           ""ruleId"": ""CS0169"",
+          ""kind"": ""warning"",
           ""locations"": [
             {{
               ""analysisTarget"": [
@@ -190,29 +200,37 @@ public class C
             }}
           ],
           ""fullMessage"": ""The field 'C.x' is never used"",
+          ""isSuppressedInSource"": true,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry""
+          ],
           ""properties"": {{
             ""severity"": ""Warning"",
             ""warningLevel"": ""3"",
             ""defaultSeverity"": ""Warning"",
             ""title"": ""Field is never used"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""True"",
-            ""customTags"": ""Compiler;Telemetry""
+            ""isEnabledByDefault"": ""True""
           }}
         }},
         {{
           ""ruleId"": ""CS5001"",
+          ""kind"": ""error"",
           ""locations"": [
           ],
           ""fullMessage"": ""Program does not contain a static 'Main' method suitable for an entry point"",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry"",
+            ""NotConfigurable""
+          ],
           ""properties"": {{
             ""severity"": ""Error"",
             ""defaultSeverity"": ""Error"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": ""Compiler;Telemetry;NotConfigurable""
+            ""isEnabledByDefault"": ""True""
           }}
         }}
       ]
@@ -255,7 +273,7 @@ public class C
             var actualOutput = File.ReadAllText(errorLogFile).Trim();
 
             var expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd);
-            var expectedIssues = AnalyzerForErrorLogTest.GetExpectedErrorLogIssuesText(cmd.Compilation);
+            var expectedIssues = AnalyzerForErrorLogTest.GetExpectedErrorLogResultsText(cmd.Compilation);
             var expectedText = expectedHeader + expectedIssues;
             Assert.Equal(expectedText, actualOutput);
 

src/Compilers/Core/Portable/CommandLine/ErrorLogger.cs

@@ -21,7 +21,7 @@ namespace Microsoft.CodeAnalysis
     internal partial class ErrorLogger : IDisposable
     {
         // Internal for testing purposes.
-        internal const string OutputFormatVersion = "0.1";
+        internal const string OutputFormatVersion = "0.4";
 
         private readonly JsonWriter _writer;
 
@@ -40,7 +40,7 @@ public ErrorLogger(Stream stream, string toolName, string toolFileVersion, Versi
 
             WriteToolInfo(toolName, toolFileVersion, toolAssemblyVersion);
 
-            _writer.WriteArrayStart("issues");
+            _writer.WriteArrayStart("results");
         }
 
         private void WriteToolInfo(string name, string fileVersion, Version assemblyVersion)
@@ -54,8 +54,9 @@ private void WriteToolInfo(string name, string fileVersion, Version assemblyVers
 
         internal void LogDiagnostic(Diagnostic diagnostic, CultureInfo culture)
         {
-            _writer.WriteObjectStart(); // issue
+            _writer.WriteObjectStart(); // result
             _writer.Write("ruleId", diagnostic.Id);
+            _writer.Write("kind", GetKind(diagnostic.Severity));
 
             WriteLocations(diagnostic.Location, diagnostic.AdditionalLocations);
 
@@ -76,9 +77,13 @@ internal void LogDiagnostic(Diagnostic diagnostic, CultureInfo culture)
                 _writer.Write("fullMessage", description);
             }
 
+            _writer.Write("isSuppressedInSource", diagnostic.IsSuppressed);
+
+            WriteTags(diagnostic);
+
             WriteProperties(diagnostic, culture);
 
-            _writer.WriteObjectEnd(); // issue
+            _writer.WriteObjectEnd(); // result
         }
 
         private void WriteLocations(Location location, IReadOnlyList<Location> additionalLocations)
@@ -143,6 +148,21 @@ private static string GetUri(SyntaxTree syntaxTree)
             return uri.ToString();
         }
 
+        private void WriteTags(Diagnostic diagnostic)
+        {
+            if (diagnostic.CustomTags.Count > 0)
+            {
+                _writer.WriteArrayStart("tags");
+
+                foreach (string tag in diagnostic.CustomTags)
+                {
+                    _writer.Write(tag);
+                }
+
+                _writer.WriteArrayEnd();
+            }
+        }
+
         private void WriteProperties(Diagnostic diagnostic, CultureInfo culture)
         {
             _writer.WriteObjectStart("properties");
@@ -172,13 +192,6 @@ private void WriteProperties(Diagnostic diagnostic, CultureInfo culture)
 
             _writer.Write("isEnabledByDefault", diagnostic.IsEnabledByDefault.ToString());
 
-            _writer.Write("isSuppressedInSource", diagnostic.IsSuppressed.ToString());
-
-            if (diagnostic.CustomTags.Count > 0)
-            {
-                _writer.Write("customTags", diagnostic.CustomTags.WhereNotNull().Join(";"));
-            }
-
             foreach (var pair in diagnostic.Properties.OrderBy(x => x.Key, StringComparer.Ordinal))
             {
                 _writer.Write("customProperties." + pair.Key, pair.Value);
@@ -187,10 +200,30 @@ private void WriteProperties(Diagnostic diagnostic, CultureInfo culture)
             _writer.WriteObjectEnd(); // properties
         }
 
+        private static string GetKind(DiagnosticSeverity severity)
+        {
+            switch (severity)
+            {
+                case DiagnosticSeverity.Info:
+                    return "note";
+
+                case DiagnosticSeverity.Error:
+                    return "error";
+
+                case DiagnosticSeverity.Warning:
+                case DiagnosticSeverity.Hidden:
+                default:
+                    // note that in the hidden or default cases, we still write out the actual severity as a
+                    // property so no information is lost. We have to conform to the SARIF spec for kind,
+                    // which allows only pass, warning, error, or notApplicable.
+                    return "warning";
+            }
+        }
+
         public void Dispose()
         {
-            _writer.WriteArrayEnd();  // issues
-            _writer.WriteObjectEnd(); // single runLog
+            _writer.WriteArrayEnd();  // results
+            _writer.WriteObjectEnd(); // runLog
             _writer.WriteArrayEnd();  // runLogs
             _writer.WriteObjectEnd(); // root
             _writer.Dispose();

src/Compilers/VisualBasic/Test/CommandLine/ErrorLoggerTests.vb

@@ -46,7 +46,7 @@ End Class
 
             Dim expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd)
             Dim expectedIssues = "
-      ""issues"": [
+      ""results"": [
       ]
     }
   ]
@@ -91,9 +91,10 @@ End Class
 
             Dim expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd)
             Dim expectedIssues = String.Format("
-      ""issues"": [
+      ""results"": [
         {{
           ""ruleId"": ""BC42024"",
+          ""kind"": ""warning"",
           ""locations"": [
             {{
               ""analysisTarget"": [
@@ -110,29 +111,37 @@ End Class
             }}
           ],
           ""fullMessage"": ""Unused local variable: 'x'."",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry""
+          ],
           ""properties"": {{
             ""severity"": ""Warning"",
             ""warningLevel"": ""1"",
             ""defaultSeverity"": ""Warning"",
             ""title"": ""Unused local variable"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": ""Compiler;Telemetry""
+            ""isEnabledByDefault"": ""True""
           }}
         }},
         {{
           ""ruleId"": ""BC30420"",
+          ""kind"": ""error"",
           ""locations"": [
           ],
           ""fullMessage"": ""'Sub Main' was not found in '{1}'."",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry"",
+            ""NotConfigurable""
+          ],
           ""properties"": {{
             ""severity"": ""Error"",
             ""defaultSeverity"": ""Error"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": ""Compiler;Telemetry;NotConfigurable""
+            ""isEnabledByDefault"": ""True""
           }}
         }}
       ]
@@ -182,9 +191,10 @@ End Class
 
             Dim expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd)
             Dim expectedIssues = String.Format("
-      ""issues"": [
+      ""results"": [
         {{
           ""ruleId"": ""BC42024"",
+          ""kind"": ""warning"",
           ""locations"": [
             {{
               ""analysisTarget"": [
@@ -201,29 +211,37 @@ End Class
             }}
           ],
           ""fullMessage"": ""Unused local variable: 'x'."",
+          ""isSuppressedInSource"": true,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry""
+          ],
           ""properties"": {{
             ""severity"": ""Warning"",
             ""warningLevel"": ""1"",
             ""defaultSeverity"": ""Warning"",
             ""title"": ""Unused local variable"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""True"",
-            ""customTags"": ""Compiler;Telemetry""
+            ""isEnabledByDefault"": ""True""
           }}
         }},
         {{
           ""ruleId"": ""BC30420"",
+          ""kind"": ""error"",
           ""locations"": [
           ],
           ""fullMessage"": ""'Sub Main' was not found in '{1}'."",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            ""Compiler"",
+            ""Telemetry"",
+            ""NotConfigurable""
+          ],
           ""properties"": {{
             ""severity"": ""Error"",
             ""defaultSeverity"": ""Error"",
             ""category"": ""Compiler"",
-            ""isEnabledByDefault"": ""True"",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": ""Compiler;Telemetry;NotConfigurable""
+            ""isEnabledByDefault"": ""True""
           }}
         }}
       ]
@@ -271,7 +289,7 @@ End Class
             Dim actualOutput = File.ReadAllText(errorLogFile).Trim()
 
             Dim expectedHeader = GetExpectedErrorLogHeader(actualOutput, cmd)
-            Dim expectedIssues = AnalyzerForErrorLogTest.GetExpectedErrorLogIssuesText(cmd.Compilation)
+            Dim expectedIssues = AnalyzerForErrorLogTest.GetExpectedErrorLogResultsText(cmd.Compilation)
 
             Dim expectedText = expectedHeader + expectedIssues
             Assert.Equal(expectedText, actualOutput)

src/Test/Utilities/Desktop/CommonDiagnosticAnalyzers.cs

@@ -77,7 +77,7 @@ private static string GetExpectedPropertiesMapText()
                 return expectedText;
             }
 
-            public static string GetExpectedErrorLogIssuesText(Compilation compilation)
+            public static string GetExpectedErrorLogResultsText(Compilation compilation)
             {
                 var tree = compilation.SyntaxTrees.First();
                 var root = tree.GetRoot();
@@ -85,9 +85,10 @@ public static string GetExpectedErrorLogIssuesText(Compilation compilation)
                 var filePath = GetEscapedUriForPath(tree.FilePath);
 
                 return @"
-      ""issues"": [
+      ""results"": [
         {
           ""ruleId"": """ + Descriptor1.Id + @""",
+          ""kind"": """ + (Descriptor1.DefaultSeverity == DiagnosticSeverity.Error ? "error" : "warning") + @""",
           ""locations"": [
             {
               ""analysisTarget"": [
@@ -105,6 +106,10 @@ public static string GetExpectedErrorLogIssuesText(Compilation compilation)
           ],
           ""shortMessage"": """ + Descriptor1.MessageFormat + @""",
           ""fullMessage"": """ + Descriptor1.Description + @""",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            " + String.Join("," + Environment.NewLine + "            ", Descriptor1.CustomTags.Select(s => $"\"{s}\"")) + @"
+          ],
           ""properties"": {
             ""severity"": """ + Descriptor1.DefaultSeverity + @""",
             ""warningLevel"": ""1"",
@@ -112,27 +117,28 @@ public static string GetExpectedErrorLogIssuesText(Compilation compilation)
             ""title"": """ + Descriptor1.Title + @""",
             ""category"": """ + Descriptor1.Category + @""",
             ""helpLink"": """ + Descriptor1.HelpLinkUri + @""",
-            ""isEnabledByDefault"": """ + Descriptor1.IsEnabledByDefault + @""",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": """ + Descriptor1.CustomTags.Join(";") + @"""" +
+            ""isEnabledByDefault"": """ + Descriptor1.IsEnabledByDefault + @"""" +
             GetExpectedPropertiesMapText() + @"
           }
         },
         {
           ""ruleId"": """ + Descriptor2.Id + @""",
+          ""kind"": """ + (Descriptor2.DefaultSeverity == DiagnosticSeverity.Error ? "error" : "warning") + @""",
           ""locations"": [
           ],
           ""shortMessage"": """ + Descriptor2.MessageFormat + @""",
           ""fullMessage"": """ + Descriptor2.Description + @""",
+          ""isSuppressedInSource"": false,
+          ""tags"": [
+            " + String.Join("," + Environment.NewLine + "            ", Descriptor2.CustomTags.Select(s => $"\"{s}\"")) + @"
+          ],
           ""properties"": {
             ""severity"": """ + Descriptor2.DefaultSeverity + @""",
             ""defaultSeverity"": """ + Descriptor2.DefaultSeverity + @""",
             ""title"": """ + Descriptor2.Title + @""",
             ""category"": """ + Descriptor2.Category + @""",
             ""helpLink"": """ + Descriptor2.HelpLinkUri + @""",
-            ""isEnabledByDefault"": """ + Descriptor2.IsEnabledByDefault + @""",
-            ""isSuppressedInSource"": ""False"",
-            ""customTags"": """ + Descriptor2.CustomTags.Join(";") + @"""" +
+            ""isEnabledByDefault"": """ + Descriptor2.IsEnabledByDefault + @"""" +
             GetExpectedPropertiesMapText() + @"
           }
         }