Merge pull request #19598 from heejaechang/fxcoptel

log info for ab testing

src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.ProjectState.cs

@@ -5,6 +5,7 @@
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Experimentation;
 using Microsoft.CodeAnalysis.Shared.Options;
 using Microsoft.CodeAnalysis.Workspaces.Diagnostics;
 using Roslyn.Utilities;
@@ -223,6 +224,8 @@ public async Task SaveAsync(Project project, DiagnosticAnalysisResult result)
                 }
 
                 await SerializeAsync(serializer, project, result.ProjectId, _owner.NonLocalStateName, result.Others).ConfigureAwait(false);
+
+                AnalyzerABTestLogger.LogProjectDiagnostics(project, result);
             }
 
             public void ResetVersion()
@@ -241,6 +244,8 @@ public async Task MergeAsync(ActiveFileState state, Document document)
                 var syntax = state.GetAnalysisData(AnalysisKind.Syntax);
                 var semantic = state.GetAnalysisData(AnalysisKind.Semantic);
 
+                AnalyzerABTestLogger.LogDocumentDiagnostics(document, syntax.Items, semantic.Items);
+
                 var project = document.Project;
 
                 // if project didn't successfully loaded, then it is same as FSA off

src/Features/Core/Portable/Experimentation/AnalyzerABTestLogger.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Internal.Log;
+using Microsoft.CodeAnalysis.Shared.Options;
+using Microsoft.CodeAnalysis.Workspaces.Diagnostics;
+using Roslyn.Utilities;
+
+namespace Microsoft.CodeAnalysis.Experimentation
+{
+    internal static class AnalyzerABTestLogger
+    {
+        private static bool s_reportErrors = false;
+        private static readonly ConcurrentDictionary<object, object> s_reported = new ConcurrentDictionary<object, object>(concurrencyLevel: 2, capacity: 10);
+
+        private const string Name = "LiveCodeAnalysisVsix";
+
+        public static void Log(string action)
+        {
+            Logger.Log(FunctionId.Experiment_ABTesting, KeyValueLogMessage.Create(LogType.UserAction, m =>
+            {
+                m[nameof(Name)] = Name;
+                m[nameof(action)] = action;
+            }));
+        }
+
+        public static void LogInstallationStatus(Workspace workspace, LiveCodeAnalysisInstallStatus installStatus)
+        {
+            var vsixInstalled = workspace.Options.GetOption(AnalyzerABTestOptions.VsixInstalled);
+            if (!vsixInstalled && installStatus == LiveCodeAnalysisInstallStatus.Installed)
+            {
+                // first time after vsix installed
+                workspace.Options = workspace.Options.WithChangedOption(AnalyzerABTestOptions.VsixInstalled, true);
+                workspace.Options = workspace.Options.WithChangedOption(AnalyzerABTestOptions.ParticipatedInExperiment, true);
+                Log("Installed");
+
+                // set the system to report the errors.
+                s_reportErrors = true;
+            }
+
+            if (vsixInstalled && installStatus == LiveCodeAnalysisInstallStatus.NotInstalled)
+            {
+                // first time after vsix is uninstalled
+                workspace.Options = workspace.Options.WithChangedOption(AnalyzerABTestOptions.VsixInstalled, false);
+                Log("Uninstalled");
+            }
+        }
+
+        public static void LogCandidacyRequirementsTracking(long lastTriggeredTimeBinary)
+        {
+            if (lastTriggeredTimeBinary == AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction.DefaultValue)
+            {
+                Log("StartCandidacyRequirementsTracking");
+            }
+        }
+
+        public static void LogProjectDiagnostics(Project project, DiagnosticAnalysisResult result)
+        {
+            if (!s_reportErrors || !s_reported.TryAdd(project.Id, null))
+            {
+                // doesn't meet the bar to report the issue.
+                return;
+            }
+
+            // logs count of errors for this project. this won't log anything if FSA off since
+            // we don't collect any diagnostics for a project if FSA is off.
+            var map = new Dictionary<string, int>();
+            foreach (var documentId in result.DocumentIdsOrEmpty)
+            {
+                CountErrors(map, result.GetResultOrEmpty(result.SyntaxLocals, documentId));
+                CountErrors(map, result.GetResultOrEmpty(result.SemanticLocals, documentId));
+                CountErrors(map, result.GetResultOrEmpty(result.NonLocals, documentId));
+            }
+
+            CountErrors(map, result.Others);
+
+            LogErrors(project, "ProjectDignostics", project.Id.Id, map);
+        }
+
+        public static void LogDocumentDiagnostics(Document document, ImmutableArray<DiagnosticData> syntax, ImmutableArray<DiagnosticData> semantic)
+        {
+            if (!s_reportErrors || !s_reported.TryAdd(document.Id, null))
+            {
+                // doesn't meet the bar to report the issue.
+                return;
+            }
+
+            // logs count of errors for this document. this only logs errors for 
+            // this particular document. we do this since when FSA is off, this is
+            // only errors we get. otherwise, we don't get any info when FSA is off and
+            // that is default for C#.
+            var map = new Dictionary<string, int>();
+            CountErrors(map, syntax);
+            CountErrors(map, semantic);
+
+            LogErrors(document.Project, "DocumentDignostics", document.Id.Id, map);
+        }
+
+        private static void LogErrors(Project project, string action, Guid target, Dictionary<string, int> map)
+        {
+            if (map.Count == 0)
+            {
+                // nothing to report
+                return;
+            }
+
+            var fsa = ServiceFeatureOnOffOptions.IsClosedFileDiagnosticsEnabled(project);
+            Logger.Log(FunctionId.Experiment_ABTesting, KeyValueLogMessage.Create(LogType.UserAction, m =>
+            {
+                m[nameof(Name)] = Name;
+                m[nameof(action)] = action;
+                m[nameof(target)] = target.ToString();
+                m["FSA"] = fsa;
+                m["errors"] = string.Join("|", map.Select(kv => $"{kv.Key}={kv.Value}"));
+            }));
+        }
+
+        private static void CountErrors(Dictionary<string, int> map, ImmutableArray<DiagnosticData> diagnostics)
+        {
+            if (diagnostics.IsDefaultOrEmpty)
+            {
+                return;
+            }
+
+            foreach (var group in diagnostics.GroupBy(d => d.Id))
+            {
+                map[group.Key] = IDictionaryExtensions.GetValueOrDefault(map, group.Key) + group.Count();
+            }
+        }
+    }
+}

src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerABTestOptions.cs → src/Features/Core/Portable/Experimentation/AnalyzerABTestOptions.cs

@@ -3,7 +3,7 @@
 using System;
 using Microsoft.CodeAnalysis.Options;
 
-namespace Microsoft.VisualStudio.LanguageServices.Implementation.Experimentation
+namespace Microsoft.CodeAnalysis.Experimentation
 {
     internal static class AnalyzerABTestOptions
     {
@@ -25,5 +25,13 @@ internal static class AnalyzerABTestOptions
         public static readonly Option<long> LastDateTimeInfoBarShown = new Option<long>(nameof(AnalyzerABTestOptions), nameof(LastDateTimeInfoBarShown),
             defaultValue: DateTime.MinValue.ToBinary(),
             storageLocations: new LocalUserProfileStorageLocation(LocalRegistryPath + nameof(LastDateTimeInfoBarShown)));
+
+        public static readonly Option<bool> VsixInstalled = new Option<bool>(nameof(AnalyzerABTestOptions),
+            nameof(VsixInstalled), defaultValue: false,
+            storageLocations: new LocalUserProfileStorageLocation(LocalRegistryPath + nameof(VsixInstalled)));
+
+        public static readonly Option<bool> ParticipatedInExperiment = new Option<bool>(nameof(AnalyzerABTestOptions),
+            nameof(ParticipatedInExperiment), defaultValue: false,
+            storageLocations: new LocalUserProfileStorageLocation(LocalRegistryPath + nameof(ParticipatedInExperiment)));
     }
 }

src/VisualStudio/Core/Def/Implementation/Experimentation/LiveCodeAnalysisInstallStatus.cs → src/Features/Core/Portable/Experimentation/LiveCodeAnalysisInstallStatus.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-namespace Microsoft.VisualStudio.LanguageServices.Implementation.Experimentation
+namespace Microsoft.CodeAnalysis.Experimentation
 {
     internal enum LiveCodeAnalysisInstallStatus
     {

src/Features/Core/Portable/Features.csproj

@@ -134,6 +134,9 @@
     <Compile Include="Diagnostics\SymbolAnalysisContextExtensions.cs" />
     <Compile Include="DocumentHighlighting\DocumentHighlightingOptions.cs" />
     <Compile Include="EncapsulateField\EncapsulateFieldRefactoringProvider.cs" />
+    <Compile Include="Experimentation\AnalyzerABTestLogger.cs" />
+    <Compile Include="Experimentation\AnalyzerABTestOptions.cs" />
+    <Compile Include="Experimentation\LiveCodeAnalysisInstallStatus.cs" />
     <Compile Include="ExtractInterface\ExtractInterfaceCodeRefactoringProvider.cs" />
     <Compile Include="CodeFixes\FixAllOccurrences\FixSomeCodeAction.cs" />
     <Compile Include="AddPackage\InstallPackageDirectlyCodeAction.cs" />

src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerVsixSuggestedActionCallback.cs

@@ -6,6 +6,7 @@
 using Microsoft.CodeAnalysis.Editor.Implementation.Suggestions;
 using Microsoft.CodeAnalysis.Editor.Shared.Utilities;
 using Microsoft.CodeAnalysis.ErrorReporting;
+using Microsoft.CodeAnalysis.Experimentation;
 using Microsoft.CodeAnalysis.Experiments;
 using Microsoft.CodeAnalysis.Extensions;
 using Microsoft.VisualStudio.Shell;
@@ -97,6 +98,7 @@ private bool IsVsixInstalled()
                 else
                 {
                     _installStatus = installed != 0 ? LiveCodeAnalysisInstallStatus.Installed : LiveCodeAnalysisInstallStatus.NotInstalled;
+                    AnalyzerABTestLogger.LogInstallationStatus(_workspace, _installStatus);
                 }
             }
 
@@ -105,6 +107,13 @@ private bool IsVsixInstalled()
 
         private bool IsCandidate()
         {
+            // if this user ever participated in the experiement and then uninstall the vsix, then
+            // this user will never be candidate again.
+            if (_workspace.Options.GetOption(AnalyzerABTestOptions.ParticipatedInExperiment))
+            {
+                return false;
+            }
+
             // Filter for valid A/B test candidates. Candidates fill the following critera:
             //     1: Are a Dotnet user (as evidenced by the fact that this code is being run)
             //     2: Have triggered a lightbulb on 3 separate days
@@ -116,21 +125,26 @@ private bool IsCandidate()
             if (!isCandidate)
             {
                 // We store in UTC to avoid any timezone offset weirdness
-                var lastTriggeredTime = DateTime.FromBinary(options.GetOption(AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction));
+                var lastTriggeredTimeBinary = options.GetOption(AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction);
+                AnalyzerABTestLogger.LogCandidacyRequirementsTracking(lastTriggeredTimeBinary);
+
+                var lastTriggeredTime = DateTime.FromBinary(lastTriggeredTimeBinary);
                 var currentTime = DateTime.UtcNow;
                 var span = currentTime - lastTriggeredTime;
                 if (span.TotalDays >= 1)
                 {
                     options = options.WithChangedOption(AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction, currentTime.ToBinary());
+
                     var usageCount = options.GetOption(AnalyzerABTestOptions.UsedSuggestedActionCount);
-                    usageCount++;
-                    options = options.WithChangedOption(AnalyzerABTestOptions.UsedSuggestedActionCount, usageCount);
+                    options = options.WithChangedOption(AnalyzerABTestOptions.UsedSuggestedActionCount, ++usageCount);
 
                     if (usageCount >= 3)
                     {
                         isCandidate = true;
                         options = options.WithChangedOption(AnalyzerABTestOptions.HasMetCandidacyRequirements, true);
+                        AnalyzerABTestLogger.Log(nameof(AnalyzerABTestOptions.HasMetCandidacyRequirements));
                     }
+
                     _workspace.Options = options;
                 }
             }
@@ -144,6 +158,8 @@ private void ShowInfoBarIfNecessary()
             _infoBarChecked = true;
             if (_experimentationService.IsExperimentEnabled(AnalyzerEnabledFlight))
             {
+                AnalyzerABTestLogger.Log(nameof(AnalyzerEnabledFlight));
+
                 // If we got true from the experimentation service, then we're in the treatment
                 // group, and the experiment is enabled. We determine if the infobar has been
                 // displayed in the past 24 hours. If it hasn't been displayed, then we do so now.
@@ -154,6 +170,7 @@ private void ShowInfoBarIfNecessary()
                 if (timeSinceLastShown.TotalDays >= 1)
                 {
                     _workspace.Options = _workspace.Options.WithChangedOption(AnalyzerABTestOptions.LastDateTimeInfoBarShown, utcNow.ToBinary());
+                    AnalyzerABTestLogger.Log("InfoBarShown");
 
                     var infoBarService = _workspace.Services.GetRequiredService<IInfoBarService>();
                     infoBarService.ShowInfoBarInGlobalView(
@@ -173,11 +190,13 @@ private void ShowInfoBarIfNecessary()
         private void OpenInstallHyperlink()
         {
             System.Diagnostics.Process.Start(AnalyzerVsixHyperlink);
+            AnalyzerABTestLogger.Log(nameof(AnalyzerVsixHyperlink));
         }
 
         private void DoNotShowAgain()
         {
             _workspace.Options = _workspace.Options.WithChangedOption(AnalyzerABTestOptions.NeverShowAgain, true);
+            AnalyzerABTestLogger.Log(nameof(AnalyzerABTestOptions.NeverShowAgain));
         }
     }
 }

src/VisualStudio/Core/Def/ServicesVisualStudio.csproj

@@ -63,8 +63,6 @@
     <Compile Include="Implementation\AnalyzerDependency\IIgnorableAssemblyList.cs" />
     <Compile Include="Implementation\AnalyzerDependency\IBindingRedirectionService.cs" />
     <Compile Include="Implementation\Experimentation\AnalyzerVsixSuggestedActionCallback.cs" />
-    <Compile Include="Implementation\Experimentation\AnalyzerABTestOptions.cs" />
-    <Compile Include="Implementation\Experimentation\LiveCodeAnalysisInstallStatus.cs" />
     <Compile Include="Implementation\Extensions\ServiceProviderExtensions.cs" />
     <Compile Include="ID.InteractiveCommands.cs" />
     <Compile Include="Implementation\FindReferences\VisualStudioDefinitionsAndReferencesFactory.cs" />

src/Workspaces/Core/Portable/Log/FunctionId.cs

@@ -392,5 +392,6 @@ internal enum FunctionId
         CompilationService_GetCompilationAsync,
         SolutionCreator_AssetDifferences,
         Extension_InfoBar,
+        Experiment_ABTesting,
     }
 }