diff --git a/src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.ProjectState.cs b/src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.ProjectState.cs index 58212225acfb66054bf81e55b60d6ace92a6c615..0f4939afaed8c753f85dd3865f6fd1f9d42b52d1 100644 --- a/src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.ProjectState.cs +++ b/src/Features/Core/Portable/Diagnostics/EngineV2/DiagnosticIncrementalAnalyzer.ProjectState.cs @@ -5,6 +5,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Experimentation; using Microsoft.CodeAnalysis.Shared.Options; using Microsoft.CodeAnalysis.Workspaces.Diagnostics; using Roslyn.Utilities; @@ -223,6 +224,8 @@ public async Task SaveAsync(Project project, DiagnosticAnalysisResult result) } await SerializeAsync(serializer, project, result.ProjectId, _owner.NonLocalStateName, result.Others).ConfigureAwait(false); + + AnalyzerABTestLogger.LogProjectDiagnostics(project, result); } public void ResetVersion() @@ -241,6 +244,8 @@ public async Task MergeAsync(ActiveFileState state, Document document) var syntax = state.GetAnalysisData(AnalysisKind.Syntax); var semantic = state.GetAnalysisData(AnalysisKind.Semantic); + AnalyzerABTestLogger.LogDocumentDiagnostics(document, syntax.Items, semantic.Items); + var project = document.Project; // if project didn't successfully loaded, then it is same as FSA off diff --git a/src/Features/Core/Portable/Experimentation/AnalyzerABTestLogger.cs b/src/Features/Core/Portable/Experimentation/AnalyzerABTestLogger.cs new file mode 100644 index 0000000000000000000000000000000000000000..fd2f932d98ccd4e19afea08dcb1e153f4b0dabd9 --- /dev/null +++ b/src/Features/Core/Portable/Experimentation/AnalyzerABTestLogger.cs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Internal.Log; +using Microsoft.CodeAnalysis.Shared.Options; +using Microsoft.CodeAnalysis.Workspaces.Diagnostics; +using Roslyn.Utilities; + +namespace Microsoft.CodeAnalysis.Experimentation +{ + internal static class AnalyzerABTestLogger + { + private static bool s_reportErrors = false; + private static readonly ConcurrentDictionary s_reported = new ConcurrentDictionary(concurrencyLevel: 2, capacity: 10); + + private const string Name = "LiveCodeAnalysisVsix"; + + public static void Log(string action) + { + Logger.Log(FunctionId.Experiment_ABTesting, KeyValueLogMessage.Create(LogType.UserAction, m => + { + m[nameof(Name)] = Name; + m[nameof(action)] = action; + })); + } + + public static void LogInstallationStatus(Workspace workspace, LiveCodeAnalysisInstallStatus installStatus) + { + var vsixInstalled = workspace.Options.GetOption(AnalyzerABTestOptions.VsixInstalled); + if (!vsixInstalled && installStatus == LiveCodeAnalysisInstallStatus.Installed) + { + // first time after vsix installed + workspace.Options = workspace.Options.WithChangedOption(AnalyzerABTestOptions.VsixInstalled, true); + workspace.Options = workspace.Options.WithChangedOption(AnalyzerABTestOptions.ParticipatedInExperiment, true); + Log("Installed"); + + // set the system to report the errors. + s_reportErrors = true; + } + + if (vsixInstalled && installStatus == LiveCodeAnalysisInstallStatus.NotInstalled) + { + // first time after vsix is uninstalled + workspace.Options = workspace.Options.WithChangedOption(AnalyzerABTestOptions.VsixInstalled, false); + Log("Uninstalled"); + } + } + + public static void LogCandidacyRequirementsTracking(long lastTriggeredTimeBinary) + { + if (lastTriggeredTimeBinary == AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction.DefaultValue) + { + Log("StartCandidacyRequirementsTracking"); + } + } + + public static void LogProjectDiagnostics(Project project, DiagnosticAnalysisResult result) + { + if (!s_reportErrors || !s_reported.TryAdd(project.Id, null)) + { + // doesn't meet the bar to report the issue. + return; + } + + // logs count of errors for this project. this won't log anything if FSA off since + // we don't collect any diagnostics for a project if FSA is off. + var map = new Dictionary(); + foreach (var documentId in result.DocumentIdsOrEmpty) + { + CountErrors(map, result.GetResultOrEmpty(result.SyntaxLocals, documentId)); + CountErrors(map, result.GetResultOrEmpty(result.SemanticLocals, documentId)); + CountErrors(map, result.GetResultOrEmpty(result.NonLocals, documentId)); + } + + CountErrors(map, result.Others); + + LogErrors(project, "ProjectDignostics", project.Id.Id, map); + } + + public static void LogDocumentDiagnostics(Document document, ImmutableArray syntax, ImmutableArray semantic) + { + if (!s_reportErrors || !s_reported.TryAdd(document.Id, null)) + { + // doesn't meet the bar to report the issue. + return; + } + + // logs count of errors for this document. this only logs errors for + // this particular document. we do this since when FSA is off, this is + // only errors we get. otherwise, we don't get any info when FSA is off and + // that is default for C#. + var map = new Dictionary(); + CountErrors(map, syntax); + CountErrors(map, semantic); + + LogErrors(document.Project, "DocumentDignostics", document.Id.Id, map); + } + + private static void LogErrors(Project project, string action, Guid target, Dictionary map) + { + if (map.Count == 0) + { + // nothing to report + return; + } + + var fsa = ServiceFeatureOnOffOptions.IsClosedFileDiagnosticsEnabled(project); + Logger.Log(FunctionId.Experiment_ABTesting, KeyValueLogMessage.Create(LogType.UserAction, m => + { + m[nameof(Name)] = Name; + m[nameof(action)] = action; + m[nameof(target)] = target.ToString(); + m["FSA"] = fsa; + m["errors"] = string.Join("|", map.Select(kv => $"{kv.Key}={kv.Value}")); + })); + } + + private static void CountErrors(Dictionary map, ImmutableArray diagnostics) + { + if (diagnostics.IsDefaultOrEmpty) + { + return; + } + + foreach (var group in diagnostics.GroupBy(d => d.Id)) + { + map[group.Key] = IDictionaryExtensions.GetValueOrDefault(map, group.Key) + group.Count(); + } + } + } +} diff --git a/src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerABTestOptions.cs b/src/Features/Core/Portable/Experimentation/AnalyzerABTestOptions.cs similarity index 75% rename from src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerABTestOptions.cs rename to src/Features/Core/Portable/Experimentation/AnalyzerABTestOptions.cs index d3a5116923821293e38d010d0cb61ed13ea696be..b617366602af5d7f3723b0dbc7ef5ad3b172787f 100644 --- a/src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerABTestOptions.cs +++ b/src/Features/Core/Portable/Experimentation/AnalyzerABTestOptions.cs @@ -3,7 +3,7 @@ using System; using Microsoft.CodeAnalysis.Options; -namespace Microsoft.VisualStudio.LanguageServices.Implementation.Experimentation +namespace Microsoft.CodeAnalysis.Experimentation { internal static class AnalyzerABTestOptions { @@ -25,5 +25,13 @@ internal static class AnalyzerABTestOptions public static readonly Option LastDateTimeInfoBarShown = new Option(nameof(AnalyzerABTestOptions), nameof(LastDateTimeInfoBarShown), defaultValue: DateTime.MinValue.ToBinary(), storageLocations: new LocalUserProfileStorageLocation(LocalRegistryPath + nameof(LastDateTimeInfoBarShown))); + + public static readonly Option VsixInstalled = new Option(nameof(AnalyzerABTestOptions), + nameof(VsixInstalled), defaultValue: false, + storageLocations: new LocalUserProfileStorageLocation(LocalRegistryPath + nameof(VsixInstalled))); + + public static readonly Option ParticipatedInExperiment = new Option(nameof(AnalyzerABTestOptions), + nameof(ParticipatedInExperiment), defaultValue: false, + storageLocations: new LocalUserProfileStorageLocation(LocalRegistryPath + nameof(ParticipatedInExperiment))); } } diff --git a/src/VisualStudio/Core/Def/Implementation/Experimentation/LiveCodeAnalysisInstallStatus.cs b/src/Features/Core/Portable/Experimentation/LiveCodeAnalysisInstallStatus.cs similarity index 77% rename from src/VisualStudio/Core/Def/Implementation/Experimentation/LiveCodeAnalysisInstallStatus.cs rename to src/Features/Core/Portable/Experimentation/LiveCodeAnalysisInstallStatus.cs index 39ff06483f01079fa3450de3c98d769f2ca92b46..2f78c070b3dd68020484b502bd65435a56ad400c 100644 --- a/src/VisualStudio/Core/Def/Implementation/Experimentation/LiveCodeAnalysisInstallStatus.cs +++ b/src/Features/Core/Portable/Experimentation/LiveCodeAnalysisInstallStatus.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace Microsoft.VisualStudio.LanguageServices.Implementation.Experimentation +namespace Microsoft.CodeAnalysis.Experimentation { internal enum LiveCodeAnalysisInstallStatus { diff --git a/src/Features/Core/Portable/Features.csproj b/src/Features/Core/Portable/Features.csproj index fcee5c48cd89f53b8b9f51bfa0e6503e303b46f3..7254e8866c0b6a90475331358346bbb65117bc48 100644 --- a/src/Features/Core/Portable/Features.csproj +++ b/src/Features/Core/Portable/Features.csproj @@ -134,6 +134,9 @@ + + + diff --git a/src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerVsixSuggestedActionCallback.cs b/src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerVsixSuggestedActionCallback.cs index 07bb6058bb9d0b1ab3878c1838488e008e4da309..65d2a9c7f28179039337b9fd28939f5ca3467d74 100644 --- a/src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerVsixSuggestedActionCallback.cs +++ b/src/VisualStudio/Core/Def/Implementation/Experimentation/AnalyzerVsixSuggestedActionCallback.cs @@ -6,6 +6,7 @@ using Microsoft.CodeAnalysis.Editor.Implementation.Suggestions; using Microsoft.CodeAnalysis.Editor.Shared.Utilities; using Microsoft.CodeAnalysis.ErrorReporting; +using Microsoft.CodeAnalysis.Experimentation; using Microsoft.CodeAnalysis.Experiments; using Microsoft.CodeAnalysis.Extensions; using Microsoft.VisualStudio.Shell; @@ -97,6 +98,7 @@ private bool IsVsixInstalled() else { _installStatus = installed != 0 ? LiveCodeAnalysisInstallStatus.Installed : LiveCodeAnalysisInstallStatus.NotInstalled; + AnalyzerABTestLogger.LogInstallationStatus(_workspace, _installStatus); } } @@ -105,6 +107,13 @@ private bool IsVsixInstalled() private bool IsCandidate() { + // if this user ever participated in the experiement and then uninstall the vsix, then + // this user will never be candidate again. + if (_workspace.Options.GetOption(AnalyzerABTestOptions.ParticipatedInExperiment)) + { + return false; + } + // Filter for valid A/B test candidates. Candidates fill the following critera: // 1: Are a Dotnet user (as evidenced by the fact that this code is being run) // 2: Have triggered a lightbulb on 3 separate days @@ -116,21 +125,26 @@ private bool IsCandidate() if (!isCandidate) { // We store in UTC to avoid any timezone offset weirdness - var lastTriggeredTime = DateTime.FromBinary(options.GetOption(AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction)); + var lastTriggeredTimeBinary = options.GetOption(AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction); + AnalyzerABTestLogger.LogCandidacyRequirementsTracking(lastTriggeredTimeBinary); + + var lastTriggeredTime = DateTime.FromBinary(lastTriggeredTimeBinary); var currentTime = DateTime.UtcNow; var span = currentTime - lastTriggeredTime; if (span.TotalDays >= 1) { options = options.WithChangedOption(AnalyzerABTestOptions.LastDateTimeUsedSuggestionAction, currentTime.ToBinary()); + var usageCount = options.GetOption(AnalyzerABTestOptions.UsedSuggestedActionCount); - usageCount++; - options = options.WithChangedOption(AnalyzerABTestOptions.UsedSuggestedActionCount, usageCount); + options = options.WithChangedOption(AnalyzerABTestOptions.UsedSuggestedActionCount, ++usageCount); if (usageCount >= 3) { isCandidate = true; options = options.WithChangedOption(AnalyzerABTestOptions.HasMetCandidacyRequirements, true); + AnalyzerABTestLogger.Log(nameof(AnalyzerABTestOptions.HasMetCandidacyRequirements)); } + _workspace.Options = options; } } @@ -144,6 +158,8 @@ private void ShowInfoBarIfNecessary() _infoBarChecked = true; if (_experimentationService.IsExperimentEnabled(AnalyzerEnabledFlight)) { + AnalyzerABTestLogger.Log(nameof(AnalyzerEnabledFlight)); + // If we got true from the experimentation service, then we're in the treatment // group, and the experiment is enabled. We determine if the infobar has been // displayed in the past 24 hours. If it hasn't been displayed, then we do so now. @@ -154,6 +170,7 @@ private void ShowInfoBarIfNecessary() if (timeSinceLastShown.TotalDays >= 1) { _workspace.Options = _workspace.Options.WithChangedOption(AnalyzerABTestOptions.LastDateTimeInfoBarShown, utcNow.ToBinary()); + AnalyzerABTestLogger.Log("InfoBarShown"); var infoBarService = _workspace.Services.GetRequiredService(); infoBarService.ShowInfoBarInGlobalView( @@ -173,11 +190,13 @@ private void ShowInfoBarIfNecessary() private void OpenInstallHyperlink() { System.Diagnostics.Process.Start(AnalyzerVsixHyperlink); + AnalyzerABTestLogger.Log(nameof(AnalyzerVsixHyperlink)); } private void DoNotShowAgain() { _workspace.Options = _workspace.Options.WithChangedOption(AnalyzerABTestOptions.NeverShowAgain, true); + AnalyzerABTestLogger.Log(nameof(AnalyzerABTestOptions.NeverShowAgain)); } } } diff --git a/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj b/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj index 684d1931f58b190369ed42c103eecdf30d8f69f6..2698d0eb98e7507695e7f44c39cf233dc069d0c1 100644 --- a/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj +++ b/src/VisualStudio/Core/Def/ServicesVisualStudio.csproj @@ -63,8 +63,6 @@ - - diff --git a/src/Workspaces/Core/Portable/Log/FunctionId.cs b/src/Workspaces/Core/Portable/Log/FunctionId.cs index aa88bec368db7cb72e07e2e8c46282670318136f..a7781252a14c6f74e99666888a228f70c901265e 100644 --- a/src/Workspaces/Core/Portable/Log/FunctionId.cs +++ b/src/Workspaces/Core/Portable/Log/FunctionId.cs @@ -392,5 +392,6 @@ internal enum FunctionId CompilationService_GetCompilationAsync, SolutionCreator_AssetDifferences, Extension_InfoBar, + Experiment_ABTesting, } }