Merge pull request #36100 from ivanbasov/retry

support retries in integration tests

eng/build.ps1

@@ -378,6 +378,7 @@ function TestUsingOptimizedRunner() {
     }
 
     $dlls += @(Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir)
+    $args += " -testVsi"
   } else {
     $dlls = Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir
     $args += " -trait:Feature=NetCore"

src/EditorFeatures/TestUtilities/Threading/TestInfo.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System.Collections.Immutable;
+using System.IO;
+using System.Xml.Linq;
+using System.Xml.XPath;
+
+namespace Roslyn.Test.Utilities
+{
+    public readonly struct TestInfo
+    {
+        public decimal Time { get; }
+
+        public TestInfo(decimal time)
+        {
+            Time = time;
+        }
+
+        public static ImmutableDictionary<string, TestInfo> GetPassedTestsInfo()
+        {
+            var result = ImmutableDictionary.CreateBuilder<string, TestInfo>();
+            var filePath = System.Environment.GetEnvironmentVariable("OutputXmlFilePath");
+            if (filePath != null)
+            {
+                if (File.Exists(filePath))
+                {
+                    var doc = XDocument.Load(filePath);
+                    foreach (var test in doc.XPathSelectElements("/assemblies/assembly/collection/test[@result='Pass']"))
+                    {
+                        if (decimal.TryParse(test.Attribute("time").Value, out var time))
+                        {
+                            result.Add(test.Attribute("name").Value, new TestInfo(time));
+                        }
+                    }
+                }
+            }
+
+            return result.ToImmutable();
+        }
+    }
+}

src/EditorFeatures/TestUtilities/Threading/WpfTestRunner.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Reflection;
 using System.Threading;
@@ -25,10 +26,16 @@ namespace Roslyn.Test.Utilities
     /// </summary>
     public sealed class WpfTestRunner : XunitTestRunner
     {
+        private static readonly ImmutableDictionary<string, TestInfo> _passedTests;
         private static string s_wpfFactRequirementReason;
 
         public WpfTestSharedData SharedData { get; }
 
+        static WpfTestRunner()
+        {
+            _passedTests = TestInfo.GetPassedTestsInfo();
+        }
+
         public WpfTestRunner(
             WpfTestSharedData sharedData,
             ITest test,
@@ -63,9 +70,16 @@ protected override Task<decimal> InvokeTestMethodAsync(ExceptionAggregator aggre
                         // Reset our flag ensuring that part of this test actually needs WpfFact
                         s_wpfFactRequirementReason = null;
 
-                        // Just call back into the normal xUnit dispatch process now that we are on an STA Thread with no synchronization context.
-                        var invoker = new XunitTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource);
-                        return invoker.RunAsync().JoinUsingDispatcher(CancellationTokenSource.Token);
+                        if (_passedTests.TryGetValue(Test.DisplayName, out var info))
+                        {
+                            return info.Time;
+                        }
+                        else
+                        {
+                            // Just call back into the normal xUnit dispatch process now that we are on an STA Thread with no synchronization context.
+                            var invoker = new XunitTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource);
+                            return invoker.RunAsync().JoinUsingDispatcher(CancellationTokenSource.Token);
+                        }
                     }
                     finally
                     {

src/EditorFeatures/TestUtilities/Threading/WpfTheoryTestCase.cs

@@ -13,12 +13,15 @@ public class WpfTheoryTestCase : XunitTheoryTestCase
     {
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
-        public WpfTheoryTestCase() { }
+        public WpfTheoryTestCase()
+        {
+        }
 
         public WpfTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod)
             : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod)
         {
         }
+
         public override Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
         {
             var runner = new WpfTheoryTestCaseRunner(WpfTestSharedData.Instance, this, DisplayName, SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource);

src/Tools/Source/RunTests/ProcessTestExecutor.cs

@@ -69,6 +69,19 @@ private string GetResultsFilePath(AssemblyInfo assemblyInfo)
         }
 
         public async Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
+        {
+            var result = await RunTestAsyncInternal(assemblyInfo, retry: false, cancellationToken);
+
+            // For integration tests (TestVsi), we make one more attempt to re-run failed tests.
+            if (Options.TestVsi && !Options.UseHtml && !result.Succeeded)
+            {
+                return await RunTestAsyncInternal(assemblyInfo, retry: true, cancellationToken);
+            }
+
+            return result;
+        }
+
+        private async Task<TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
         {
             try
             {
@@ -81,14 +94,26 @@ public async Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, Cancellati
                 // NOTE: xUnit doesn't always create the log directory
                 Directory.CreateDirectory(resultsDir);
 
-                // NOTE: xUnit seems to have an occasional issue creating logs create
-                // an empty log just in case, so our runner will still fail.
-                File.Create(resultsFilePath).Close();
-
                 // Define environment variables for processes started via ProcessRunner.
                 var environmentVariables = new Dictionary<string, string>();
                 Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
 
+                if (retry)
+                {
+                    // Copy the results file path, since the new xunit run will overwrite it
+                    var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
+                    File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
+
+                    ConsoleUtil.WriteLine("Starting a retry. It will run once again tests failed.");
+                    // If running the process with this varialbe added, we assume that this file contains 
+                    // xml logs from the first attempt.
+                    environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
+                }
+
+                // NOTE: xUnit seems to have an occasional issue creating logs create
+                // an empty log just in case, so our runner will still fail.
+                File.Create(resultsFilePath).Close();
+
                 var start = DateTime.UtcNow;
                 var xunitProcessInfo = ProcessRunner.CreateProcess(
                     ProcessRunner.CreateProcessStartInfo(