Merge pull request #2188 from jglick/flakiness

Use rerunFailingTestsCount rather than @RandomlyFails
(cherry picked from commit c54c363b)

pom.xml

@@ -365,7 +365,7 @@ THE SOFTWARE.
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.16</version> <!-- ignoring ${maven-surefire-plugin.version} -->
+          <version>2.19.1</version> <!-- ignoring ${maven-surefire-plugin.version} -->
           <configuration>
             <argLine>-noverify</argLine> <!-- some versions of JDK7/8 causes VerifyError during mock tests: http://code.google.com/p/powermock/issues/detail?id=504 -->
             <systemPropertyVariables>

test/pom.xml

@@ -39,7 +39,6 @@ THE SOFTWARE.
   <properties>
     <concurrency>2</concurrency> <!-- may use e.g. 2C for 2 × (number of cores) -->
     <mavenDebug>false</mavenDebug>
-    <ignore.random.failures>false</ignore.random.failures>
     <jacocoSurefireArgs /><!-- empty by default -->
   </properties>
 
@@ -236,7 +235,6 @@ THE SOFTWARE.
               <hudson.ClassicPluginStrategy.useAntClassLoader>true</hudson.ClassicPluginStrategy.useAntClassLoader>
               <hudson.maven.debug>${mavenDebug}</hudson.maven.debug>
               <buildDirectory>${project.build.directory}</buildDirectory>
-              <ignore.random.failures>${ignore.random.failures}</ignore.random.failures>
           </systemPropertyVariables>
           <reuseForks>true</reuseForks>
           <forkCount>${concurrency}</forkCount>
@@ -307,6 +305,7 @@ THE SOFTWARE.
       </activation>
       <properties>
         <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
+        <surefire.rerunFailingTestsCount>4</surefire.rerunFailingTestsCount>
       </properties>
     </profile>
     <profile>

test/src/test/groovy/hudson/cli/BuildCommandTest.groovy

@@ -35,7 +35,6 @@ import org.junit.Test
 import org.jvnet.hudson.test.Issue
 import org.jvnet.hudson.test.CaptureEnvironmentBuilder
 import org.jvnet.hudson.test.JenkinsRule
-import org.jvnet.hudson.test.RandomlyFails
 import org.jvnet.hudson.test.TestBuilder
 import org.jvnet.hudson.test.TestExtension
 import org.kohsuke.stapler.StaplerRequest
@@ -159,7 +158,7 @@ public class BuildCommandTest {
         }
     }
 
-    @RandomlyFails("Started test0 #1")
+    // TODO randomly fails: Started test0 #1
     @Test void consoleOutput() {
         def p = j.createFreeStyleProject()
         def cli = new CLI(j.URL)
@@ -174,7 +173,7 @@ public class BuildCommandTest {
         }
     }
 
-    @RandomlyFails("Started test0 #1")
+    // TODO randomly fails: Started test0 #1
     @Test void consoleOutputWhenBuildSchedulingRefused() {
         def p = j.createFreeStyleProject()
         def cli = new CLI(j.URL)

test/src/test/java/hudson/model/UpdateCenter2Test.java

@@ -31,9 +31,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.jvnet.hudson.test.Issue;
 import org.jvnet.hudson.test.JenkinsRule;
-import org.jvnet.hudson.test.RandomlyFails;
-
-import java.io.IOException;
 
 /**
  *
@@ -47,7 +44,7 @@ public class UpdateCenter2Test {
     /**
      * Makes sure a plugin installs fine.
      */
-    @RandomlyFails("SocketTimeoutException from goTo due to GET http://localhost:…/update-center.json?…")
+    // TODO randomly fails: SocketTimeoutException from goTo due to GET http://localhost:…/update-center.json?…
     @Test public void install() throws Exception {
         UpdateSite.neverUpdate = false;
         j.jenkins.pluginManager.doCheckUpdatesServer(); // load the metadata

test/src/test/java/hudson/model/UpdateCenterTest.java

@@ -47,11 +47,8 @@ public class UpdateCenterTest {
             doData("http://updates.jenkins-ci.org/update-center.json?version=build");
             doData("http://updates.jenkins-ci.org/stable/update-center.json?version=build");
         } catch (Exception x) {
-            if (Boolean.getBoolean("ignore.random.failures")) {
-                assumeNoException("Might be no Internet connectivity, or might start failing due to expiring certificate through no fault of code changes", x);
-            } else {
-                throw x;
-            }
+            // TODO this should not be in core at all; should be in repo built by a separate job somewhere
+            assumeNoException("Might be no Internet connectivity, or might start failing due to expiring certificate through no fault of code changes", x);
         }
     }
     private void doData(String location) throws Exception {

test/src/test/java/hudson/slaves/NodeProvisionerTest.java

@@ -39,7 +39,6 @@ import static org.junit.Assert.*;
 import org.junit.Rule;
 import org.junit.Test;
 import org.jvnet.hudson.test.JenkinsRule;
-import org.jvnet.hudson.test.RandomlyFails;
 import org.jvnet.hudson.test.SleepBuilder;
 
 /**
@@ -85,7 +84,7 @@ public class NodeProvisionerTest {
     /**
      * Scenario: schedule a build and see if one slave is provisioned.
      */
-    @RandomlyFails("fragile")
+    // TODO fragile
     @Test public void autoProvision() throws Exception {
         BulkChange bc = new BulkChange(r.jenkins);
         try {
@@ -107,7 +106,7 @@ public class NodeProvisionerTest {
     /**
      * Scenario: we got a lot of jobs all of the sudden, and we need to fire up a few nodes.
      */
-    @RandomlyFails("fragile")
+    // TODO fragile
     @Test public void loadSpike() throws Exception {
         BulkChange bc = new BulkChange(r.jenkins);
         try {
@@ -126,7 +125,7 @@ public class NodeProvisionerTest {
     /**
      * Scenario: make sure we take advantage of statically configured slaves.
      */
-    @RandomlyFails("fragile")
+    // TODO fragile
     @Test public void baselineSlaveUsage() throws Exception {
         BulkChange bc = new BulkChange(r.jenkins);
         try {
@@ -147,7 +146,7 @@ public class NodeProvisionerTest {
     /**
      * Scenario: loads on one label shouldn't translate to load on another label.
      */
-    @RandomlyFails("fragile")
+    // TODO fragile
     @Test public void labels() throws Exception {
         BulkChange bc = new BulkChange(r.jenkins);
         try {

test/src/test/java/hudson/tasks/FingerprinterTest.java

@@ -54,7 +54,6 @@ import org.junit.Test;
 import org.jvnet.hudson.test.Issue;
 
 import org.jvnet.hudson.test.JenkinsRule;
-import org.jvnet.hudson.test.RandomlyFails;
 import org.jvnet.hudson.test.recipes.LocalData;
 
 /**
@@ -315,7 +314,7 @@ public class FingerprinterTest {
     }
 
     @SuppressWarnings("unchecked")
-    @RandomlyFails("for p3.upstreamProjects expected:<[hudson.model.FreeStyleProject@590e5b8[test0]]> but was:<[]>")
+    // TODO randomly fails: for p3.upstreamProjects expected:<[hudson.model.FreeStyleProject@590e5b8[test0]]> but was:<[]>
     @Issue("JENKINS-18417")
     @Test
     public void fingerprintCleanup() throws Exception {