Introduced @RandomlyFails annotation to make it easier to skip unreliable functional tests.

First tried using excludedGroups and trying to work around SUREFIRE-984.
Made some progress, but not ideal since marked tests are omitted from output, rather than being listed as ignored.
Throwing AssumptionViolatedException from JenkinsRule is much simpler and seems to work better.
https://github.com/junit-team/junit/issues/116 notes its limitations.

test/pom.xml

@@ -42,6 +42,7 @@ THE SOFTWARE.
   <properties>
     <concurrency>2</concurrency> <!-- may use e.g. 2C for 2 × (number of cores) -->
     <mavenDebug>false</mavenDebug>
+    <ignore.random.failures>false</ignore.random.failures>
   </properties>
 
   <dependencies>
@@ -187,6 +188,7 @@ THE SOFTWARE.
               <hudson.ClassicPluginStrategy.useAntClassLoader>true</hudson.ClassicPluginStrategy.useAntClassLoader>
               <hudson.maven.debug>${mavenDebug}</hudson.maven.debug>
               <buildDirectory>${project.build.directory}</buildDirectory>
+              <ignore.random.failures>${ignore.random.failures}</ignore.random.failures>
           </systemPropertyVariables>
           <reuseForks>true</reuseForks>
           <forkCount>${concurrency}</forkCount>

test/src/main/java/org/jvnet/hudson/test/JenkinsRule.java

@@ -215,6 +215,7 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.junit.Assert.*;
+import org.junit.internal.AssumptionViolatedException;
 import static org.junit.matchers.JUnitMatchers.containsString;
 
 import org.junit.rules.TemporaryFolder;
@@ -314,6 +315,14 @@ public class JenkinsRule implements TestRule, MethodRule, RootAction {
      * @throws Throwable if setup fails (which will disable {@code after}
      */
     public void before() throws Throwable {
+        // Not ideal (https://github.com/junit-team/junit/issues/116) but basically works.
+        if (Boolean.getBoolean("ignore.random.failures")) {
+            RandomlyFails rf = testDescription.getAnnotation(RandomlyFails.class);
+            if (rf != null) {
+                throw new AssumptionViolatedException("Known to randomly fail: " + rf.value());
+            }
+        }
+
         env = new TestEnvironment(testDescription);
         env.pin();
         recipe();
@@ -484,6 +493,11 @@ public class JenkinsRule implements TestRule, MethodRule, RootAction {
                             throw new BreakException();
                         } catch (BreakException e) {}
 
+                        RandomlyFails rf = testDescription.getAnnotation(RandomlyFails.class);
+                        if (rf != null) {
+                            System.err.println("Note: known to randomly fail: " + rf.value());
+                        }
+
                         // dump threads
                         ThreadInfo[] threadInfos = Functions.getThreadInfos();
                         Functions.ThreadGroupMap m = Functions.sortThreadsAndGetGroupMap(threadInfos);

test/src/main/java/org/jvnet/hudson/test/RandomlyFails.java

+/*
+ * The MIT License
+ *
+ * Copyright 2014 Jesse Glick.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package org.jvnet.hudson.test;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.Test;
+
+/**
+ * Marker for test methods which have been observed to fail for no easily understood reason.
+ * They may be useful to keep in sources so developers and certain CI jobs can run them,
+ * but failures should not block a validated merge build for example.
+ * Only effective on {@link Test}-annotated (but not {@link WithoutJenkins}-annotated) methods in a class using {@link JenkinsRule}
+ * when the system property {@code ignore.random.failures} is true.
+ * @since 1.547
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface RandomlyFails {
+
+    /** Observed failure message or other explanation. */
+    String value();
+
+}

test/src/test/groovy/hudson/cli/BuildCommandTest.groovy

@@ -25,10 +25,10 @@ package hudson.cli
 
 import org.apache.commons.io.output.TeeOutputStream
 import static org.junit.Assert.*
-import org.junit.Assume
 import org.junit.Rule
 import org.junit.Test
 import org.jvnet.hudson.test.JenkinsRule
+import org.jvnet.hudson.test.RandomlyFails
 import org.jvnet.hudson.test.TestBuilder
 import org.jvnet.hudson.test.TestExtension
 
@@ -140,8 +140,8 @@ public class BuildCommandTest {
         }
     }
 
+    @RandomlyFails("Started test0 #1")
     @Test void consoleOutput() {
-        Assume.assumeFalse("Started test0 #1", System.getenv("JOB_URL") != null);
         def p = j.createFreeStyleProject()
         def cli = new CLI(j.URL)
         try {
@@ -155,8 +155,8 @@ public class BuildCommandTest {
         }
     }
 
+    @RandomlyFails("Started test0 #1")
     @Test void consoleOutputWhenBuildSchedulingRefused() {
-        Assume.assumeFalse("Started test0 #1", System.getenv("JOB_URL") != null);
         def p = j.createFreeStyleProject()
         def cli = new CLI(j.URL)
         try {

test/src/test/groovy/hudson/matrix/MatrixProjectTest.groovy

@@ -67,10 +67,10 @@ import java.util.concurrent.CountDownLatch
 
 import static hudson.model.Node.Mode.EXCLUSIVE
 import static org.junit.Assert.*
-import org.junit.Assume
 import org.junit.Rule
 import org.junit.Test
 import org.jvnet.hudson.test.JenkinsRule
+import org.jvnet.hudson.test.RandomlyFails
 import org.junit.rules.TemporaryFolder
 
 /**
@@ -108,9 +108,9 @@ public class MatrixProjectTest {
     /**
      * Tests that axes are available as build variables in the Maven builds.
      */
+    @RandomlyFails("Not a v4.0.0 POM. for project org.jvnet.maven-antrun-extended-plugin:maven-antrun-extended-plugin at /home/jenkins/.m2/repository/org/jvnet/maven-antrun-extended-plugin/maven-antrun-extended-plugin/1.40/maven-antrun-extended-plugin-1.40.pom")
     @Test
     public void testBuildAxisInMaven() throws Exception {
-        Assume.assumeFalse("Not a v4.0.0 POM. for project org.jvnet.maven-antrun-extended-plugin:maven-antrun-extended-plugin at /home/jenkins/.m2/repository/org/jvnet/maven-antrun-extended-plugin/maven-antrun-extended-plugin/1.40/maven-antrun-extended-plugin-1.40.pom", System.getenv("JOB_URL") != null);
         MatrixProject p = createMatrixProject();
         Maven.MavenInstallation maven = j.configureDefaultMaven();
         p.getBuildersList().add(new Maven('-Dprop=${db} validate',maven.getName()));

test/src/test/groovy/hudson/model/UpdateCenter2Test.groovy

@@ -24,10 +24,10 @@
 package hudson.model
 
 import org.jvnet.hudson.test.JenkinsRule
+import org.jvnet.hudson.test.RandomlyFails
 import org.junit.Rule
 import org.junit.Test
 import static org.junit.Assert.*
-import org.junit.Assume
 import hudson.model.UpdateCenter.DownloadJob.Success
 import hudson.model.UpdateSite
 
@@ -43,8 +43,8 @@ public class UpdateCenter2Test {
     /**
      * Makes sure a plugin installs fine.
      */
+    @RandomlyFails("SocketTimeoutException from goTo due to GET http://localhost:…/update-center.json?…")
     @Test void install() {
-        Assume.assumeFalse("SocketTimeoutException from goTo due to GET http://localhost:…/update-center.json?…", System.getenv("JOB_URL") != null);
         UpdateSite.neverUpdate = false;
         j.createWebClient().goTo("") // load the metadata
         def job = j.jenkins.updateCenter.getPlugin("changelog-history").deploy().get(); // this seems like one of the smallest plugin

test/src/test/java/hudson/tasks/FingerprinterTest.java

@@ -47,13 +47,13 @@ import static org.junit.Assert.*;
 
 import hudson.util.StreamTaskListener;
 import jenkins.model.Jenkins;
-import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.jvnet.hudson.test.Bug;
 
 import org.jvnet.hudson.test.JenkinsRule;
+import org.jvnet.hudson.test.RandomlyFails;
 import org.jvnet.hudson.test.recipes.LocalData;
 
 /**
@@ -294,10 +294,10 @@ public class FingerprinterTest {
     }
 
     @SuppressWarnings("unchecked")
+    @RandomlyFails("for p3.upstreamProjects expected:<[hudson.model.FreeStyleProject@590e5b8[test0]]> but was:<[]>")
     @Bug(18417)
     @Test
     public void fingerprintCleanup() throws Exception {
-        Assume.assumeFalse("for p3.upstreamProjects expected:<[hudson.model.FreeStyleProject@590e5b8[test0]]> but was:<[]>", System.getenv("JOB_URL") != null);
         // file names shouldn't matter
         FreeStyleProject p1 = createFreeStyleProjectWithFingerprints(singleContents, singleFiles);
         FreeStyleProject p2 = createFreeStyleProjectWithFingerprints(singleContents, singleFiles2);

test/src/test/java/hudson/tasks/junit/JUnitResultArchiverTest.java

@@ -37,12 +37,12 @@ import org.jvnet.hudson.test.recipes.LocalData;
 import com.gargoylesoftware.htmlunit.html.HtmlForm;
 import com.gargoylesoftware.htmlunit.html.HtmlPage;
 import static org.junit.Assert.*;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.jvnet.hudson.test.JenkinsRule;
 import org.jvnet.hudson.test.JenkinsRule.WebClient;
+import org.jvnet.hudson.test.RandomlyFails;
 
 public class JUnitResultArchiverTest {
 
@@ -75,9 +75,9 @@ public class JUnitResultArchiverTest {
 
 	}
 
+    @RandomlyFails("TimeoutException from basic")
    @LocalData
    @Test public void slave() throws Exception {
-        Assume.assumeFalse("TimeoutException from basic", System.getenv("JOB_URL") != null);
         DumbSlave s = j.createOnlineSlave();
         project.setAssignedLabel(s.getSelfLabel());