8020614: OutputAnalyzer.shouldHaveExitValue() should print stdout/stderr output

Summary: OutputAnalyzer.shouldHaveExitValue() should print stdout/stderr output
Reviewed-by: kvn, ctornqvi, dholmes

test/testlibrary/OutputAnalyzerReportingTest.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+/*
+ * @test
+ * @summary Test the OutputAnalyzer reporting functionality,
+ *     such as printing additional diagnostic info
+ *     (exit code, stdout, stderr, command line, etc.)
+ * @library /testlibrary
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import com.oracle.java.testlibrary.OutputAnalyzer;
+import com.oracle.java.testlibrary.ProcessTools;
+
+
+public class OutputAnalyzerReportingTest {
+
+    public static void main(String[] args) throws Exception {
+        // Create the output analyzer under test
+        String stdout = "aaaaaa";
+        String stderr = "bbbbbb";
+        OutputAnalyzer output = new OutputAnalyzer(stdout, stderr);
+
+        // Expected summary values should be the same for all cases,
+        // since the outputAnalyzer object is the same
+        String expectedExitValue = "-1";
+        String expectedSummary =
+                " stdout: [" + stdout + "];\n" +
+                " stderr: [" + stderr + "]\n" +
+                " exitValue = " + expectedExitValue + "\n";
+
+
+        DiagnosticSummaryTestRunner testRunner =
+                new DiagnosticSummaryTestRunner();
+
+        // should have exit value
+        testRunner.init(expectedSummary);
+        int unexpectedExitValue = 2;
+        try {
+            output.shouldHaveExitValue(unexpectedExitValue);
+        } catch (RuntimeException e) { }
+        testRunner.closeAndCheckResults();
+
+        // should not contain
+        testRunner.init(expectedSummary);
+        try {
+            output.shouldNotContain(stdout);
+        } catch (RuntimeException e) { }
+        testRunner.closeAndCheckResults();
+
+        // should contain
+        testRunner.init(expectedSummary);
+        try {
+            output.shouldContain("unexpected-stuff");
+        } catch (RuntimeException e) { }
+        testRunner.closeAndCheckResults();
+
+        // should not match
+        testRunner.init(expectedSummary);
+        try {
+            output.shouldNotMatch("[a]");
+        } catch (RuntimeException e) { }
+        testRunner.closeAndCheckResults();
+
+        // should match
+        testRunner.init(expectedSummary);
+        try {
+            output.shouldMatch("[qwerty]");
+        } catch (RuntimeException e) { }
+        testRunner.closeAndCheckResults();
+
+    }
+
+    private static class DiagnosticSummaryTestRunner {
+        private ByteArrayOutputStream byteStream =
+                new ByteArrayOutputStream(10000);
+
+        private String expectedSummary = "";
+        private PrintStream errStream;
+
+
+        public void init(String expectedSummary) {
+            this.expectedSummary = expectedSummary;
+            byteStream.reset();
+            errStream = new PrintStream(byteStream);
+            System.setErr(errStream);
+        }
+
+        public void closeAndCheckResults() {
+            // check results
+            errStream.close();
+            String stdErrStr = byteStream.toString();
+            if (!stdErrStr.contains(expectedSummary)) {
+                throw new RuntimeException("The output does not contain "
+                    + "the diagnostic message, or the message is incorrect");
+            }
+        }
+    }
+
+}

test/testlibrary/com/oracle/java/testlibrary/OutputAnalyzer.java

@@ -76,7 +76,8 @@ public final class OutputAnalyzer {
    */
   public void shouldContain(String expectedString) {
     if (!stdout.contains(expectedString) && !stderr.contains(expectedString)) {
-      throw new RuntimeException("'" + expectedString + "' missing from stdout/stderr: [" + stdout + stderr + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + expectedString + "' missing from stdout/stderr \n");
     }
   }
 
@@ -88,7 +89,8 @@ public final class OutputAnalyzer {
    */
   public void stdoutShouldContain(String expectedString) {
     if (!stdout.contains(expectedString)) {
-      throw new RuntimeException("'" + expectedString + "' missing from stdout: [" + stdout + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + expectedString + "' missing from stdout \n");
     }
   }
 
@@ -100,7 +102,8 @@ public final class OutputAnalyzer {
    */
   public void stderrShouldContain(String expectedString) {
     if (!stderr.contains(expectedString)) {
-      throw new RuntimeException("'" + expectedString + "' missing from stderr: [" + stderr + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + expectedString + "' missing from stderr \n");
     }
   }
 
@@ -112,10 +115,12 @@ public final class OutputAnalyzer {
    */
   public void shouldNotContain(String notExpectedString) {
     if (stdout.contains(notExpectedString)) {
-      throw new RuntimeException("'" + notExpectedString + "' found in stdout: [" + stdout + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + notExpectedString + "' found in stdout \n");
     }
     if (stderr.contains(notExpectedString)) {
-      throw new RuntimeException("'" + notExpectedString + "' found in stderr: [" + stderr + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + notExpectedString + "' found in stderr \n");
     }
   }
 
@@ -127,7 +132,8 @@ public final class OutputAnalyzer {
    */
   public void stdoutShouldNotContain(String notExpectedString) {
     if (stdout.contains(notExpectedString)) {
-      throw new RuntimeException("'" + notExpectedString + "' found in stdout: [" + stdout + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + notExpectedString + "' found in stdout \n");
     }
   }
 
@@ -139,7 +145,8 @@ public final class OutputAnalyzer {
    */
   public void stderrShouldNotContain(String notExpectedString) {
     if (stderr.contains(notExpectedString)) {
-      throw new RuntimeException("'" + notExpectedString + "' found in stderr: [" + stderr + "]\n");
+        reportDiagnosticSummary();
+        throw new RuntimeException("'" + notExpectedString + "' found in stderr \n");
     }
   }
 
@@ -154,9 +161,9 @@ public final class OutputAnalyzer {
       Matcher stdoutMatcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stdout);
       Matcher stderrMatcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stderr);
       if (!stdoutMatcher.find() && !stderrMatcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' missing from stdout/stderr: [" + stdout + stderr
-                  + "]\n");
+                + "' missing from stdout/stderr \n");
       }
   }
 
@@ -170,8 +177,9 @@ public final class OutputAnalyzer {
   public void stdoutShouldMatch(String pattern) {
       Matcher matcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stdout);
       if (!matcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' missing from stdout: [" + stdout + "]\n");
+                + "' missing from stdout \n");
       }
   }
 
@@ -185,8 +193,9 @@ public final class OutputAnalyzer {
   public void stderrShouldMatch(String pattern) {
       Matcher matcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stderr);
       if (!matcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' missing from stderr: [" + stderr + "]\n");
+                + "' missing from stderr \n");
       }
   }
 
@@ -200,13 +209,15 @@ public final class OutputAnalyzer {
   public void shouldNotMatch(String pattern) {
       Matcher matcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stdout);
       if (matcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' found in stdout: [" + stdout + "]\n");
+                  + "' found in stdout \n");
       }
       matcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stderr);
       if (matcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' found in stderr: [" + stderr + "]\n");
+                  + "' found in stderr \n");
       }
   }
 
@@ -220,8 +231,9 @@ public final class OutputAnalyzer {
   public void stdoutShouldNotMatch(String pattern) {
       Matcher matcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stdout);
       if (matcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' found in stdout: [" + stdout + "]\n");
+                  + "' found in stdout \n");
       }
   }
 
@@ -235,23 +247,45 @@ public final class OutputAnalyzer {
   public void stderrShouldNotMatch(String pattern) {
       Matcher matcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stderr);
       if (matcher.find()) {
+          reportDiagnosticSummary();
           throw new RuntimeException("'" + pattern
-                  + "' found in stderr: [" + stderr + "]\n");
+                  + "' found in stderr \n");
       }
   }
 
   /**
-   * Verifiy the exit value of the process
+   * Verify the exit value of the process
    *
    * @param expectedExitValue Expected exit value from process
    * @throws RuntimeException If the exit value from the process did not match the expected value
    */
   public void shouldHaveExitValue(int expectedExitValue) {
       if (getExitValue() != expectedExitValue) {
-          throw new RuntimeException("Exit value " + getExitValue() + " , expected to get " + expectedExitValue);
+          reportDiagnosticSummary();
+          throw new RuntimeException("Expected to get exit value of ["
+                  + expectedExitValue + "]\n");
       }
   }
 
+
+  /**
+   * Report summary that will help to diagnose the problem
+   * Currently includes:
+   *  - standard input produced by the process under test
+   *  - standard output
+   *  - exit code
+   *  Note: the command line is printed by the ProcessTools
+   */
+    private void reportDiagnosticSummary() {
+        String msg =
+            " stdout: [" + stdout + "];\n" +
+            " stderr: [" + stderr + "]\n" +
+            " exitValue = " + getExitValue() + "\n";
+
+        System.err.println(msg);
+    }
+
+
   /**
    * Get the contents of the output buffer (stdout and stderr)
    *

test/testlibrary/com/oracle/java/testlibrary/ProcessTools.java

@@ -31,6 +31,7 @@ import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 import sun.management.VMManagement;
 
@@ -106,6 +107,22 @@ public final class ProcessTools {
     return pid;
   }
 
+  /**
+   * Get the string containing input arguments passed to the VM
+   *
+   * @return arguments
+   */
+  public static String getVmInputArguments() {
+    RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
+
+    List<String> args = runtime.getInputArguments();
+    StringBuilder result = new StringBuilder();
+    for (String arg : args)
+        result.append(arg).append(' ');
+
+    return result.toString();
+  }
+
   /**
    * Get platform specific VM arguments (e.g. -d64 on 64bit Solaris)
    *
@@ -132,8 +149,13 @@ public final class ProcessTools {
     Collections.addAll(args, getPlatformSpecificVMArgs());
     Collections.addAll(args, command);
 
-    return new ProcessBuilder(args.toArray(new String[args.size()]));
+    // Reporting
+    StringBuilder cmdLine = new StringBuilder();
+    for (String cmd : args)
+        cmdLine.append(cmd).append(' ');
+    System.out.println("Command line: [" + cmdLine.toString() + "]");
 
+    return new ProcessBuilder(args.toArray(new String[args.size()]));
   }
 
 }