8066448: SmallCodeCacheStartup.java exits with exit code 1

Summary: Check for VirtualMachineError in case VM initialization fails.
Reviewed-by: kvn

hotspot/src/share/vm/oops/method.cpp

@@ -936,7 +936,7 @@ address Method::make_adapters(methodHandle mh, TRAPS) {
   // so making them eagerly shouldn't be too expensive.
   AdapterHandlerEntry* adapter = AdapterHandlerLibrary::get_adapter(mh);
   if (adapter == NULL ) {
-    THROW_MSG_NULL(vmSymbols::java_lang_VirtualMachineError(), "out of space in CodeCache for adapters");
+    THROW_MSG_NULL(vmSymbols::java_lang_VirtualMachineError(), "Out of space in CodeCache for adapters");
   }
 
   mh->set_adapter_entry(adapter);

hotspot/test/compiler/startup/SmallCodeCacheStartup.java

@@ -24,22 +24,29 @@
 /*
  * @test
  * @bug 8023014
- * @summary Test ensures that there is no crash if there is not enough ReservedCodeacacheSize
+ * @summary Test ensures that there is no crash if there is not enough ReservedCodeCacheSize
  *          to initialize all compiler threads. The option -Xcomp gives the VM more time to
- *          to trigger the old bug.
+ *          trigger the old bug.
  * @library /testlibrary
  */
 import com.oracle.java.testlibrary.*;
+import static com.oracle.java.testlibrary.Asserts.assertTrue;
 
 public class SmallCodeCacheStartup {
-  public static void main(String[] args) throws Exception {
-      ProcessBuilder pb = ProcessTools.createJavaProcessBuilder("-XX:ReservedCodeCacheSize=3m",
-                                                                "-XX:CICompilerCount=64",
-                                                                "-Xcomp",
-                                                                "-version");
-      OutputAnalyzer analyzer = new OutputAnalyzer(pb.start());
-      analyzer.shouldHaveExitValue(0);
+    public static void main(String[] args) throws Exception {
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder("-XX:ReservedCodeCacheSize=3m",
+                                                                  "-XX:CICompilerCount=64",
+                                                                  "-Xcomp",
+                                                                  "-version");
+        OutputAnalyzer analyzer = new OutputAnalyzer(pb.start());
+        try {
+            analyzer.shouldHaveExitValue(0);
+        } catch (RuntimeException e) {
+            // Error occurred during initialization, did we run out of adapter space?
+            assertTrue(analyzer.getOutput().contains("VirtualMachineError: Out of space in CodeCache"),
+                    "Expected VirtualMachineError");
+        }
 
-      System.out.println("TEST PASSED");
+        System.out.println("TEST PASSED");
   }
 }