8010389: After fix for 7107135 a failed dlopen() call results in a VM crash

Summary: Call dlerror() in VM thread as necessary.
Reviewed-by: coleenp, dholmes

src/os/linux/vm/os_linux.cpp

@@ -1811,13 +1811,15 @@ bool os::Linux::_stack_is_executable = false;
 class VM_LinuxDllLoad: public VM_Operation {
  private:
   const char *_filename;
+  char *_ebuf;
+  int _ebuflen;
   void *_lib;
  public:
-  VM_LinuxDllLoad(const char *fn) :
-    _filename(fn), _lib(NULL) {}
+  VM_LinuxDllLoad(const char *fn, char *ebuf, int ebuflen) :
+    _filename(fn), _ebuf(ebuf), _ebuflen(ebuflen), _lib(NULL) {}
   VMOp_Type type() const { return VMOp_LinuxDllLoad; }
   void doit() {
-    _lib = os::Linux::dll_load_inner(_filename);
+    _lib = os::Linux::dll_load_in_vmthread(_filename, _ebuf, _ebuflen);
     os::Linux::_stack_is_executable = true;
   }
   void* loaded_library() { return _lib; }
@@ -1865,13 +1867,13 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
             // This is for the case where the DLL has an static
             // constructor function that executes JNI code. We cannot
             // load such DLLs in the VMThread.
-            result = ::dlopen(filename, RTLD_LAZY);
+            result = os::Linux::dlopen_helper(filename, ebuf, ebuflen);
           }
 
           ThreadInVMfromNative tiv(jt);
           debug_only(VMNativeEntryWrapper vew;)
 
-          VM_LinuxDllLoad op(filename);
+          VM_LinuxDllLoad op(filename, ebuf, ebuflen);
           VMThread::execute(&op);
           if (LoadExecStackDllInVMThread) {
             result = op.loaded_library();
@@ -1883,7 +1885,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
   }
 
   if (!load_attempted) {
-    result = ::dlopen(filename, RTLD_LAZY);
+    result = os::Linux::dlopen_helper(filename, ebuf, ebuflen);
   }
 
   if (result != NULL) {
@@ -1892,11 +1894,6 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
   }
 
   Elf32_Ehdr elf_head;
-
-  // Read system error message into ebuf
-  // It may or may not be overwritten below
-  ::strncpy(ebuf, ::dlerror(), ebuflen-1);
-  ebuf[ebuflen-1]='\0';
   int diag_msg_max_length=ebuflen-strlen(ebuf);
   char* diag_msg_buf=ebuf+strlen(ebuf);
 
@@ -2039,10 +2036,19 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
   return NULL;
 }
 
-void * os::Linux::dll_load_inner(const char *filename) {
+void * os::Linux::dlopen_helper(const char *filename, char *ebuf, int ebuflen) {
+  void * result = ::dlopen(filename, RTLD_LAZY);
+  if (result == NULL) {
+    ::strncpy(ebuf, ::dlerror(), ebuflen - 1);
+    ebuf[ebuflen-1] = '\0';
+  }
+  return result;
+}
+
+void * os::Linux::dll_load_in_vmthread(const char *filename, char *ebuf, int ebuflen) {
   void * result = NULL;
   if (LoadExecStackDllInVMThread) {
-    result = ::dlopen(filename, RTLD_LAZY);
+    result = dlopen_helper(filename, ebuf, ebuflen);
   }
 
   // Since 7019808, libjvm.so is linked with -noexecstack. If the VM loads a

src/os/linux/vm/os_linux.hpp

@@ -95,7 +95,8 @@ class Linux {
 
  public:
   static bool _stack_is_executable;
-  static void *dll_load_inner(const char *name);
+  static void *dlopen_helper(const char *name, char *ebuf, int ebuflen);
+  static void *dll_load_in_vmthread(const char *name, char *ebuf, int ebuflen);
 
   static void init_thread_fpu_state();
   static int  get_fpu_control_word();

test/runtime/8010389/VMThreadDlopen.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.File;
+
+/*
+ * @test
+ * @key regression
+ * @bug 8010389
+ * @run main/othervm -Djava.library.path=. VMThreadDlopen
+ */
+
+public class VMThreadDlopen {
+    public static void main(String[] args) throws Exception {
+        File file = new File("libbroken.so");
+        file.createNewFile();
+        try {
+            System.loadLibrary("broken");
+        } catch (UnsatisfiedLinkError e) {
+            e.printStackTrace();
+            // expected
+        }
+    }
+}