7107135: Stack guard pages are no more protected after loading a shared...

7107135: Stack guard pages are no more protected after loading a shared library with executable stack
Summary: Detect the execstack attribute of the loaded library and attempt to fix the stack guard using Safepoint op.
Reviewed-by: dholmes, zgu
Contributed-by: ioi.lam@oracle.com

src/os/linux/vm/globals_linux.hpp

 /*
- * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,9 @@
   product(bool, UseHugeTLBFS, false,                                    \
           "Use MAP_HUGETLB for large pages")                            \
                                                                         \
+  product(bool, LoadExecStackDllInVMThread, true,                       \
+          "Load DLLs with executable-stack attribute in the VM Thread") \
+                                                                        \
   product(bool, UseSHM, false,                                          \
           "Use SYSV shared memory for large pages")
 

src/os/linux/vm/os_linux.cpp

@@ -44,6 +44,7 @@
 #include "runtime/extendedPC.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/interfaceSupport.hpp"
+#include "runtime/init.hpp"
 #include "runtime/java.hpp"
 #include "runtime/javaCalls.hpp"
 #include "runtime/mutexLocker.hpp"
@@ -61,6 +62,7 @@
 #include "utilities/decoder.hpp"
 #include "utilities/defaultStream.hpp"
 #include "utilities/events.hpp"
+#include "utilities/elfFile.hpp"
 #include "utilities/growableArray.hpp"
 #include "utilities/vmError.hpp"
 
@@ -1796,9 +1798,93 @@ bool os::dll_address_to_library_name(address addr, char* buf,
   // in case of error it checks if .dll/.so was built for the
   // same architecture as Hotspot is running on
 
+
+// Remember the stack's state. The Linux dynamic linker will change
+// the stack to 'executable' at most once, so we must safepoint only once.
+bool os::Linux::_stack_is_executable = false;
+
+// VM operation that loads a library.  This is necessary if stack protection
+// of the Java stacks can be lost during loading the library.  If we
+// do not stop the Java threads, they can stack overflow before the stacks
+// are protected again.
+class VM_LinuxDllLoad: public VM_Operation {
+ private:
+  const char *_filename;
+  void *_lib;
+ public:
+  VM_LinuxDllLoad(const char *fn) :
+    _filename(fn), _lib(NULL) {}
+  VMOp_Type type() const { return VMOp_LinuxDllLoad; }
+  void doit() {
+    _lib = os::Linux::dll_load_inner(_filename);
+    os::Linux::_stack_is_executable = true;
+  }
+  void* loaded_library() { return _lib; }
+};
+
 void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
 {
-  void * result= ::dlopen(filename, RTLD_LAZY);
+  void * result = NULL;
+  bool load_attempted = false;
+
+  // Check whether the library to load might change execution rights
+  // of the stack. If they are changed, the protection of the stack
+  // guard pages will be lost. We need a safepoint to fix this.
+  //
+  // See Linux man page execstack(8) for more info.
+  if (os::uses_stack_guard_pages() && !os::Linux::_stack_is_executable) {
+    ElfFile ef(filename);
+    if (!ef.specifies_noexecstack()) {
+      if (!is_init_completed()) {
+        os::Linux::_stack_is_executable = true;
+        // This is OK - No Java threads have been created yet, and hence no
+        // stack guard pages to fix.
+        //
+        // This should happen only when you are building JDK7 using a very
+        // old version of JDK6 (e.g., with JPRT) and running test_gamma.
+        //
+        // Dynamic loader will make all stacks executable after
+        // this function returns, and will not do that again.
+        assert(Threads::first() == NULL, "no Java threads should exist yet.");
+      } else {
+        warning("You have loaded library %s which might have disabled stack guard. "
+                "The VM will try to fix the stack guard now.\n"
+                "It's highly recommended that you fix the library with "
+                "'execstack -c <libfile>', or link it with '-z noexecstack'.",
+                filename);
+
+        assert(Thread::current()->is_Java_thread(), "must be Java thread");
+        JavaThread *jt = JavaThread::current();
+        if (jt->thread_state() != _thread_in_native) {
+          // This happens when a compiler thread tries to load a hsdis-<arch>.so file
+          // that requires ExecStack. Cannot enter safe point. Let's give up.
+          warning("Unable to fix stack guard. Giving up.");
+        } else {
+          if (!LoadExecStackDllInVMThread) {
+            // This is for the case where the DLL has an static
+            // constructor function that executes JNI code. We cannot
+            // load such DLLs in the VMThread.
+            result = ::dlopen(filename, RTLD_LAZY);
+          }
+
+          ThreadInVMfromNative tiv(jt);
+          debug_only(VMNativeEntryWrapper vew;)
+
+          VM_LinuxDllLoad op(filename);
+          VMThread::execute(&op);
+          if (LoadExecStackDllInVMThread) {
+            result = op.loaded_library();
+          }
+          load_attempted = true;
+        }
+      }
+    }
+  }
+
+  if (!load_attempted) {
+    result = ::dlopen(filename, RTLD_LAZY);
+  }
+
   if (result != NULL) {
     // Successful loading
     return result;
@@ -1952,6 +2038,38 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen)
   return NULL;
 }
 
+void * os::Linux::dll_load_inner(const char *filename) {
+  void * result = NULL;
+  if (LoadExecStackDllInVMThread) {
+    result = ::dlopen(filename, RTLD_LAZY);
+  }
+
+  // Since 7019808, libjvm.so is linked with -noexecstack. If the VM loads a
+  // library that requires an executable stack, or which does not have this
+  // stack attribute set, dlopen changes the stack attribute to executable. The
+  // read protection of the guard pages gets lost.
+  //
+  // Need to check _stack_is_executable again as multiple VM_LinuxDllLoad
+  // may have been queued at the same time.
+
+  if (!_stack_is_executable) {
+    JavaThread *jt = Threads::first();
+
+    while (jt) {
+      if (!jt->stack_guard_zone_unused() &&        // Stack not yet fully initialized
+          jt->stack_yellow_zone_enabled()) {       // No pending stack overflow exceptions
+        if (!os::guard_memory((char *) jt->stack_red_zone_base() - jt->stack_red_zone_size(),
+                              jt->stack_yellow_zone_size() + jt->stack_red_zone_size())) {
+          warning("Attempt to reguard stack yellow zone failed.");
+        }
+      }
+      jt = jt->next();
+    }
+  }
+
+  return result;
+}
+
 /*
  * glibc-2.0 libdl is not MT safe.  If you are building with any glibc,
  * chances are you might want to run the generated bits against glibc-2.0

src/os/linux/vm/os_linux.hpp

@@ -94,6 +94,9 @@ class Linux {
   static void print_libversion_info(outputStream* st);
 
  public:
+  static bool _stack_is_executable;
+  static void *dll_load_inner(const char *name);
+
   static void init_thread_fpu_state();
   static int  get_fpu_control_word();
   static void set_fpu_control_word(int fpu_control);

src/os_cpu/linux_sparc/vm/os_linux_sparc.cpp

@@ -410,6 +410,11 @@ inline static bool checkOverflow(sigcontext* uc,
       // to handle_unexpected_exception way down below.
       thread->disable_stack_red_zone();
       tty->print_raw_cr("An irrecoverable stack overflow has occurred.");
+
+      // This is a likely cause, but hard to verify. Let's just print
+      // it as a hint.
+      tty->print_raw_cr("Please check if any of your loaded .so files has "
+                        "enabled executable stack (see man page execstack(8))");
     } else {
       // Accessing stack address below sp may cause SEGV if current
       // thread has MAP_GROWSDOWN stack. This should only happen when

src/os_cpu/linux_x86/vm/os_linux_x86.cpp

@@ -305,6 +305,11 @@ JVM_handle_linux_signal(int sig,
           // to handle_unexpected_exception way down below.
           thread->disable_stack_red_zone();
           tty->print_raw_cr("An irrecoverable stack overflow has occurred.");
+
+          // This is a likely cause, but hard to verify. Let's just print
+          // it as a hint.
+          tty->print_raw_cr("Please check if any of your loaded .so files has "
+                            "enabled executable stack (see man page execstack(8))");
         } else {
           // Accessing stack address below sp may cause SEGV if current
           // thread has MAP_GROWSDOWN stack. This should only happen when

src/share/vm/runtime/thread.hpp

 /*
- * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1289,6 +1289,7 @@ class JavaThread: public Thread {
   void enable_stack_red_zone();
   void disable_stack_red_zone();
 
+  inline bool stack_guard_zone_unused();
   inline bool stack_yellow_zone_disabled();
   inline bool stack_yellow_zone_enabled();
 
@@ -1759,6 +1760,10 @@ inline CompilerThread* JavaThread::as_CompilerThread() {
   return (CompilerThread*)this;
 }
 
+inline bool JavaThread::stack_guard_zone_unused() {
+  return _stack_guard_state == stack_guard_unused;
+}
+
 inline bool JavaThread::stack_yellow_zone_disabled() {
   return _stack_guard_state == stack_guard_yellow_disabled;
 }

src/share/vm/runtime/vm_operations.hpp

@@ -94,6 +94,7 @@
   template(ReportJavaOutOfMemory)                 \
   template(JFRCheckpoint)                         \
   template(Exit)                                  \
+  template(LinuxDllLoad)                          \
 
 class VM_Operation: public CHeapObj<mtInternal> {
  public:

src/share/vm/utilities/elfFile.cpp

@@ -197,4 +197,28 @@ ElfStringTable* ElfFile::get_string_table(int index) {
   return NULL;
 }
 
+#ifdef LINUX
+bool ElfFile::specifies_noexecstack() {
+  Elf_Phdr phdr;
+  if (!m_file)  return true;
+
+  if (!fseek(m_file, m_elfHdr.e_phoff, SEEK_SET)) {
+    for (int index = 0; index < m_elfHdr.e_phnum; index ++) {
+      if (fread((void*)&phdr, sizeof(Elf_Phdr), 1, m_file) != 1) {
+        m_status = NullDecoder::file_invalid;
+        return false;
+      }
+      if (phdr.p_type == PT_GNU_STACK) {
+        if (phdr.p_flags == (PF_R | PF_W))  {
+          return true;
+        } else {
+          return false;
+        }
+      }
+    }
+  }
+  return false;
+}
+#endif
+
 #endif // _WINDOWS

src/share/vm/utilities/elfFile.hpp

@@ -43,6 +43,7 @@ typedef Elf64_Addr      Elf_Addr;
 
 typedef Elf64_Ehdr      Elf_Ehdr;
 typedef Elf64_Shdr      Elf_Shdr;
+typedef Elf64_Phdr      Elf_Phdr;
 typedef Elf64_Sym       Elf_Sym;
 
 #if !defined(_ALLBSD_SOURCE) || defined(__APPLE__)
@@ -59,6 +60,7 @@ typedef Elf32_Addr      Elf_Addr;
 
 typedef Elf32_Ehdr      Elf_Ehdr;
 typedef Elf32_Shdr      Elf_Shdr;
+typedef Elf32_Phdr      Elf_Phdr;
 typedef Elf32_Sym       Elf_Sym;
 
 #if !defined(_ALLBSD_SOURCE) || defined(__APPLE__)
@@ -123,6 +125,14 @@ protected:
    ElfFile*  next() const { return m_next; }
    void set_next(ElfFile* file) { m_next = file; }
 
+ public:
+  // Returns true if the elf file is marked NOT to require an executable stack,
+  // or if the file could not be opened.
+  // Returns false if the elf file requires an executable stack, the stack flag
+  // is not set at all, or if the file can not be read.
+  // On systems other than linux it always returns false.
+  bool specifies_noexecstack() NOT_LINUX({ return false; });
+
  protected:
     ElfFile*         m_next;
 

test/runtime/7107135/Test.java

+/*
+ * Copyright (c) 2002-2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 SAP AG.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+class Test {
+
+    static boolean loadLib(String libName){
+        try {
+            System.loadLibrary(libName);
+            System.out.println("Loaded library "+ libName + ".");
+            return true;
+        } catch (SecurityException e) {
+            System.out.println("loadLibrary(\"" + libName + "\") throws: " + e + "\n");
+        } catch (UnsatisfiedLinkError e) {
+            System.out.println("loadLibrary(\"" + libName + "\") throws: " + e + "\n");
+        }
+        return false;
+    }
+
+    public static int counter = 1;
+
+    static int Runner() {
+        counter = counter * -1;
+        int i = counter;
+        if(counter < 2) counter += Runner();
+        return i;
+    }
+
+    public static int run() {
+        try{
+            Runner();
+        } catch (StackOverflowError e) {
+            System.out.println("Caught stack overflow error.");
+            return 0;
+        } catch (OutOfMemoryError e) {
+            return 0;
+        }
+        return 2;
+    }
+
+    public static void main(String argv[]) {
+        loadLib(argv[0]);
+        System.exit(run());
+    }
+}

test/runtime/7107135/Test7107135.sh

+#!/bin/sh
+
+#
+#  Copyright (c) 1998, 2013, Oracle and/or its affiliates. All rights reserved.
+#  Copyright (c) 2011 SAP AG.  All Rights Reserved.
+#  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+#  This code is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License version 2 only, as
+#  published by the Free Software Foundation.
+#
+#  This code is distributed in the hope that it will be useful, but WITHOUT
+#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+#  version 2 for more details (a copy is included in the LICENSE file that
+#  accompanied this code).
+#
+#  You should have received a copy of the GNU General Public License version
+#  2 along with this work; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#  Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+#  or visit www.oracle.com if you need additional information or have any
+#  questions.
+#
+
+##
+## @test Test7107135.sh
+## @bug 7107135
+## @summary Stack guard pages lost after loading library with executable stack.
+## @run shell Test7107135.sh
+##
+
+if [ "${TESTSRC}" = "" ]
+then TESTSRC=.
+fi
+
+if [ "${TESTJAVA}" = "" ]
+then
+  PARENT=`dirname \`which java\``
+  TESTJAVA=`dirname ${PARENT}`
+  echo "TESTJAVA not set, selecting " ${TESTJAVA}
+  echo "If this is incorrect, try setting the variable manually."
+fi
+
+BIT_FLAG=""
+
+# set platform-dependent variables
+OS=`uname -s`
+case "$OS" in
+  Linux)
+    NULL=/dev/null
+    PS=":"
+    FS="/"
+    ;;
+  *)
+    NULL=NUL
+    PS=";"
+    FS="\\"
+    echo "Test passed; only valid for Linux"
+    exit 0;
+    ;;
+esac
+
+ARCH=`uname -m`
+
+THIS_DIR=`pwd`
+
+cp ${TESTSRC}${FS}*.java ${THIS_DIR}
+${TESTJAVA}${FS}bin${FS}javac *.java
+
+gcc -fPIC -shared -c -o test.o -I${TESTJAVA}${FS}include -I${TESTJAVA}${FS}include${FS}linux ${TESTSRC}${FS}test.c
+ld -shared -z   execstack -o libtest-rwx.so test.o
+ld -shared -z noexecstack -o libtest-rw.so  test.o
+
+
+LD_LIBRARY_PATH=${THIS_DIR}
+echo   LD_LIBRARY_PATH = ${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH
+
+# This should not fail.
+echo Check testprogram. Expected to pass:
+echo ${TESTJAVA}${FS}bin${FS}java -cp ${THIS_DIR} Test test-rw
+${TESTJAVA}${FS}bin${FS}java -cp ${THIS_DIR} Test test-rw
+
+echo
+echo Test changing of stack protection:
+echo ${TESTJAVA}${FS}bin${FS}java -cp ${THIS_DIR} Test test-rw
+${TESTJAVA}${FS}bin${FS}java -cp ${THIS_DIR} Test test-rwx
+
+if [ "$?" == "0" ]
+then
+  echo
+  echo ${TESTJAVA}${FS}bin${FS}java -cp ${THIS_DIR} TestMT test-rwx
+  ${TESTJAVA}${FS}bin${FS}java -cp ${THIS_DIR} TestMT test-rwx
+fi
+
+exit $?

test/runtime/7107135/TestMT.java

+/*
+ * Copyright (c) 2002-2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 SAP AG.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+class TestMT {
+
+    static boolean loadLib(String libName) {
+        try {
+            System.loadLibrary(libName);
+            System.out.println("Loaded library "+ libName + ".");
+            return true;
+        } catch (SecurityException e) {
+            System.out.println("loadLibrary(\"" + libName + "\") throws: " + e + "\n");
+        } catch (UnsatisfiedLinkError e) {
+            System.out.println("loadLibrary(\"" + libName + "\") throws: " + e + "\n");
+        }
+        return false;
+    }
+
+    public static int counter        = 1;
+    static int Runner() {
+        counter = counter * -1;
+        int i = counter;
+        if (counter < 2) counter += Runner();
+        return i;
+    }
+
+    public static int run(String msg) {
+        try {
+            Runner();
+        } catch (StackOverflowError e) {
+            System.out.println(msg + " caught stack overflow error.");
+            return 0;
+        } catch (OutOfMemoryError e) {
+            return 0;
+        }
+        return 2;
+    }
+
+    public static void main(String argv[]) {
+        try {
+            for (int i = 0; i < 20; i++) {
+                Thread t = new DoStackOverflow("SpawnedThread " + i);
+                t.start();
+            }
+            run("Main thread");
+            loadLib("test-rwx");
+            run("Main thread");
+        } catch (Exception e) {
+            System.out.println(e);
+        }
+    }
+
+    static class DoStackOverflow extends Thread {
+        public DoStackOverflow(String name) {
+            super(name);
+        }
+        public void run() {
+            for (int i = 0; i < 10; ++i) {
+                TestMT.run(getName());
+                yield();
+            }
+        }
+    }
+}

test/runtime/7107135/test.c

+/*
+ * Copyright (c) 2002-2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011 SAP AG.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include "jni.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JNIEXPORT jint JNICALL Java_Test_someMethod(JNIEnv *env, jobject mainObject) {
+  return 3;
+}
+
+#ifdef __cplusplus
+}
+#endif