8187045: [linux] Not all libraries in the VM are linked with -z,noexecstack

Reviewed-by: ihse, dholmes

make/autoconf/flags.m4

@@ -1311,6 +1311,7 @@ AC_DEFUN([FLAGS_SETUP_COMPILER_FLAGS_FOR_JDK_HELPER],
   $2LDFLAGS_JDKLIB="${$2LDFLAGS_JDK}"
 
   $2LDFLAGS_JDKLIB="${$2LDFLAGS_JDKLIB} ${SHARED_LIBRARY_FLAGS}"
+  $2LDFLAGS_JDKLIB="${$2LDFLAGS_JDKLIB} ${LDFLAGS_NO_EXEC_STACK}"
   if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
     $2JAVA_BASE_LDFLAGS="${$2JAVA_BASE_LDFLAGS} \
         -libpath:${OUTPUTDIR}/support/modules_libs/java.base"
@@ -1388,6 +1389,7 @@ $2LDFLAGS_JDKLIB="${$2LDFLAGS_JDKLIB} ${$2JAVA_BASE_LDFLAGS}"
   AC_SUBST($2JDKEXE_LIBS)
   AC_SUBST($2LDFLAGS_CXX_JDK)
   AC_SUBST($2LDFLAGS_HASH_STYLE)
+  AC_SUBST($2LDFLAGS_NO_EXEC_STACK)
 
   AC_SUBST($2JVM_CFLAGS)
   AC_SUBST($2JVM_LDFLAGS)

make/autoconf/generated-configure.sh

@@ -723,6 +723,7 @@ OPENJDK_BUILD_JVM_LIBS
 OPENJDK_BUILD_JVM_ASFLAGS
 OPENJDK_BUILD_JVM_LDFLAGS
 OPENJDK_BUILD_JVM_CFLAGS
+OPENJDK_BUILD_LDFLAGS_NO_EXEC_STACK
 OPENJDK_BUILD_LDFLAGS_HASH_STYLE
 OPENJDK_BUILD_LDFLAGS_CXX_JDK
 OPENJDK_BUILD_JDKEXE_LIBS
@@ -738,6 +739,7 @@ JVM_LIBS
 JVM_ASFLAGS
 JVM_LDFLAGS
 JVM_CFLAGS
+LDFLAGS_NO_EXEC_STACK
 LDFLAGS_HASH_STYLE
 LDFLAGS_CXX_JDK
 JDKEXE_LIBS
@@ -5115,7 +5117,7 @@ VS_SDK_PLATFORM_NAME_2013=
 #CUSTOM_AUTOCONF_INCLUDE
 
 # Do not change or remove the following line, it is needed for consistency checks:
-DATE_WHEN_GENERATED=1506333008
+DATE_WHEN_GENERATED=1506397140
 
 ###############################################################################
 #
@@ -52024,6 +52026,7 @@ fi
   LDFLAGS_JDKLIB="${LDFLAGS_JDK}"
 
   LDFLAGS_JDKLIB="${LDFLAGS_JDKLIB} ${SHARED_LIBRARY_FLAGS}"
+  LDFLAGS_JDKLIB="${LDFLAGS_JDKLIB} ${LDFLAGS_NO_EXEC_STACK}"
   if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
     JAVA_BASE_LDFLAGS="${JAVA_BASE_LDFLAGS} \
         -libpath:${OUTPUTDIR}/support/modules_libs/java.base"
@@ -52109,6 +52112,7 @@ LDFLAGS_JDKLIB="${LDFLAGS_JDKLIB} ${JAVA_BASE_LDFLAGS}"
 
 
 
+
   # Special extras...
   if test "x$TOOLCHAIN_TYPE" = xsolstudio; then
     if test "x$OPENJDK_BUILD_CPU_ARCH" = "xsparc"; then
@@ -52903,6 +52907,7 @@ fi
   OPENJDK_BUILD_LDFLAGS_JDKLIB="${OPENJDK_BUILD_LDFLAGS_JDK}"
 
   OPENJDK_BUILD_LDFLAGS_JDKLIB="${OPENJDK_BUILD_LDFLAGS_JDKLIB} ${SHARED_LIBRARY_FLAGS}"
+  OPENJDK_BUILD_LDFLAGS_JDKLIB="${OPENJDK_BUILD_LDFLAGS_JDKLIB} ${LDFLAGS_NO_EXEC_STACK}"
   if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
     OPENJDK_BUILD_JAVA_BASE_LDFLAGS="${OPENJDK_BUILD_JAVA_BASE_LDFLAGS} \
         -libpath:${OUTPUTDIR}/support/modules_libs/java.base"
@@ -52988,6 +52993,7 @@ OPENJDK_BUILD_LDFLAGS_JDKLIB="${OPENJDK_BUILD_LDFLAGS_JDKLIB} ${OPENJDK_BUILD_JA
 
 
 
+
   # Tests are only ever compiled for TARGET
   # Flags for compiling test libraries
   CFLAGS_TESTLIB="$COMMON_CCXXFLAGS_JDK $CFLAGS_JDK $PICFLAG $CFLAGS_JDKLIB_EXTRA"

make/autoconf/spec.gmk.in

@@ -387,6 +387,7 @@ CFLAGS_JDKEXE:=@CFLAGS_JDKEXE@
 CXXFLAGS_JDKEXE:=@CXXFLAGS_JDKEXE@
 
 LDFLAGS_HASH_STYLE := @LDFLAGS_HASH_STYLE@
+LDFLAGS_NO_EXEC_STACK := @LDFLAGS_NO_EXEC_STACK@
 
 JVM_CFLAGS := @JVM_CFLAGS@
 JVM_CFLAGS_SYMBOLS := @JVM_CFLAGS_SYMBOLS@

make/hotspot/lib/CompileLibjsig.gmk

 #
-# Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2017, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -36,7 +36,7 @@ ifneq ($(OPENJDK_TARGET_OS), windows)
   ifeq ($(STATIC_BUILD), false)
     ifeq ($(OPENJDK_TARGET_OS), linux)
       LIBJSIG_CFLAGS := -fPIC -D_GNU_SOURCE -D_REENTRANT $(EXTRA_CFLAGS)
-      LIBJSIG_LDFLAGS := $(LDFLAGS_HASH_STYLE) $(EXTRA_CFLAGS)
+      LIBJSIG_LDFLAGS := $(LDFLAGS_HASH_STYLE) ${LDFLAGS_NO_EXEC_STACK} $(EXTRA_CFLAGS)
       LIBJSIG_LIBS := $(LIBDL)
 
       # NOTE: The old build compiled this library without -soname.

src/hotspot/share/prims/whitebox.cpp

@@ -73,6 +73,9 @@
 #include "utilities/nativeCallStack.hpp"
 #endif // INCLUDE_NMT
 
+#ifdef LINUX
+#include "utilities/elfFile.hpp"
+#endif
 
 #define SIZE_T_MAX_VALUE ((size_t) -1)
 
@@ -1823,6 +1826,20 @@ WB_ENTRY(void, WB_RemoveCompilerDirective(JNIEnv* env, jobject o, jint count))
   DirectivesStack::pop(count);
 WB_END
 
+// Checks that the library libfile has the noexecstack bit set.
+WB_ENTRY(jboolean, WB_CheckLibSpecifiesNoexecstack(JNIEnv* env, jobject o, jstring libfile))
+  jboolean ret = false;
+#ifdef LINUX
+  // Can't be in VM when we call JNI.
+  ThreadToNativeFromVM ttnfv(thread);
+  const char* lf = env->GetStringUTFChars(libfile, NULL);
+  CHECK_JNI_EXCEPTION_(env, 0);
+  ret = (jboolean) ElfFile::specifies_noexecstack(lf);
+  env->ReleaseStringUTFChars(libfile, lf);
+#endif
+  return ret;
+WB_END
+
 #define CC (char*)
 
 static JNINativeMethod methods[] = {
@@ -2027,6 +2044,8 @@ static JNINativeMethod methods[] = {
                                                       (void*)&WB_GetConcurrentGCPhases},
   {CC"requestConcurrentGCPhase0", CC"(Ljava/lang/String;)Z",
                                                       (void*)&WB_RequestConcurrentGCPhase},
+  {CC"checkLibSpecifiesNoexecstack", CC"(Ljava/lang/String;)Z",
+                                                      (void*)&WB_CheckLibSpecifiesNoexecstack},
 };
 
 #undef CC

test/hotspot/jtreg/runtime/execstack/TestCheckJDK.java

+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test Testexecstack.java
+ * @summary Searches for all libraries in test VM and checks that they
+ *          have the noexecstack bit set.
+ * @requires (os.family == "linux")
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * @build sun.hotspot.WhiteBox
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                                sun.hotspot.WhiteBox$WhiteBoxPermission
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
+ *                   TestCheckJDK
+ */
+
+import jdk.test.lib.Asserts;
+import sun.hotspot.WhiteBox;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class TestCheckJDK {
+    static boolean testPassed = true;
+    private static final WhiteBox WB = WhiteBox.getWhiteBox();
+
+    static void checkExecStack(Path file) {
+        String filename = file.toString();
+        if (filename.endsWith(".so")) {
+            if (!WB.checkLibSpecifiesNoexecstack(filename)) {
+                System.out.println("Library does not have the noexecstack bit set: " + filename);
+                testPassed = false;
+            }
+        }
+    }
+
+    public static void main(String[] args) throws Throwable {
+        String vmInstallDir = System.getProperty("java.home");
+
+        Files.walk(Paths.get(vmInstallDir)).filter(Files::isRegularFile).forEach(TestCheckJDK::checkExecStack);
+
+        Asserts.assertTrue(testPassed,
+            "The tested VM contains libs that don't have the noexecstack " +
+            "bit set. They must be linked with -z,noexecstack.");
+    }
+}

test/lib/sun/hotspot/WhiteBox.java

@@ -529,4 +529,7 @@ public class WhiteBox {
   // Compiler Directive
   public native int addCompilerDirective(String compDirect);
   public native void removeCompilerDirective(int count);
+
+  // Returns true on linux if library has the noexecstack flag set.
+  public native boolean checkLibSpecifiesNoexecstack(String libfilename);
 }