8187045: [linux] Not all libraries in the VM are linked with -z,noexecstack

Reviewed-by: dholmes, erikj

agent/src/os/linux/Makefile

 #
-# Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -76,6 +76,9 @@ ifneq ($(_HAS_HASH_STYLE_GNU),)
 endif
 LFLAGS_LIBSA += $(LDFLAGS_HASH_STYLE)
 
+LDFLAGS_NO_EXEC_STACK="-Wl,-z,noexecstack"
+LFLAGS_LIBSA += $(LDFLAGS_NO_EXEC_STACK)
+
 $(LIBSA): $(ARCH) $(OBJS) mapfile
         $(GCC) -shared $(LFLAGS_LIBSA) -o $(LIBSA) $(OBJS) $(LIBS)
 

make/linux/makefiles/gcc.make

 #
-# Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -303,6 +303,8 @@ endif
 
 LFLAGS += $(LDFLAGS_HASH_STYLE)
 
+LDFLAGS_NO_EXEC_STACK="-Wl,-z,noexecstack"
+
 # Use $(MAPFLAG:FILENAME=real_file_name) to specify a map file.
 MAPFLAG = -Xlinker --version-script=FILENAME
 

make/linux/makefiles/jsig.make

 #
-# Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -44,7 +44,7 @@ LIBJSIG_MAPFILE = $(MAKEFILES_DIR)/mapfile-vers-jsig
 # cause problems with interposing. See CR: 6466665
 # LFLAGS_JSIG += $(MAPFLAG:FILENAME=$(LIBJSIG_MAPFILE))
 
-LFLAGS_JSIG += -D_GNU_SOURCE -D_REENTRANT $(LDFLAGS_HASH_STYLE)
+LFLAGS_JSIG += -D_GNU_SOURCE -D_REENTRANT $(LDFLAGS_HASH_STYLE) $(LDFLAGS_NO_EXEC_STACK)
 
 # DEBUG_BINARIES overrides everything, use full -g debug information
 ifeq ($(DEBUG_BINARIES), true)

src/share/vm/prims/whitebox.cpp

 /*
- * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -158,6 +158,9 @@ WB_ENTRY(void, WB_AddToSystemClassLoaderSearch(JNIEnv* env, jobject o, jstring s
 }
 WB_END
 
+#ifdef LINUX
+#include "utilities/elfFile.hpp"
+#endif
 
 WB_ENTRY(jlong, WB_GetCompressedOopsMaxHeapSize(JNIEnv* env, jobject o)) {
   return (jlong)Arguments::max_heap_for_compressed_oops();
@@ -1010,6 +1013,21 @@ void WhiteBox::register_methods(JNIEnv* env, jclass wbclass, JavaThread* thread,
   }
 }
 
+// Checks that the library libfile has the noexecstack bit set.
+WB_ENTRY(jboolean, WB_CheckLibSpecifiesNoexecstack(JNIEnv* env, jobject o, jstring libfile))
+  jboolean ret = false;
+#ifdef LINUX
+  // Can't be in VM when we call JNI.
+  ThreadToNativeFromVM ttnfv(thread);
+  const char* lf = env->GetStringUTFChars(libfile, NULL);
+  CHECK_JNI_EXCEPTION_(env, 0);
+  ElfFile ef(lf);
+  ret = (jboolean) ef.specifies_noexecstack();
+  env->ReleaseStringUTFChars(libfile, lf);
+#endif
+  return ret;
+WB_END
+
 #define CC (char*)
 
 static JNINativeMethod methods[] = {
@@ -1121,6 +1139,8 @@ static JNINativeMethod methods[] = {
                                                       (void*)&WB_GetNMethod         },
   {CC"isMonitorInflated",  CC"(Ljava/lang/Object;)Z", (void*)&WB_IsMonitorInflated  },
   {CC"forceSafepoint",     CC"()V",                   (void*)&WB_ForceSafepoint     },
+  {CC"checkLibSpecifiesNoexecstack", CC"(Ljava/lang/String;)Z",
+                                                      (void*)&WB_CheckLibSpecifiesNoexecstack},
 };
 
 #undef CC

test/runtime/execstack/TestCheckJDK.java

+/*
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test Testexecstack.java
+ * @summary Searches for all libraries in test VM and checks that they
+ *          have the noexecstack bit set.
+ * @requires (os.family == "linux")
+ * @library /testlibrary /testlibrary/whitebox
+ * @build sun.hotspot.WhiteBox
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ *                                sun.hotspot.WhiteBox$WhiteBoxPermission
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
+ *                   TestCheckJDK
+ */
+
+import com.oracle.java.testlibrary.Asserts;
+import sun.hotspot.WhiteBox;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class TestCheckJDK {
+    static boolean testPassed = true;
+    private static final WhiteBox WB = WhiteBox.getWhiteBox();
+
+    static void checkExecStack(Path file) {
+        String filename = file.toString();
+        if (filename.endsWith(".so")) {
+            if (!WB.checkLibSpecifiesNoexecstack(filename)) {
+                System.out.println("Library does not have the noexecstack bit set: " + filename);
+                testPassed = false;
+            }
+        }
+    }
+
+    public static void main(String[] args) throws Throwable {
+        String vmInstallDir = System.getProperty("java.home");
+
+        Files.walk(Paths.get(vmInstallDir)).filter(Files::isRegularFile).forEach(TestCheckJDK::checkExecStack);
+
+        Asserts.assertTrue(testPassed,
+            "The tested VM contains libs that don't have the noexecstack " +
+            "bit set. They must be linked with -z,noexecstack.");
+    }
+}

test/testlibrary/whitebox/sun/hotspot/WhiteBox.java

 /*
- * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -235,4 +235,7 @@ public class WhiteBox {
 
   // Class Data Sharing
   public native boolean isSharedClass(Class<?> c);
+
+  // Returns true on linux if library has the noexecstack flag set.
+  public native boolean checkLibSpecifiesNoexecstack(String libfilename);
 }