8023956: Provide a work-around to broken Linux 32 bit "Exec Shield" using CS...

8023956: Provide a work-around to broken Linux 32 bit "Exec Shield" using CS for NX emulation (crashing with SI_KERNEL)
Summary: Execute some code at a high virtual address value, and keep mapped
Reviewed-by: coleenp, zgu

src/os/linux/vm/os_linux.cpp

@@ -4839,6 +4839,10 @@ jint os::init_2(void)
 
   Linux::capture_initial_stack(JavaThread::stack_size_at_create());
 
+#if defined(IA32)
+  workaround_expand_exec_shield_cs_limit();
+#endif
+
   Linux::libpthread_init();
   if (PrintMiscellaneous && (Verbose || WizardMode)) {
      tty->print_cr("[HotSpot is running with %s, %s(%s)]\n",

src/os_cpu/linux_x86/vm/os_linux_x86.cpp

@@ -876,3 +876,46 @@ void os::verify_stack_alignment() {
 #endif
 }
 #endif
+
+
+/*
+ * IA32 only: execute code at a high address in case buggy NX emulation is present. I.e. avoid CS limit
+ * updates (JDK-8023956).
+ */
+void os::workaround_expand_exec_shield_cs_limit() {
+#if defined(IA32)
+  size_t page_size = os::vm_page_size();
+  /*
+   * Take the highest VA the OS will give us and exec
+   *
+   * Although using -(pagesz) as mmap hint works on newer kernel as you would
+   * think, older variants affected by this work-around don't (search forward only).
+   *
+   * On the affected distributions, we understand the memory layout to be:
+   *
+   *   TASK_LIMIT= 3G, main stack base close to TASK_LIMT.
+   *
+   * A few pages south main stack will do it.
+   *
+   * If we are embedded in an app other than launcher (initial != main stack),
+   * we don't have much control or understanding of the address space, just let it slide.
+   */
+  char* hint = (char*) (Linux::initial_thread_stack_bottom() -
+                        ((StackYellowPages + StackRedPages + 1) * page_size));
+  char* codebuf = os::reserve_memory(page_size, hint);
+  if ( (codebuf == NULL) || (!os::commit_memory(codebuf, page_size, true)) ) {
+    return; // No matter, we tried, best effort.
+  }
+  if (PrintMiscellaneous && (Verbose || WizardMode)) {
+     tty->print_cr("[CS limit NX emulation work-around, exec code at: %p]", codebuf);
+  }
+
+  // Some code to exec: the 'ret' instruction
+  codebuf[0] = 0xC3;
+
+  // Call the code in the codebuf
+  __asm__ volatile("call *%0" : : "r"(codebuf));
+
+  // keep the page mapped so CS limit isn't reduced.
+#endif
+}

src/os_cpu/linux_x86/vm/os_linux_x86.hpp

@@ -36,4 +36,17 @@
   // Note: Currently only used in 64 bit Windows implementations
   static bool register_code_area(char *low, char *high) { return true; }
 
+  /*
+   * Work-around for broken NX emulation using CS limit, Red Hat patch "Exec-Shield"
+   * (IA32 only).
+   *
+   * Map and execute at a high VA to prevent CS lazy updates race with SMP MM
+   * invalidation.Further code generation by the JVM will no longer cause CS limit
+   * updates.
+   *
+   * Affects IA32: RHEL 5 & 6, Ubuntu 10.04 (LTS), 10.10, 11.04, 11.10, 12.04.
+   * @see JDK-8023956
+   */
+  static void workaround_expand_exec_shield_cs_limit();
+
 #endif // OS_CPU_LINUX_X86_VM_OS_LINUX_X86_HPP