8017498: JVM crashes when native code calls sigaction(sig) where sig>=0x20

Summary: Added (sig < MAXSIGNUM) check in jsig.c
Reviewed-by: dholmes, acorn

hotspot/src/os/linux/vm/jsig.c

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -107,7 +107,7 @@ static sa_handler_t set_signal(int sig, sa_handler_t disp, bool is_sigset) {
 
   signal_lock();
 
-  sigused = (MASK(sig) & jvmsigs) != 0;
+  sigused = (sig < MAXSIGNUM) && ((MASK(sig) & jvmsigs) != 0);
   if (jvm_signal_installed && sigused) {
     /* jvm has installed its signal handler for this signal. */
     /* Save the handler. Don't really install it. */
@@ -116,7 +116,7 @@ static sa_handler_t set_signal(int sig, sa_handler_t disp, bool is_sigset) {
 
     signal_unlock();
     return oldhandler;
-  } else if (jvm_signal_installing) {
+  } else if (sig < MAXSIGNUM && jvm_signal_installing) {
     /* jvm is installing its signal handlers. Install the new
      * handlers and save the old ones. jvm uses sigaction().
      * Leave the piece here just in case. */
@@ -165,7 +165,7 @@ int sigaction(int sig, const struct sigaction *act, struct sigaction *oact) {
 
   signal_lock();
 
-  sigused = (MASK(sig) & jvmsigs) != 0;
+  sigused = (sig < MAXSIGNUM) && ((MASK(sig) & jvmsigs) != 0);
   if (jvm_signal_installed && sigused) {
     /* jvm has installed its signal handler for this signal. */
     /* Save the handler. Don't really install it. */
@@ -178,7 +178,7 @@ int sigaction(int sig, const struct sigaction *act, struct sigaction *oact) {
 
     signal_unlock();
     return 0;
-  } else if (jvm_signal_installing) {
+  } else if (sig < MAXSIGNUM && jvm_signal_installing) {
     /* jvm is installing its signal handlers. Install the new
      * handlers and save the old ones. */
     res = call_os_sigaction(sig, act, &oldAct);

hotspot/test/runtime/jsig/Test8017498.sh

+#!/bin/sh
+
+#
+#  Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+#  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+#  This code is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License version 2 only, as
+#  published by the Free Software Foundation.
+#
+#  This code is distributed in the hope that it will be useful, but WITHOUT
+#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+#  version 2 for more details (a copy is included in the LICENSE file that
+#  accompanied this code).
+#
+#  You should have received a copy of the GNU General Public License version
+#  2 along with this work; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#  Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+#  or visit www.oracle.com if you need additional information or have any
+#  questions.
+#
+
+##
+## @test Test8017498.sh
+## @bug 8017498
+## @summary sigaction(sig) results in process hang/timed-out if sig is much greater than SIGRTMAX
+## @run shell Test8017498.sh
+##
+
+if [ "${TESTSRC}" = "" ]
+then
+  TESTSRC=${PWD}
+  echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+## Adding common setup Variables for running shell tests.
+. ${TESTSRC}/../../test_env.sh
+
+# set platform-dependent variables
+OS=`uname -s`
+case "$OS" in
+  Linux)
+    echo "Testing on Linux"
+    if [ "$VM_BITS" = "64" ]
+    then
+        LD_PRELOAD=${TESTJAVA}${FS}jre${FS}lib${FS}amd64${FS}libjsig.so
+    else
+        LD_PRELOAD=${TESTJAVA}${FS}jre${FS}lib${FS}i386${FS}libjsig.so
+    fi
+    echo LD_PRELOAD = ${LD_PRELOAD}
+    export LD_PRELOAD=${LD_PRELOAD}
+    ;;
+  *)
+    NULL=NUL
+    PS=";"
+    FS="\\"
+    echo "Test passed; only valid for Linux"
+    exit 0;
+    ;;
+esac
+
+THIS_DIR=.
+
+cp ${TESTSRC}${FS}*.java ${THIS_DIR}
+${TESTJAVA}${FS}bin${FS}javac *.java
+
+gcc -fPIC -shared -o ${TESTSRC}${FS}libTestJNI.so -I${TESTJAVA}${FS}include -I${TESTJAVA}${FS}include${FS}linux ${TESTSRC}${FS}TestJNI.c
+
+# run the java test in the background
+echo ${TESTJAVA}${FS}bin${FS}java -Djava.library.path=${TESTSRC}${FS} -server TestJNI 100 > test.out 2>&1 &
+${TESTJAVA}${FS}bin${FS}java -Djava.library.path=${TESTSRC}${FS} -server TestJNI 100 > test.out 2>&1 &
+
+# obtain the process id
+C_PID=$!
+
+# sleep for 1s
+sleep 1
+
+# reset LD_PRELOAD
+unset LD_PRELOAD
+
+# check the output file (test.out)
+grep "old handler" test.out > ${NULL}
+if [ $? = 0 ]
+then
+    echo "Test Passed"
+    exit 0
+else
+    kill -9 ${C_PID}
+    echo "Test Failed"
+    exit 1
+fi

hotspot/test/runtime/jsig/TestJNI.c

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdio.h>
+#include <jni.h>
+#define __USE_GNU
+#include <signal.h>
+#include <sys/ucontext.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void sig_handler(int sig, siginfo_t *info, ucontext_t *context) {
+    int thrNum;
+
+    printf( " HANDLER (1) " );
+    // Move forward RIP to skip failing instruction
+    context->uc_mcontext.gregs[REG_RIP] += 6;
+}
+
+JNIEXPORT void JNICALL Java_TestJNI_doSomething(JNIEnv *env, jclass klass, jint val) {
+    struct sigaction act;
+    struct sigaction oact;
+    pthread_attr_t attr;
+    stack_t stack;
+
+    act.sa_flags = SA_ONSTACK|SA_RESTART|SA_SIGINFO;
+    sigfillset(&act.sa_mask);
+    act.sa_handler = SIG_DFL;
+    act.sa_sigaction = (void (*)())sig_handler;
+    sigaction(0x20+val, &act, &oact);
+
+    printf( " doSomething(%d) " , val);
+    printf( " old handler = %p " , oact.sa_handler);
+}
+
+#ifdef __cplusplus
+}
+#endif
+

hotspot/test/runtime/jsig/TestJNI.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+public class TestJNI {
+    static {
+        System.loadLibrary("TestJNI");
+    }
+    public static native void doSomething(int val);
+    public static void main(String[] args) {
+        int intArg = 43;
+        if (args.length > 0) {
+            try {
+                intArg = Integer.parseInt(args[0]);
+            } catch (NumberFormatException e) {
+                System.err.println("arg " + args[0] + " must be an integer");
+                System.exit(1);
+            }
+        }
+        TestJNI.doSomething(intArg);
+    }
+}
+