7020373: JSR rewriting can overflow memory address size variables

Summary: Abort if incoming classfile's parameters would cause overflows
Reviewed-by: coleenp, dcubed, never

src/share/vm/oops/generateOopMap.cpp

@@ -963,10 +963,21 @@ void GenerateOopMap::init_basic_blocks() {
   // initialize the CellTypeState-related information.
   init_state();
 
-  // We allocate space for all state-vectors for all basicblocks in one huge chuck.
-  // Then in the next part of the code, we set a pointer in each _basic_block that
-  // points to each piece.
-  CellTypeState *basicBlockState = NEW_RESOURCE_ARRAY(CellTypeState, bbNo * _state_len);
+  // We allocate space for all state-vectors for all basicblocks in one huge
+  // chunk.  Then in the next part of the code, we set a pointer in each
+  // _basic_block that points to each piece.
+
+  // The product of bbNo and _state_len can get large if there are lots of
+  // basic blocks and stack/locals/monitors.  Need to check to make sure
+  // we don't overflow the capacity of a pointer.
+  if ((unsigned)bbNo > UINTPTR_MAX / sizeof(CellTypeState) / _state_len) {
+    report_error("The amount of memory required to analyze this method "
+                 "exceeds addressable range");
+    return;
+  }
+
+  CellTypeState *basicBlockState =
+      NEW_RESOURCE_ARRAY(CellTypeState, bbNo * _state_len);
   memset(basicBlockState, 0, bbNo * _state_len * sizeof(CellTypeState));
 
   // Make a pass over the basicblocks and assign their state vectors.

test/runtime/7020373/Test7020373.sh

+#!/bin/sh
+
+##
+## @test
+## @bug 7020373
+## @key cte_test
+## @summary JSR rewriting can overflow memory address size variables
+## @run shell Test7020373.sh
+##
+
+if [ "${TESTSRC}" = "" ]
+then TESTSRC=.
+fi
+
+if [ "${TESTJAVA}" = "" ]
+then
+  PARENT=`dirname \`which java\``
+  TESTJAVA=`dirname ${PARENT}`
+  echo "TESTJAVA not set, selecting " ${TESTJAVA}
+  echo "If this is incorrect, try setting the variable manually."
+fi
+
+if [ "${TESTCLASSES}" = "" ]
+then
+  echo "TESTCLASSES not set.  Test cannot execute.  Failed."
+  exit 1
+fi
+
+BIT_FLAG=""
+
+# set platform-dependent variables
+OS=`uname -s`
+case "$OS" in
+  SunOS | Linux )
+    NULL=/dev/null
+    PS=":"
+    FS="/"
+    ## for solaris, linux it's HOME
+    FILE_LOCATION=$HOME
+    if [ -f ${FILE_LOCATION}${FS}JDK64BIT -a ${OS} = "SunOS" ]
+    then
+        BIT_FLAG=`cat ${FILE_LOCATION}${FS}JDK64BIT | grep -v '^#'`
+    fi
+    ;;
+  Windows_* )
+    NULL=NUL
+    PS=";"
+    FS="\\"
+    ;;
+  * )
+    echo "Unrecognized system!"
+    exit 1;
+    ;;
+esac
+
+JEMMYPATH=${CPAPPEND}
+CLASSPATH=.${PS}${TESTCLASSES}${PS}${JEMMYPATH} ; export CLASSPATH
+
+THIS_DIR=`pwd`
+
+${TESTJAVA}${FS}bin${FS}java ${BIT_FLAG} -version
+
+${TESTJAVA}${FS}bin${FS}jar xvf ${TESTSRC}${FS}testcase.jar
+
+${TESTJAVA}${FS}bin${FS}java ${BIT_FLAG} OOMCrashClass4000_1 > test.out 2>&1
+
+cat test.out
+
+egrep "SIGSEGV|An unexpected error has been detected" test.out
+
+if [ $? = 0 ]
+then
+    echo "Test Failed"
+    exit 1
+else
+    grep "java.lang.LinkageError" test.out
+    if [ $? = 0 ]
+    then
+        echo "Test Passed"
+        exit 0
+    else
+        echo "Test Failed"
+        exit 1
+    fi
+fi