8152479: Coded byte streams

Reviewed-by: hseigel

src/share/vm/interpreter/bytecodeStream.cpp

 /*
- * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,12 +31,12 @@ Bytecodes::Code RawBytecodeStream::raw_next_special(Bytecodes::Code code) {
   // set next bytecode position
   address bcp = RawBytecodeStream::bcp();
   address end = method()->code_base() + end_bci();
-  int l = Bytecodes::raw_special_length_at(bcp, end);
-  if (l <= 0 || (_bci + l) > _end_bci) {
+  int len = Bytecodes::raw_special_length_at(bcp, end);
+  // Very large tableswitch or lookupswitch size can cause _next_bci to overflow.
+  if (len <= 0 || (_bci > _end_bci - len) || (_bci - len >= _next_bci)) {
     code = Bytecodes::_illegal;
   } else {
-    _next_bci += l;
-    assert(_bci < _next_bci, "length must be > 0");
+    _next_bci += len;
     // set attributes
     _is_wide = false;
     // check for special (uncommon) cases

src/share/vm/interpreter/bytecodeStream.hpp

 /*
- * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -149,12 +149,15 @@ class RawBytecodeStream: public BaseBytecodeStream {
     code        = Bytecodes::code_or_bp_at(bcp);
 
     // set next bytecode position
-    int l = Bytecodes::length_for(code);
-    if (l > 0 && (_bci + l) <= _end_bci) {
+    int len = Bytecodes::length_for(code);
+    if (len > 0 && (_bci <= _end_bci - len)) {
       assert(code != Bytecodes::_wide && code != Bytecodes::_tableswitch
              && code != Bytecodes::_lookupswitch, "can't be special bytecode");
       _is_wide = false;
-      _next_bci += l;
+      _next_bci += len;
+      if (_next_bci <= _bci) { // Check for integer overflow
+        code = Bytecodes::_illegal;
+      }
       _raw_code = code;
       return code;
     } else {
@@ -203,19 +206,23 @@ class BytecodeStream: public BaseBytecodeStream {
       // note that we cannot advance before having the
       // tty bytecode otherwise the stepping is wrong!
       // (carefull: length_for(...) must be used first!)
-      int l = Bytecodes::length_for(code);
-      if (l == 0) l = Bytecodes::length_at(_method(), bcp);
-      _next_bci  += l;
-      assert(_bci < _next_bci, "length must be > 0");
-      // set attributes
-      _is_wide      = false;
-      // check for special (uncommon) cases
-      if (code == Bytecodes::_wide) {
-        raw_code = (Bytecodes::Code)bcp[1];
-        code = raw_code;  // wide BCs are always Java-normal
-        _is_wide = true;
+      int len = Bytecodes::length_for(code);
+      if (len == 0) len = Bytecodes::length_at(_method(), bcp);
+      if (len <= 0 || (_bci > _end_bci - len) || (_bci - len >= _next_bci)) {
+        raw_code = code = Bytecodes::_illegal;
+      } else {
+        _next_bci  += len;
+        assert(_bci < _next_bci, "length must be > 0");
+        // set attributes
+        _is_wide      = false;
+        // check for special (uncommon) cases
+        if (code == Bytecodes::_wide) {
+          raw_code = (Bytecodes::Code)bcp[1];
+          code = raw_code;  // wide BCs are always Java-normal
+          _is_wide = true;
+        }
+        assert(Bytecodes::is_java_code(code), "sanity check");
       }
-      assert(Bytecodes::is_java_code(code), "sanity check");
     }
     _raw_code = raw_code;
     _code = code;