8208585: Make crypto code more robust

Reviewed-by: ascarpino, mschoene

src/share/classes/com/sun/crypto/provider/RSACipher.java

 /*
- * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -329,7 +329,7 @@ public final class RSACipher extends CipherSpi {
         if ((inLen == 0) || (in == null)) {
             return;
         }
-        if (bufOfs + inLen > buffer.length) {
+        if (inLen > (buffer.length - bufOfs)) {
             bufOfs = buffer.length + 1;
             return;
         }

src/share/classes/sun/security/pkcs11/P11Signature.java

 /*
- * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -472,6 +472,10 @@ final class P11Signature extends SignatureSpi {
         if (len == 0) {
             return;
         }
+        // check for overflow
+        if (len + bytesProcessed < 0) {
+            throw new ProviderException("Processed bytes limits exceeded.");
+        }
         switch (type) {
         case T_UPDATE:
             try {

src/share/classes/sun/security/provider/DSA.java

 /*
- * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -491,7 +491,7 @@ abstract class DSA extends SignatureSpi {
                 }
             }
             protected void engineUpdate(byte[] input, int offset, int len) {
-                if (ofs + len > digestBuffer.length) {
+                if (len > (digestBuffer.length - ofs)) {
                     ofs = Integer.MAX_VALUE;
                 } else {
                     System.arraycopy(input, offset, digestBuffer, ofs, len);
@@ -500,7 +500,7 @@ abstract class DSA extends SignatureSpi {
             }
             protected final void engineUpdate(ByteBuffer input) {
                 int inputLen = input.remaining();
-                if (ofs + inputLen > digestBuffer.length) {
+                if (inputLen > (digestBuffer.length - ofs)) {
                     ofs = Integer.MAX_VALUE;
                 } else {
                     input.get(digestBuffer, ofs, inputLen);

src/windows/classes/sun/security/mscapi/RSASignature.java

 /*
- * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -132,7 +132,7 @@ abstract class RSASignature extends java.security.SignatureSpi
         @Override
         protected void engineUpdate(byte[] b, int off, int len)
                 throws SignatureException {
-            if (offset + len > precomputedDigest.length) {
+            if (len > (precomputedDigest.length - offset)) {
                 offset = RAW_RSA_MAX + 1;
                 return;
             }
@@ -147,7 +147,7 @@ abstract class RSASignature extends java.security.SignatureSpi
             if (len <= 0) {
                 return;
             }
-            if (offset + len > precomputedDigest.length) {
+            if (len > (precomputedDigest.length - offset)) {
                 offset = RAW_RSA_MAX + 1;
                 return;
             }