8149070: Enforce update ordering

Summary: Make sure that ISE is thrown when updateAAD is called after update.
Reviewed-by: mullan

src/share/classes/com/sun/crypto/provider/AESCipher.java

 /*
- * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -172,6 +172,11 @@ abstract class AESCipher extends CipherSpi {
      */
     private final int fixedKeySize; // in bytes, -1 if no restriction
 
+    /*
+     * needed to enforce ISE thrown when updateAAD is called after update for GCM mode.
+     */
+    private boolean updateCalled;
+
     /**
      * Creates an instance of AES cipher with default ECB mode and
      * PKCS5Padding.
@@ -304,6 +309,7 @@ abstract class AESCipher extends CipherSpi {
     protected void engineInit(int opmode, Key key, SecureRandom random)
         throws InvalidKeyException {
         checkKeySize(key, fixedKeySize);
+        updateCalled = false;
         core.init(opmode, key, random);
     }
 
@@ -336,6 +342,7 @@ abstract class AESCipher extends CipherSpi {
                               SecureRandom random)
         throws InvalidKeyException, InvalidAlgorithmParameterException {
         checkKeySize(key, fixedKeySize);
+        updateCalled = false;
         core.init(opmode, key, params, random);
     }
 
@@ -344,6 +351,7 @@ abstract class AESCipher extends CipherSpi {
                               SecureRandom random)
         throws InvalidKeyException, InvalidAlgorithmParameterException {
         checkKeySize(key, fixedKeySize);
+        updateCalled = false;
         core.init(opmode, key, params, random);
     }
 
@@ -368,6 +376,7 @@ abstract class AESCipher extends CipherSpi {
      */
     protected byte[] engineUpdate(byte[] input, int inputOffset,
                                   int inputLen) {
+        updateCalled = true;
         return core.update(input, inputOffset, inputLen);
     }
 
@@ -397,6 +406,7 @@ abstract class AESCipher extends CipherSpi {
     protected int engineUpdate(byte[] input, int inputOffset, int inputLen,
                                byte[] output, int outputOffset)
         throws ShortBufferException {
+        updateCalled = true;
         return core.update(input, inputOffset, inputLen, output,
                            outputOffset);
     }
@@ -433,7 +443,9 @@ abstract class AESCipher extends CipherSpi {
      */
     protected byte[] engineDoFinal(byte[] input, int inputOffset, int inputLen)
         throws IllegalBlockSizeException, BadPaddingException {
-        return core.doFinal(input, inputOffset, inputLen);
+        byte[] out = core.doFinal(input, inputOffset, inputLen);
+        updateCalled = false;
+        return out;
     }
 
     /**
@@ -476,8 +488,10 @@ abstract class AESCipher extends CipherSpi {
                                 byte[] output, int outputOffset)
         throws IllegalBlockSizeException, ShortBufferException,
                BadPaddingException {
-        return core.doFinal(input, inputOffset, inputLen, output,
-                            outputOffset);
+        int outLen = core.doFinal(input, inputOffset, inputLen, output,
+                                  outputOffset);
+        updateCalled = false;
+        return outLen;
     }
 
     /**
@@ -574,6 +588,9 @@ abstract class AESCipher extends CipherSpi {
      */
     @Override
     protected void engineUpdateAAD(byte[] src, int offset, int len) {
+        if (core.getMode() == CipherCore.GCM_MODE && updateCalled) {
+            throw new IllegalStateException("AAD must be supplied before encryption/decryption starts");
+        }
         core.updateAAD(src, offset, len);
     }
 
@@ -606,6 +623,9 @@ abstract class AESCipher extends CipherSpi {
      */
     @Override
     protected void engineUpdateAAD(ByteBuffer src) {
+        if (core.getMode() == CipherCore.GCM_MODE && updateCalled) {
+            throw new IllegalStateException("AAD must be supplied before encryption/decryption starts");
+        }
         if (src != null) {
             int aadLen = src.limit() - src.position();
             if (aadLen != 0) {

src/share/classes/com/sun/crypto/provider/CipherCore.java

 /*
- * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -124,7 +124,7 @@ final class CipherCore {
     private static final int PCBC_MODE = 4;
     private static final int CTR_MODE = 5;
     private static final int CTS_MODE = 6;
-    private static final int GCM_MODE = 7;
+    static final int GCM_MODE = 7;
 
     /*
      * variables used for performing the GCM (key+iv) uniqueness check.
@@ -196,7 +196,7 @@ final class CipherCore {
             cipher = new CounterMode(rawImpl);
             unitBytes = 1;
             padding = null;
-        }  else if (modeUpperCase.startsWith("GCM")) {
+        }  else if (modeUpperCase.equals("GCM")) {
             // can only be used for block ciphers w/ 128-bit block size
             if (blockSize != 16) {
                 throw new NoSuchAlgorithmException
@@ -223,6 +223,15 @@ final class CipherCore {
         }
     }
 
+    /**
+     * Returns the mode of this cipher.
+     *
+     * @return the parsed cipher mode
+     */
+    int getMode() {
+        return cipherMode;
+    }
+
     private static int getNumOfUnit(String mode, int offset, int blockSize)
         throws NoSuchAlgorithmException {
         int result = blockSize; // use blockSize as default value

src/share/classes/com/sun/crypto/provider/GaloisCounterMode.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -319,20 +319,22 @@ final class GaloisCounterMode extends FeedbackCipher {
 
     // Feed the AAD data to GHASH, pad if necessary
     void processAAD() {
-        if (aadBuffer != null && aadBuffer.size() > 0) {
-            byte[] aad = aadBuffer.toByteArray();
-            sizeOfAAD = aad.length;
-            aadBuffer = null;
-
-            int lastLen = aad.length % AES_BLOCK_SIZE;
-            if (lastLen != 0) {
-                ghashAllToS.update(aad, 0, aad.length - lastLen);
-                byte[] padded = expandToOneBlock(aad, aad.length - lastLen,
-                                                 lastLen);
-                ghashAllToS.update(padded);
-            } else {
-                ghashAllToS.update(aad);
+        if (aadBuffer != null) {
+            if (aadBuffer.size() > 0) {
+                byte[] aad = aadBuffer.toByteArray();
+                sizeOfAAD = aad.length;
+
+                int lastLen = aad.length % AES_BLOCK_SIZE;
+                if (lastLen != 0) {
+                    ghashAllToS.update(aad, 0, aad.length - lastLen);
+                    byte[] padded = expandToOneBlock(aad, aad.length - lastLen,
+                                                     lastLen);
+                    ghashAllToS.update(padded);
+                } else {
+                    ghashAllToS.update(aad);
+                }
             }
+            aadBuffer = null;
         }
     }