8144445: Maximum size checking in Marlin ArrayCache utility methods is not optimal

Reviewed-by: prr, flar

src/share/classes/sun/java2d/marlin/ArrayCache.java

@@ -166,18 +166,31 @@ public final class ArrayCache implements MarlinConst {
      * @return new array size
      */
     public static int getNewSize(final int curSize, final int needSize) {
+        // check if needSize is negative or integer overflow:
+        if (needSize < 0) {
+            // hard overflow failure - we can't even accommodate
+            // new items without overflowing
+            throw new ArrayIndexOutOfBoundsException(
+                          "array exceeds maximum capacity !");
+        }
+        assert curSize >= 0;
         final int initial = (curSize & MASK_CLR_1);
         int size;
         if (initial > THRESHOLD_ARRAY_SIZE) {
             size = initial + (initial >> 1); // x(3/2)
         } else {
-            size = (initial) << 1; // x2
+            size = (initial << 1); // x2
         }
         // ensure the new size is >= needed size:
         if (size < needSize) {
-            // align to 4096:
+            // align to 4096 (may overflow):
             size = ((needSize >> 12) + 1) << 12;
         }
+        // check integer overflow:
+        if (size < 0) {
+            // resize to maximum capacity:
+            size = Integer.MAX_VALUE;
+        }
         return size;
     }
 
@@ -188,26 +201,29 @@ public final class ArrayCache implements MarlinConst {
      * @return new array size
      */
     public static long getNewLargeSize(final long curSize, final long needSize) {
+        // check if needSize is negative or integer overflow:
+        if ((needSize >> 31L) != 0L) {
+            // hard overflow failure - we can't even accommodate
+            // new items without overflowing
+            throw new ArrayIndexOutOfBoundsException(
+                          "array exceeds maximum capacity !");
+        }
+        assert curSize >= 0L;
         long size;
         if (curSize > THRESHOLD_HUGE_ARRAY_SIZE) {
             size = curSize + (curSize >> 2L); // x(5/4)
         } else  if (curSize > THRESHOLD_LARGE_ARRAY_SIZE) {
             size = curSize + (curSize >> 1L); // x(3/2)
         } else {
-            size = curSize << 1L; // x2
+            size = (curSize << 1L); // x2
         }
         // ensure the new size is >= needed size:
         if (size < needSize) {
             // align to 4096:
-            size = ((needSize >> 12) + 1) << 12;
+            size = ((needSize >> 12L) + 1L) << 12L;
         }
-        if (size >= Integer.MAX_VALUE) {
-            if (curSize >= Integer.MAX_VALUE) {
-                // hard overflow failure - we can't even accommodate
-                // new items without overflowing
-                throw new ArrayIndexOutOfBoundsException(
-                              "array exceeds maximum capacity !");
-            }
+        // check integer overflow:
+        if (size > Integer.MAX_VALUE) {
             // resize to maximum capacity:
             size = Integer.MAX_VALUE;
         }

src/share/classes/sun/java2d/marlin/Stroker.java

@@ -1265,14 +1265,15 @@ final class Stroker implements PathConsumer2D, MarlinConst {
         }
 
         private void ensureSpace(final int n) {
-            if (end + n > curves.length) {
+            // use substraction to avoid integer overflow:
+            if (curves.length - end < n) {
                 if (doStats) {
                     RendererContext.stats.stat_array_stroker_polystack_curves
                         .add(end + n);
                 }
                 curves = rdrCtx.widenDirtyFloatArray(curves, end, end + n);
             }
-            if (numCurves + 1 > curveTypes.length) {
+            if (curveTypes.length <= numCurves) {
                 if (doStats) {
                     RendererContext.stats.stat_array_stroker_polystack_curveTypes
                         .add(numCurves + 1);

test/sun/java2d/marlin/ArrayCacheSizeTest.java

+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import sun.java2d.marlin.ArrayCache;
+
+/**
+ * @test
+ * @bug 8144445
+ * @summary Check the ArrayCache getNewLargeSize() method
+ * @run main ArrayCacheSizeTest
+ */
+public class ArrayCacheSizeTest {
+
+    public static void main(String[] args) {
+        testNewSize();
+        testNewLargeSize();
+    }
+
+    private static void testNewSize() {
+        testNewSize(0, 1);
+        testNewSize(0, 100000);
+
+        testNewSize(4096, 4097);
+        testNewSize(4096 * 16, 4096 * 16 + 1);
+
+        testNewSize(4096 * 4096 * 4, 4096 * 4096 * 4 + 1);
+
+        testNewSize(4096 * 4096 * 4, Integer.MAX_VALUE);
+
+        testNewSize(Integer.MAX_VALUE - 1000, Integer.MAX_VALUE);
+
+        testNewSizeExpectAIOB(Integer.MAX_VALUE - 1000, Integer.MAX_VALUE + 1);
+        testNewSizeExpectAIOB(1, -1);
+        testNewSizeExpectAIOB(Integer.MAX_VALUE, -1);
+    }
+
+    private static void testNewSizeExpectAIOB(final int curSize,
+                                              final int needSize) {
+        try {
+            testNewSize(curSize, needSize);
+            throw new RuntimeException("ArrayIndexOutOfBoundsException not thrown");
+        } catch (ArrayIndexOutOfBoundsException aiobe) {
+            System.out.println("ArrayIndexOutOfBoundsException expected.");
+        } catch (RuntimeException re) {
+            throw re;
+        } catch (Throwable th) {
+            throw new RuntimeException("Unexpected exception", th);
+        }
+    }
+
+    private static void testNewSize(final int curSize,
+                                    final int needSize) {
+
+        int size = ArrayCache.getNewSize(curSize, needSize);
+
+        System.out.println("getNewSize(" + curSize + ", " + needSize
+            + ") = " + size);
+
+        if (size < 0 || size < needSize) {
+            throw new IllegalStateException("Invalid getNewSize("
+                + curSize + ", " + needSize + ") = " + size + " !");
+        }
+    }
+
+    private static void testNewLargeSize() {
+        testNewLargeSize(0, 1);
+        testNewLargeSize(0, 100000);
+
+        testNewLargeSize(4096, 4097);
+        testNewLargeSize(4096 * 16, 4096 * 16 + 1);
+
+        testNewLargeSize(4096 * 4096 * 4, 4096 * 4096 * 4 + 1);
+
+        testNewLargeSize(4096 * 4096 * 4, Integer.MAX_VALUE);
+
+        testNewLargeSize(Integer.MAX_VALUE - 1000, Integer.MAX_VALUE);
+
+        testNewLargeSizeExpectAIOB(Integer.MAX_VALUE - 1000, Integer.MAX_VALUE + 1L);
+        testNewLargeSizeExpectAIOB(1, -1L);
+        testNewLargeSizeExpectAIOB(Integer.MAX_VALUE, -1L);
+    }
+
+    private static void testNewLargeSizeExpectAIOB(final long curSize,
+                                                   final long needSize) {
+        try {
+            testNewLargeSize(curSize, needSize);
+            throw new RuntimeException("ArrayIndexOutOfBoundsException not thrown");
+        } catch (ArrayIndexOutOfBoundsException aiobe) {
+            System.out.println("ArrayIndexOutOfBoundsException expected.");
+        } catch (RuntimeException re) {
+            throw re;
+        } catch (Throwable th) {
+            throw new RuntimeException("Unexpected exception", th);
+        }
+    }
+
+    private static void testNewLargeSize(final long curSize,
+                                         final long needSize) {
+
+        long size = ArrayCache.getNewLargeSize(curSize, needSize);
+
+        System.out.println("getNewLargeSize(" + curSize + ", " + needSize
+            + ") = " + size);
+
+        if (size < 0 || size < needSize || size > Integer.MAX_VALUE) {
+            throw new IllegalStateException("Invalid getNewLargeSize("
+                + curSize + ", " + needSize + ") = " + size + " !");
+        }
+    }
+
+}