8007014: Improve image handling

Reviewed-by: prr, mschoene, jgodinez

src/share/classes/sun/awt/image/ByteComponentRaster.java

@@ -868,6 +868,15 @@ public class ByteComponentRaster extends SunWritableRaster {
      * or if data buffer has not enough capacity.
      */
     protected final void verify() {
+        /* Need to re-verify the dimensions since a sample model may be
+         * specified to the constructor
+         */
+        if (width <= 0 || height <= 0 ||
+            height > (Integer.MAX_VALUE / width))
+        {
+            throw new RasterFormatException("Invalid raster dimension");
+        }
+
         for (int i = 0; i < dataOffsets.length; i++) {
             if (dataOffsets[i] < 0) {
                 throw new RasterFormatException("Data offsets for band " + i
@@ -905,13 +914,14 @@ public class ByteComponentRaster extends SunWritableRaster {
         lastPixelOffset += lastScanOffset;
 
         for (int i = 0; i < numDataElements; i++) {
-            size = lastPixelOffset + dataOffsets[i];
             if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
                 throw new RasterFormatException("Incorrect band offset: "
                             + dataOffsets[i]);
 
             }
 
+            size = lastPixelOffset + dataOffsets[i];
+
             if (size > maxSize) {
                 maxSize = size;
             }

src/share/classes/sun/awt/image/BytePackedRaster.java

@@ -1368,11 +1368,35 @@ public class BytePackedRaster extends SunWritableRaster {
             throw new RasterFormatException("Data offsets must be >= 0");
         }
 
+        /* Need to re-verify the dimensions since a sample model may be
+         * specified to the constructor
+         */
+        if (width <= 0 || height <= 0 ||
+            height > (Integer.MAX_VALUE / width))
+        {
+            throw new RasterFormatException("Invalid raster dimension");
+        }
+
+
+        /*
+         * pixelBitstride was verified in constructor, so just make
+         * sure that it is safe to multiply it by width.
+         */
+        if ((width - 1) > Integer.MAX_VALUE / pixelBitStride) {
+            throw new RasterFormatException("Invalid raster dimension");
+        }
+
+        if (scanlineStride < 0 ||
+            scanlineStride > (Integer.MAX_VALUE / height))
+        {
+            throw new RasterFormatException("Invalid scanline stride");
+        }
+
         int lastbit = (dataBitOffset
                        + (height-1) * scanlineStride * 8
                        + (width-1) * pixelBitStride
                        + pixelBitStride - 1);
-        if (lastbit / 8 >= data.length) {
+        if (lastbit < 0 || lastbit / 8 >= data.length) {
             throw new RasterFormatException("raster dimensions overflow " +
                                             "array bounds");
         }

src/share/classes/sun/awt/image/IntegerComponentRaster.java

@@ -208,7 +208,7 @@ public class IntegerComponentRaster extends SunWritableRaster {
                                             " SinglePixelPackedSampleModel");
         }
 
-        verify(false);
+        verify();
     }
 
 
@@ -629,16 +629,26 @@ public class IntegerComponentRaster extends SunWritableRaster {
     }
 
     /**
-     * Verify that the layout parameters are consistent with
-     * the data.  If strictCheck
-     * is false, this method will check for ArrayIndexOutOfBounds conditions.  If
-     * strictCheck is true, this method will check for additional error
-     * conditions such as line wraparound (width of a line greater than
-     * the scanline stride).
-     * @return   String   Error string, if the layout is incompatible with
-     *                    the data.  Otherwise returns null.
+     * Verify that the layout parameters are consistent with the data.
+     *
+     * The method verifies whether scanline stride and pixel stride do not
+     * cause an integer overflow during calculation of a position of the pixel
+     * in data buffer. It also verifies whether the data buffer has enough data
+     *  to correspond the raster layout attributes.
+     *
+     * @throws RasterFormatException if an integer overflow is detected,
+     * or if data buffer has not enough capacity.
      */
-    private void verify (boolean strictCheck) {
+    protected final void verify() {
+        /* Need to re-verify the dimensions since a sample model may be
+         * specified to the constructor
+         */
+        if (width <= 0 || height <= 0 ||
+            height > (Integer.MAX_VALUE / width))
+        {
+            throw new RasterFormatException("Invalid raster dimension");
+        }
+
         if (dataOffsets[0] < 0) {
             throw new RasterFormatException("Data offset ("+dataOffsets[0]+
                                             ") must be >= 0");
@@ -647,17 +657,46 @@ public class IntegerComponentRaster extends SunWritableRaster {
         int maxSize = 0;
         int size;
 
-        for (int i=0; i < numDataElements; i++) {
-            size = (height-1)*scanlineStride + (width-1)*pixelStride +
-                dataOffsets[i];
+        // we can be sure that width and height are greater than 0
+        if (scanlineStride < 0 ||
+            scanlineStride > (Integer.MAX_VALUE / height))
+        {
+            // integer overflow
+            throw new RasterFormatException("Incorrect scanline stride: "
+                    + scanlineStride);
+        }
+        int lastScanOffset = (height - 1) * scanlineStride;
+
+        if (pixelStride < 0 ||
+            pixelStride > (Integer.MAX_VALUE / width))
+        {
+            // integer overflow
+            throw new RasterFormatException("Incorrect pixel stride: "
+                    + pixelStride);
+        }
+        int lastPixelOffset = (width - 1) * pixelStride;
+
+        if (lastPixelOffset > (Integer.MAX_VALUE - lastScanOffset)) {
+            // integer overflow
+            throw new RasterFormatException("Incorrect raster attributes");
+        }
+        lastPixelOffset += lastScanOffset;
+
+        for (int i = 0; i < numDataElements; i++) {
+            if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
+                throw new RasterFormatException("Incorrect band offset: "
+                            + dataOffsets[i]);
+            }
+
+            size = lastPixelOffset + dataOffsets[i];
+
             if (size > maxSize) {
                 maxSize = size;
             }
         }
         if (data.length < maxSize) {
-            throw new RasterFormatException("Data array too small (should be "+
-                                          maxSize
-                                          +" but is "+data.length+" )");
+            throw new RasterFormatException("Data array too small (should be "
+                    + maxSize + " )");
         }
     }
 

src/share/classes/sun/awt/image/IntegerInterleavedRaster.java

@@ -151,7 +151,7 @@ public class IntegerInterleavedRaster extends IntegerComponentRaster {
             throw new RasterFormatException("IntegerInterleavedRasters must have"+
                                             " SinglePixelPackedSampleModel");
         }
-        verify(false);
+        verify();
     }
 
 
@@ -540,31 +540,6 @@ public class IntegerInterleavedRaster extends IntegerComponentRaster {
         return createCompatibleWritableRaster(width,height);
     }
 
-    /**
-     * Verify that the layout parameters are consistent with
-     * the data.  If strictCheck
-     * is false, this method will check for ArrayIndexOutOfBounds conditions.  If
-     * strictCheck is true, this method will check for additional error
-     * conditions such as line wraparound (width of a line greater than
-     * the scanline stride).
-     * @return   String   Error string, if the layout is incompatible with
-     *                    the data.  Otherwise returns null.
-     */
-    private void verify (boolean strictCheck) {
-        int maxSize = 0;
-        int size;
-
-        size = (height-1)*scanlineStride + (width-1) + dataOffsets[0];
-        if (size > maxSize) {
-            maxSize = size;
-        }
-        if (data.length < maxSize) {
-            throw new RasterFormatException("Data array too small (should be "+
-                                          maxSize
-                                          +" but is "+data.length+" )");
-        }
-    }
-
     public String toString() {
         return new String ("IntegerInterleavedRaster: width = "+width
                            +" height = " + height

src/share/classes/sun/awt/image/ShortComponentRaster.java

@@ -802,6 +802,15 @@ public class ShortComponentRaster extends SunWritableRaster {
      * or if data buffer has not enough capacity.
      */
     protected final void verify() {
+        /* Need to re-verify the dimensions since a sample model may be
+         * specified to the constructor
+         */
+        if (width <= 0 || height <= 0 ||
+            height > (Integer.MAX_VALUE / width))
+        {
+            throw new RasterFormatException("Invalid raster dimension");
+        }
+
         for (int i = 0; i < dataOffsets.length; i++) {
             if (dataOffsets[i] < 0) {
                 throw new RasterFormatException("Data offsets for band " + i
@@ -839,12 +848,13 @@ public class ShortComponentRaster extends SunWritableRaster {
         lastPixelOffset += lastScanOffset;
 
         for (int i = 0; i < numDataElements; i++) {
-            size = lastPixelOffset + dataOffsets[i];
             if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
                 throw new RasterFormatException("Incorrect band offset: "
                             + dataOffsets[i]);
             }
 
+            size = lastPixelOffset + dataOffsets[i];
+
             if (size > maxSize) {
                 maxSize = size;
             }

src/share/native/sun/awt/image/awt_parseImage.c

@@ -34,6 +34,7 @@
 #include "java_awt_color_ColorSpace.h"
 #include "awt_Mlib.h"
 #include "safe_alloc.h"
+#include "safe_math.h"
 
 static int setHints(JNIEnv *env, BufImageS_t *imageP);
 

src/share/native/sun/awt/medialib/awt_ImagingLib.c

@@ -42,6 +42,7 @@
 #include "awt_Mlib.h"
 #include "gdefs.h"
 #include "safe_alloc.h"
+#include "safe_math.h"
 
 /***************************************************************************
  *                               Definitions                               *
@@ -1993,13 +1994,23 @@ cvtCustomToDefault(JNIEnv *env, BufImageS_t *imageP, int component,
     unsigned char *dP = dataP;
 #define NUM_LINES    10
     int numLines = NUM_LINES;
-    int nbytes = rasterP->width*4*NUM_LINES;
+    /* it is safe to calculate the scan length, because width has been verified
+     * on creation of the mlib image
+     */
+    int scanLength = rasterP->width * 4;
+
+    int nbytes = 0;
+    if (!SAFE_TO_MULT(numLines, scanLength)) {
+        return -1;
+    }
+
+    nbytes = numLines * scanLength;
 
     for (y=0; y < rasterP->height; y+=numLines) {
         /* getData, one scanline at a time */
         if (y+numLines > rasterP->height) {
             numLines = rasterP->height - y;
-            nbytes = rasterP->width*4*numLines;
+            nbytes = numLines * scanLength;
         }
         jpixels = (*env)->CallObjectMethod(env, imageP->jimage,
                                            g_BImgGetRGBMID, 0, y,
@@ -2129,8 +2140,14 @@ allocateArray(JNIEnv *env, BufImageS_t *imageP,
     if (cvtToDefault) {
         int status = 0;
         *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, 4, width, height);
+        if (*mlibImagePP == NULL) {
+            return -1;
+        }
         cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
-        /* Make sure the image is cleared */
+        /* Make sure the image is cleared.
+         * NB: the image dimension is already verified, so we can
+         * safely calculate the length of the buffer.
+         */
         memset(cDataP, 0, width*height*4);
 
         if (!isSrc) {
@@ -2380,6 +2397,9 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
     case sun_awt_image_IntegerComponentRaster_TYPE_BYTE_PACKED_SAMPLES:
         *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
                                         width, height);
+        if (*mlibImagePP == NULL) {
+            return -1;
+        }
         if (!isSrc) return 0;
         cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
         return expandPackedBCR(env, rasterP, -1, cDataP);
@@ -2388,6 +2408,9 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
         if (rasterP->sppsm.maxBitSize <= 8) {
             *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
                                             width, height);
+            if (*mlibImagePP == NULL) {
+                return -1;
+            }
             if (!isSrc) return 0;
             cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
             return expandPackedSCR(env, rasterP, -1, cDataP);
@@ -2397,6 +2420,9 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
         if (rasterP->sppsm.maxBitSize <= 8) {
             *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
                                             width, height);
+            if (*mlibImagePP == NULL) {
+                return -1;
+            }
             if (!isSrc) return 0;
             cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
             return expandPackedICR(env, rasterP, -1, cDataP);

src/share/native/sun/awt/medialib/mlib_ImageCreate.c

@@ -120,6 +120,7 @@
 #include "mlib_image.h"
 #include "mlib_ImageRowTable.h"
 #include "mlib_ImageCreate.h"
+#include "safe_math.h"
 
 /***************************************************************/
 mlib_image* mlib_ImageSet(mlib_image *image,
@@ -247,28 +248,50 @@ mlib_image *mlib_ImageCreate(mlib_type type,
     return NULL;
   };
 
+  if (!SAFE_TO_MULT(width, channels)) {
+    return NULL;
+  }
+
+  wb = width * channels;
+
   switch (type) {
     case MLIB_DOUBLE:
-      wb = width * channels * 8;
+      if (!SAFE_TO_MULT(wb, 8)) {
+        return NULL;
+      }
+      wb *= 8;
       break;
     case MLIB_FLOAT:
     case MLIB_INT:
-      wb = width * channels * 4;
+      if (!SAFE_TO_MULT(wb, 4)) {
+        return NULL;
+      }
+      wb *= 4;
       break;
     case MLIB_USHORT:
     case MLIB_SHORT:
-      wb = width * channels * 2;
+      if (!SAFE_TO_MULT(wb, 4)) {
+        return NULL;
+      }
+      wb *= 2;
       break;
     case MLIB_BYTE:
-      wb = width * channels;
+      // wb is ready
       break;
     case MLIB_BIT:
-      wb = (width * channels + 7) / 8;
+      if (!SAFE_TO_ADD(7, wb)) {
+        return NULL;
+      }
+      wb = (wb + 7) / 8;
       break;
     default:
       return NULL;
   }
 
+  if (!SAFE_TO_MULT(wb, height)) {
+      return NULL;
+  }
+
   data = mlib_malloc(wb * height);
   if (data == NULL) {
     return NULL;

src/share/native/sun/awt/medialib/safe_alloc.h

@@ -41,10 +41,4 @@
     (((w) > 0) && ((h) > 0) && ((sz) > 0) &&                               \
      (((0xffffffffu / ((juint)(w))) / ((juint)(h))) > ((juint)(sz))))
 
-#define SAFE_TO_MULT(a, b) \
-    (((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
-
-#define SAFE_TO_ADD(a, b) \
-    (((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
-
 #endif // __SAFE_ALLOC_H__

src/share/native/sun/awt/medialib/safe_math.h

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef __SAFE_MATH_H__
+#define __SAFE_MATH_H__
+
+#define SAFE_TO_MULT(a, b) \
+    (((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
+
+#define SAFE_TO_ADD(a, b) \
+    (((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
+
+#endif // __SAFE_MATH_H__