6914866: Sun JRE ImagingLib arbitrary code execution vulnerability

Reviewed-by: prr, hawtin

src/share/native/sun/awt/medialib/awt_ImagingLib.c

@@ -2239,7 +2239,8 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
     int dataType = BYTE_DATA_TYPE;
     int width;
     int height;
-    int size = rasterP->width * rasterP->height * rasterP->numBands;
+    int dataSize;
+    int offset;
 
     *dataPP = NULL;
 
@@ -2292,6 +2293,22 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
 #endif
     switch (rasterP->type) {
     case sun_awt_image_IntegerComponentRaster_TYPE_INT_8BIT_SAMPLES:
+        if (!((rasterP->chanOffsets[0] == 0 || SAFE_TO_ALLOC_2(rasterP->chanOffsets[0], 4)) &&
+              SAFE_TO_ALLOC_2(width, 4) &&
+              SAFE_TO_ALLOC_3(height, rasterP->scanlineStride, 4)))
+        {
+            return -1;
+        }
+        offset = 4 * rasterP->chanOffsets[0];
+        dataSize = 4 * (*env)->GetArrayLength(env, rasterP->jdata);
+
+        if (offset < 0 || offset >= dataSize ||
+            width > rasterP->scanlineStride ||
+            height * rasterP->scanlineStride * 4 > dataSize - offset)
+        {
+            // raster data buffer is too short
+            return -1;
+        }
         dataP = (void *) (*env)->GetPrimitiveArrayCritical(env, rasterP->jdata,
                                                            NULL);
         if (dataP == NULL) {
@@ -2300,11 +2317,25 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
         *mlibImagePP = (*sMlibSysFns.createStructFP)(MLIB_BYTE, 4,
                                               width, height,
                                               rasterP->scanlineStride*4,
-                                              (unsigned char *)dataP
-                                              + rasterP->chanOffsets[0]*4);
+                                              (unsigned char *)dataP + offset);
         *dataPP = dataP;
         return 0;
     case sun_awt_image_IntegerComponentRaster_TYPE_BYTE_SAMPLES:
+        if (!(SAFE_TO_ALLOC_2(width, rasterP->numBands) &&
+              SAFE_TO_ALLOC_2(height, rasterP->scanlineStride)))
+        {
+            return -1;
+        }
+        offset = rasterP->chanOffsets[0];
+        dataSize = (*env)->GetArrayLength(env, rasterP->jdata);
+
+        if (offset < 0 || offset >= dataSize ||
+            width * rasterP->numBands > rasterP->scanlineStride ||
+            height * rasterP->scanlineStride > dataSize - offset)
+        {
+            // raster data buffer is too short
+            return -1;
+        }
         dataP = (void *) (*env)->GetPrimitiveArrayCritical(env, rasterP->jdata,
                                                            NULL);
         if (dataP == NULL) {
@@ -2313,11 +2344,26 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
         *mlibImagePP = (*sMlibSysFns.createStructFP)(MLIB_BYTE, rasterP->numBands,
                                               width, height,
                                               rasterP->scanlineStride,
-                                              (unsigned char *)dataP
-                                              + rasterP->chanOffsets[0]);
+                                              (unsigned char *)dataP + offset);
         *dataPP = dataP;
         return 0;
     case sun_awt_image_IntegerComponentRaster_TYPE_USHORT_SAMPLES:
+        if (!((rasterP->chanOffsets[0] == 0 || SAFE_TO_ALLOC_2(rasterP->chanOffsets[0], 2)) &&
+              SAFE_TO_ALLOC_3(width, rasterP->numBands, 2) &&
+              SAFE_TO_ALLOC_3(height, rasterP->scanlineStride, 2)))
+        {
+              return -1;
+        }
+        offset = rasterP->chanOffsets[0] * 2;
+        dataSize = 2 * (*env)->GetArrayLength(env, rasterP->jdata);
+
+        if (offset < 0 || offset >= dataSize ||
+            width * rasterP->numBands > rasterP->scanlineStride ||
+            height * rasterP->scanlineStride * 2 > dataSize - offset)
+        {
+            // raster data buffer is too short
+             return -1;
+        }
         dataP = (void *) (*env)->GetPrimitiveArrayCritical(env, rasterP->jdata,
                                                            NULL);
         if (dataP == NULL) {
@@ -2327,8 +2373,7 @@ allocateRasterArray(JNIEnv *env, RasterS_t *rasterP,
                                                      rasterP->numBands,
                                                      width, height,
                                                      rasterP->scanlineStride*2,
-                                                     (unsigned char *)dataP
-                                                     + rasterP->chanOffsets[0]*2);
+                                                     (unsigned char *)dataP + offset);
         *dataPP = dataP;
         return 0;
 

src/share/native/sun/awt/medialib/safe_alloc.h

@@ -35,11 +35,11 @@
  */
 #define SAFE_TO_ALLOC_2(c, sz)                                             \
     (((c) > 0) && ((sz) > 0) &&                                            \
-     ((0xffffffffu / ((juint)(c))) > (sz)))
+     ((0xffffffffu / ((juint)(c))) > ((juint)(sz))))
 
 #define SAFE_TO_ALLOC_3(w, h, sz)                                          \
     (((w) > 0) && ((h) > 0) && ((sz) > 0) &&                               \
-    (((0xffffffffu / ((juint)(w))) / ((juint)(h))) > (sz)))
+     (((0xffffffffu / ((juint)(w))) / ((juint)(h))) > ((juint)(sz))))
 
 
 #endif // __SAFE_ALLOC_H__