6872358: JRE AWT setBytePixels vulnerable to Heap Overflow

Reviewed-by: prr, hawtin

make/sun/awt/mapfile-vers

@@ -53,7 +53,6 @@ SUNWprivate_1.1 {
 		Java_sun_awt_image_GifImageDecoder_initIDs;
 		Java_sun_awt_image_GifImageDecoder_parseImage;
 		Java_sun_awt_image_ImageRepresentation_initIDs;
-		Java_sun_awt_image_ImageRepresentation_setBytePixels;
 		Java_sun_awt_image_ImageRepresentation_setDiffICM;
 		Java_sun_awt_image_ImageRepresentation_setICMpixels;
 		Java_sun_awt_image_ImagingLib_convolveBI;

make/sun/awt/mapfile-vers-linux

@@ -55,7 +55,6 @@ SUNWprivate_1.1 {
 		Java_sun_awt_image_GifImageDecoder_parseImage;
 		Java_sun_awt_image_Image_initIDs;
 		Java_sun_awt_image_ImageRepresentation_initIDs;
-		Java_sun_awt_image_ImageRepresentation_setBytePixels;
 		Java_sun_awt_image_ImageRepresentation_setDiffICM;
 		Java_sun_awt_image_ImageRepresentation_setICMpixels;
 		Java_sun_awt_image_ImagingLib_convolveBI;

src/share/classes/sun/awt/image/ImageRepresentation.java

@@ -336,10 +336,6 @@ public class ImageRepresentation extends ImageWatched implements ImageConsumer
     public native void setICMpixels(int x, int y, int w, int h, int[] lut,
                                     byte[] pix, int off, int scansize,
                                     IntegerComponentRaster ict);
-
-    public native void setBytePixels(int x, int y, int w, int h, byte[] pix,
-                                     int off, int scansize,
-                                     ByteComponentRaster bct, int chanOff);
     public native int setDiffICM(int x, int y, int w, int h, int[] lut,
                                  int transPix, int numLut, IndexColorModel icm,
                                  byte[] pix, int off, int scansize,
@@ -450,7 +446,6 @@ public class ImageRepresentation extends ImageWatched implements ImageConsumer
                      (biRaster instanceof ByteComponentRaster) &&
                      (biRaster.getNumDataElements() == 1)){
                 ByteComponentRaster bt = (ByteComponentRaster) biRaster;
-                if (w*h > 200) {
                 if (off == 0 && scansize == w) {
                     bt.putByteData(x, y, w, h, pix);
                 }
@@ -464,15 +459,6 @@ public class ImageRepresentation extends ImageWatched implements ImageConsumer
                     }
                 }
             }
-                else {
-                    // Only is faster if #pixels
-                    // Note that setBytePixels modifies the raster directly
-                    // so we must mark it as changed afterwards
-                    setBytePixels(x, y, w, h, pix, off, scansize, bt,
-                                  bt.getDataOffset(0));
-                    bt.markDirty();
-                }
-            }
             else {
                 for (int yoff=y; yoff < y+h; yoff++, lineOff += scansize) {
                     poff = lineOff;

src/share/native/sun/awt/image/awt_ImageRep.c

@@ -142,84 +142,6 @@ Java_sun_awt_image_ImageRepresentation_setICMpixels(JNIEnv *env, jclass cls,
 
 }
 
-JNIEXPORT void JNICALL
-Java_sun_awt_image_ImageRepresentation_setBytePixels(JNIEnv *env, jclass cls,
-                                                     jint x, jint y, jint w,
-                                                     jint h, jbyteArray jpix,
-                                                     jint off, jint scansize,
-                                                     jobject jbct,
-                                                     jint chanOffs)
-{
-    int sStride;
-    int pixelStride;
-    jobject jdata;
-    unsigned char *srcData;
-    unsigned char *dstData;
-    unsigned char *dataP;
-    unsigned char *pixP;
-    int i;
-    int j;
-
-
-    if (JNU_IsNull(env, jpix)) {
-        JNU_ThrowNullPointerException(env, "NullPointerException");
-        return;
-    }
-
-    sStride = (*env)->GetIntField(env, jbct, g_BCRscanstrID);
-    pixelStride = (*env)->GetIntField(env, jbct, g_BCRpixstrID);
-    jdata = (*env)->GetObjectField(env, jbct, g_BCRdataID);
-
-    srcData = (unsigned char *) (*env)->GetPrimitiveArrayCritical(env, jpix,
-                                                                  NULL);
-    if (srcData == NULL) {
-        /* out of memory error already thrown */
-        return;
-    }
-
-    dstData = (unsigned char *) (*env)->GetPrimitiveArrayCritical(env, jdata,
-                                                                  NULL);
-    if (dstData == NULL) {
-        /* out of memory error already thrown */
-        (*env)->ReleasePrimitiveArrayCritical(env, jpix, srcData, JNI_ABORT);
-        return;
-    }
-
-    dataP = dstData + chanOffs + y*sStride + x*pixelStride;
-    pixP  = srcData + off;
-    if (pixelStride == 1) {
-        if (sStride == scansize && scansize == w) {
-            memcpy(dataP, pixP, w*h);
-        }
-        else {
-            for (i=0; i < h; i++) {
-                memcpy(dataP, pixP, w);
-                dataP += sStride;
-                pixP  += scansize;
-            }
-        }
-    }
-    else {
-        unsigned char *ydataP = dataP;
-        unsigned char *ypixP  = pixP;
-
-        for (i=0; i < h; i++) {
-            dataP = ydataP;
-            pixP = ypixP;
-            for (j=0; j < w; j++) {
-                *dataP = *pixP++;
-                dataP += pixelStride;
-            }
-            ydataP += sStride;
-            ypixP  += scansize;
-        }
-    }
-
-    (*env)->ReleasePrimitiveArrayCritical(env, jpix, srcData, JNI_ABORT);
-    (*env)->ReleasePrimitiveArrayCritical(env, jdata, dstData, JNI_ABORT);
-
-}
-
 JNIEXPORT jint JNICALL
 Java_sun_awt_image_ImageRepresentation_setDiffICM(JNIEnv *env, jclass cls,
                                                   jint x, jint y, jint w,