6963023: ZDI-CAN-809: Sun JRE JPEGImageWriter.writeImage Remote Code Execution Vulnerability

Reviewed-by: prr

src/share/native/sun/awt/image/jpeg/imageioJPEG.c

@@ -2614,7 +2614,8 @@ Java_com_sun_imageio_plugins_jpeg_JPEGImageWriter_writeImage
     JSAMPROW scanLinePtr;
     int i, j;
     int pixelStride;
-    unsigned char *in, *out, *pixelLimit;
+    unsigned char *in, *out, *pixelLimit, *scanLineLimit;
+    unsigned int scanLineSize, pixelBufferSize;
     int targetLine;
     pixelBufferPtr pb;
     sun_jpeg_error_ptr jerr;
@@ -2650,19 +2651,25 @@ Java_com_sun_imageio_plugins_jpeg_JPEGImageWriter_writeImage
 
     }
 
+    scanLineSize = destWidth * numBands;
     if ((inCs < 0) || (inCs > JCS_YCCK) ||
         (outCs < 0) || (outCs > JCS_YCCK) ||
         (numBands < 1) || (numBands > MAX_BANDS) ||
         (srcWidth < 0) ||
         (destWidth < 0) || (destWidth > srcWidth) ||
         (destHeight < 0) ||
-        (stepX < 0) || (stepY < 0))
+        (stepX < 0) || (stepY < 0) ||
+        ((scanLineSize / numBands) < destWidth))  /* destWidth causes an integer overflow */
     {
         JNU_ThrowByName(env, "javax/imageio/IIOException",
                         "Invalid argument to native writeImage");
         return JNI_FALSE;
     }
 
+    if (stepX > srcWidth) {
+        stepX = srcWidth;
+    }
+
     bandSize = (*env)->GetIntArrayElements(env, bandSizes, NULL);
 
     for (i = 0; i < numBands; i++) {
@@ -2710,7 +2717,7 @@ Java_com_sun_imageio_plugins_jpeg_JPEGImageWriter_writeImage
     }
 
     // Allocate a 1-scanline buffer
-    scanLinePtr = (JSAMPROW)malloc(destWidth*numBands);
+    scanLinePtr = (JSAMPROW)malloc(scanLineSize);
     if (scanLinePtr == NULL) {
         RELEASE_ARRAYS(env, data, (const JOCTET *)(dest->next_output_byte));
         JNU_ThrowByName( env,
@@ -2718,6 +2725,7 @@ Java_com_sun_imageio_plugins_jpeg_JPEGImageWriter_writeImage
                          "Writing JPEG Stream");
         return data->abortFlag;
     }
+    scanLineLimit = scanLinePtr + scanLineSize;
 
     /* Establish the setjmp return context for sun_jpeg_error_exit to use. */
     jerr = (sun_jpeg_error_ptr) cinfo->err;
@@ -2866,6 +2874,8 @@ Java_com_sun_imageio_plugins_jpeg_JPEGImageWriter_writeImage
     }
 
     targetLine = 0;
+    pixelBufferSize = srcWidth * numBands;
+    pixelStride = numBands * stepX;
 
     // for each line in destHeight
     while ((data->abortFlag == JNI_FALSE)
@@ -2886,9 +2896,9 @@ Java_com_sun_imageio_plugins_jpeg_JPEGImageWriter_writeImage
 
         in = data->pixelBuf.buf.bp;
         out = scanLinePtr;
-        pixelLimit = in + srcWidth*numBands;
-        pixelStride = numBands*stepX;
-        for (; in < pixelLimit; in += pixelStride) {
+        pixelLimit = in + ((pixelBufferSize > data->pixelBuf.byteBufferLength) ?
+                           data->pixelBuf.byteBufferLength : pixelBufferSize);
+        for (; (in < pixelLimit) && (out < scanLineLimit); in += pixelStride) {
             for (i = 0; i < numBands; i++) {
                 if (scale !=NULL && scale[i] != NULL) {
                     *out++ = scale[i][*(in+i)];