8012601: Better validation of image layouts

Reviewed-by: prr

src/share/classes/java/awt/image/BufferedImage.java

@@ -776,6 +776,7 @@ public class BufferedImage extends java.awt.Image
                 }
             }
             if (is8bit &&
+                braster.getPixelStride() == numBands &&
                 offs[0] == numBands-1 &&
                 offs[1] == numBands-2 &&
                 offs[2] == numBands-3)

src/share/classes/sun/java2d/cmm/lcms/LCMSImageLayout.java

@@ -76,6 +76,7 @@ class LCMSImageLayout {
     int width;
     int height;
     int nextRowOffset;
+    private int nextPixelOffset;
     int offset;
 
     /* This flag indicates whether the image can be processed
@@ -93,6 +94,7 @@ class LCMSImageLayout {
         this.pixelType = pixelType;
         width = np;
         height = 1;
+        nextPixelOffset = pixelSize;
         nextRowOffset = safeMult(pixelSize, np);
         offset = 0;
     }
@@ -104,6 +106,7 @@ class LCMSImageLayout {
         this.pixelType = pixelType;
         this.width = width;
         this.height = height;
+        nextPixelOffset = pixelSize;
         nextRowOffset = safeMult(pixelSize, width);
         offset = 0;
     }
@@ -221,6 +224,7 @@ class LCMSImageLayout {
                     IntegerComponentRaster intRaster = (IntegerComponentRaster)
                             image.getRaster();
                     l.nextRowOffset = safeMult(4, intRaster.getScanlineStride());
+                    l.nextPixelOffset = safeMult(4, intRaster.getPixelStride());
                     l.offset = safeMult(4, intRaster.getDataOffset(0));
                     l.dataArray = intRaster.getDataStorage();
                     l.dataArrayLength = 4 * intRaster.getDataStorage().length;
@@ -238,6 +242,8 @@ class LCMSImageLayout {
                     ByteComponentRaster byteRaster = (ByteComponentRaster)
                             image.getRaster();
                     l.nextRowOffset = byteRaster.getScanlineStride();
+                    l.nextPixelOffset = byteRaster.getPixelStride();
+
                     int firstBand = image.getSampleModel().getNumBands() - 1;
                     l.offset = byteRaster.getDataOffset(firstBand);
                     l.dataArray = byteRaster.getDataStorage();
@@ -254,6 +260,8 @@ class LCMSImageLayout {
                     ByteComponentRaster byteRaster = (ByteComponentRaster)
                             image.getRaster();
                     l.nextRowOffset = byteRaster.getScanlineStride();
+                    l.nextPixelOffset = byteRaster.getPixelStride();
+
                     l.dataArrayLength = byteRaster.getDataStorage().length;
                     l.offset = byteRaster.getDataOffset(0);
                     l.dataArray = byteRaster.getDataStorage();
@@ -270,6 +278,8 @@ class LCMSImageLayout {
                     ShortComponentRaster shortRaster = (ShortComponentRaster)
                             image.getRaster();
                     l.nextRowOffset = safeMult(2, shortRaster.getScanlineStride());
+                    l.nextPixelOffset = safeMult(2, shortRaster.getPixelStride());
+
                     l.offset = safeMult(2, shortRaster.getDataOffset(0));
                     l.dataArray = shortRaster.getDataStorage();
                     l.dataArrayLength = 2 * shortRaster.getDataStorage().length;
@@ -331,9 +341,15 @@ class LCMSImageLayout {
             throw new ImageLayoutException("Invalid image layout");
         }
 
-        int lastPixelOffset = safeMult(nextRowOffset, (height - 1));
+        if (nextPixelOffset != getBytesPerPixel(pixelType)) {
+            throw new ImageLayoutException("Invalid image layout");
+        }
+
+        int lastScanOffset = safeMult(nextRowOffset, (height - 1));
 
-        lastPixelOffset = safeAdd(lastPixelOffset, (width - 1));
+        int lastPixelOffset = safeMult(nextPixelOffset, (width -1 ));
+
+        lastPixelOffset = safeAdd(lastPixelOffset, lastScanOffset);
 
         int off = safeAdd(offset, lastPixelOffset);
 
@@ -392,6 +408,8 @@ class LCMSImageLayout {
             }
 
             l.nextRowOffset = br.getScanlineStride();
+            l.nextPixelOffset = br.getPixelStride();
+
             l.offset = br.getDataOffset(firstBand);
             l.dataArray = br.getDataStorage();
             l.dataType = DT_BYTE;
@@ -406,4 +424,25 @@ class LCMSImageLayout {
         }
         return null;
     }
+
+    /**
+     * Derives number of bytes per pixel from the pixel format.
+     * Following bit fields are used here:
+     *  [0..2] - bytes per sample
+     *  [3..6] - number of color samples per pixel
+     *  [7..9] - number of non-color samples per pixel
+     *
+     * A complete description of the pixel format can be found
+     * here: lcms2.h, lines 651 - 667.
+     *
+     * @param pixelType pixel format in lcms2 notation.
+     * @return number of bytes per pixel for given pixel format.
+     */
+    private static int getBytesPerPixel(int pixelType) {
+        int bytesPerSample = (0x7 & pixelType);
+        int colorSamplesPerPixel = 0xF & (pixelType >> 3);
+        int extraSamplesPerPixel = 0x7 & (pixelType >> 7);
+
+        return bytesPerSample * (colorSamplesPerPixel + extraSamplesPerPixel);
+    }
 }