6782574: AffineTransformOp.filter(BufferedImage, BufferedImage) fails with InternalError

Reviewed-by: igor, prr

src/share/classes/java/awt/image/SinglePixelPackedSampleModel.java

@@ -92,7 +92,8 @@ public class SinglePixelPackedSampleModel extends SampleModel
      * Constructs a SinglePixelPackedSampleModel with bitMasks.length bands.
      * Each sample is stored in a data array element in the position of
      * its corresponding bit mask.  Each bit mask must be contiguous and
-     * masks must not overlap.
+     * masks must not overlap. Bit masks exceeding data type capacity are
+     * truncated.
      * @param dataType  The data type for storing samples.
      * @param w         The width (in pixels) of the region of the
      *                  image data described.
@@ -120,7 +121,8 @@ public class SinglePixelPackedSampleModel extends SampleModel
      * and a scanline stride equal to scanlineStride data array elements.
      * Each sample is stored in a data array element in the position of
      * its corresponding bit mask.  Each bit mask must be contiguous and
-     * masks must not overlap.
+     * masks must not overlap. Bit masks exceeding data type capacity are
+     * truncated.
      * @param dataType  The data type for storing samples.
      * @param w         The width (in pixels) of the region of
      *                  image data described.
@@ -153,11 +155,13 @@ public class SinglePixelPackedSampleModel extends SampleModel
         this.bitOffsets = new int[numBands];
         this.bitSizes = new int[numBands];
 
+        int maxMask = (int)((1L << DataBuffer.getDataTypeSize(dataType)) - 1);
+
         this.maxBitSize = 0;
         for (int i=0; i<numBands; i++) {
             int bitOffset = 0, bitSize = 0, mask;
-            mask = bitMasks[i];
-
+            this.bitMasks[i] &= maxMask;
+            mask = this.bitMasks[i];
             if (mask != 0) {
                 while ((mask & 1) == 0) {
                     mask = mask >>> 1;
@@ -243,30 +247,12 @@ public class SinglePixelPackedSampleModel extends SampleModel
 
     /** Returns the number of bits per sample for all bands. */
     public int[] getSampleSize() {
-        int mask;
-        int sampleSize[] = new int [numBands];
-        for (int i=0; i<numBands; i++) {
-            sampleSize[i] = 0;
-            mask = bitMasks[i] >>> bitOffsets[i];
-            while ((mask & 1) != 0) {
-                sampleSize[i] ++;
-                mask = mask >>> 1;
-            }
-        }
-
-        return sampleSize;
+        return bitSizes.clone();
     }
 
     /** Returns the number of bits per sample for the specified band. */
     public int getSampleSize(int band) {
-        int sampleSize = 0;
-        int mask = bitMasks[band] >>> bitOffsets[band];
-        while ((mask & 1) != 0) {
-            sampleSize ++;
-            mask = mask >>> 1;
-        }
-
-        return sampleSize;
+        return bitSizes[band];
     }
 
     /** Returns the offset (in data array elements) of pixel (x,y).

src/share/native/sun/awt/image/awt_parseImage.c

@@ -178,7 +178,7 @@ int awt_parseRaster(JNIEnv *env, jobject jraster, RasterS_t *rasterP) {
         jnbits = (*env)->GetObjectField(env, rasterP->jsampleModel,
                                         g_SPPSMnBitsID);
         if (jmask == NULL || joffs == NULL || jnbits == NULL ||
-            rasterP->sppsm.maxBitSize < 0 || rasterP->sppsm.maxBitSize > 8)
+            rasterP->sppsm.maxBitSize < 0)
         {
             JNU_ThrowInternalError(env, "Can't grab SPPSM fields");
             return -1;
@@ -280,6 +280,17 @@ int awt_parseRaster(JNIEnv *env, jobject jraster, RasterS_t *rasterP) {
                                   rasterP->chanOffsets);
     }
 
+    /* additioanl check for sppsm fields validity: make sure that
+     * size of raster samples doesn't exceed the data type cpacity.
+     */
+    if (rasterP->dataType > UNKNOWN_DATA_TYPE && /* data type has been recognized */
+        rasterP->sppsm.maxBitSize > 0 && /* raster has SPP sample model */
+        rasterP->sppsm.maxBitSize > (rasterP->dataSize * 8))
+    {
+        JNU_ThrowInternalError(env, "Raster samples are too big");
+        return -1;
+    }
+
 #if 0
     fprintf(stderr,"---------------------\n");
     fprintf(stderr,"Width  : %d\n",rasterP->width);

test/java/awt/image/IncorrectSampleMaskTest.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug     6782574
+ * @summary Test verifies that incorrect sample masks are correctly handled
+ *          by the constructor of the SinglePixelPackedSampleModel class
+ *          and do not cause internal error in the medialib glue code.
+ *
+ * @run     main IncorrectSampleMaskTest
+ */
+
+import java.awt.geom.AffineTransform;
+import java.awt.image.AffineTransformOp;
+import java.awt.image.BufferedImageOp;
+import java.awt.image.DataBuffer;
+import java.awt.image.DataBufferByte;
+import java.awt.image.DataBufferInt;
+import java.awt.image.DataBufferUShort;
+import java.awt.image.Raster;
+import java.awt.image.RasterOp;
+import java.awt.image.WritableRaster;
+import java.awt.image.SinglePixelPackedSampleModel;
+
+public class IncorrectSampleMaskTest {
+    public static void main(String[] args) {
+        int[] dataTypes = new int[] {
+            DataBuffer.TYPE_BYTE,
+            DataBuffer.TYPE_USHORT,
+            DataBuffer.TYPE_INT };
+
+        for (int type : dataTypes) {
+            doTest(type);
+        }
+    }
+
+    private static final int w = 100;
+    private static final int h = 100;
+
+    private static AffineTransform at =
+        AffineTransform.getScaleInstance(0.5, 0.5);
+
+    private static RasterOp op =
+        new AffineTransformOp(at, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);
+
+    private static void doTest(int dataType) {
+        int maxSize = DataBuffer.getDataTypeSize(dataType);
+        System.out.println("Type size: " + maxSize);
+
+        int theMask = (int)(1L << (maxSize + 2)) - 1;
+        System.out.printf("theMask=%x\n", theMask);
+
+        SinglePixelPackedSampleModel sm =
+            new SinglePixelPackedSampleModel(dataType, w, h,
+                                             new int[] { theMask });
+
+
+        int[] sampleSize = sm.getSampleSize();
+        for (int s : sampleSize) {
+            if (s > maxSize) {
+                throw new RuntimeException("Test failed: sample size is too big:" + s);
+            }
+        }
+
+        System.out.println("Test medialib...");
+        DataBuffer buf = createDataBuffer(dataType);
+
+        WritableRaster wr = Raster.createWritableRaster(sm, buf, null);
+
+        op.filter(wr, null);
+        System.out.println("Test PASSED.");
+    }
+
+    private static DataBuffer createDataBuffer(int type) {
+        switch (type) {
+        case DataBuffer.TYPE_BYTE: {
+            byte[] buf = new byte[w * h];
+            return new DataBufferByte(buf, buf.length);
+        }
+        case DataBuffer.TYPE_USHORT: {
+            short[] buf = new short[w * h];
+            return new DataBufferUShort(buf, buf.length);
+        }
+        case DataBuffer.TYPE_INT: {
+            int[] buf = new int[w * h];
+            return new DataBufferInt(buf, buf.length);
+        }
+        default :
+            throw new RuntimeException("Unsupported data type.");
+        }
+    }
+}