8011992: java/awt/image/mlib/MlibOpsTest.java failed since jdk7u25b05

Reviewed-by: vadim

src/share/native/sun/awt/medialib/awt_ImagingLib.c

@@ -1298,29 +1298,39 @@ Java_sun_awt_image_ImagingLib_lookupByteBI(JNIEnv *env, jobject thisLib,
         return 0;
     }
 
-    lut_nbands = (*env)->GetArrayLength(env, jtableArrays);
-
-    ncomponents = srcImageP->cmodel.isDefaultCompatCM
-        ? 4
-        : srcImageP->cmodel.numComponents;
+    nbands = setImageHints(env, srcImageP, dstImageP, FALSE, TRUE,
+                        FALSE, &hint);
 
-    if (lut_nbands > ncomponents) {
-        lut_nbands = ncomponents;
+    if (nbands < 1 || nbands > srcImageP->cmodel.numComponents) {
+        /* Can't handle any custom images */
+        awt_freeParsedImage(srcImageP, TRUE);
+        awt_freeParsedImage(dstImageP, TRUE);
+        return 0;
     }
 
     /* Make sure that color order can be used for
      * re-ordering of lookup arrays.
      */
-    for (i = 0; i < ncomponents; i++) {
+    for (i = 0; i < nbands; i++) {
         int idx = srcImageP->hints.colorOrder[i];
 
-        if (idx < 0 || idx >= ncomponents) {
+        if (idx < 0 || idx >= nbands) {
             awt_freeParsedImage(srcImageP, TRUE);
             awt_freeParsedImage(dstImageP, TRUE);
             return 0;
         }
     }
 
+    lut_nbands = (*env)->GetArrayLength(env, jtableArrays);
+
+    ncomponents = srcImageP->cmodel.isDefaultCompatCM
+        ? 4
+        : srcImageP->cmodel.numComponents;
+
+    if (lut_nbands > ncomponents) {
+        lut_nbands = ncomponents;
+    }
+
     tbl = NULL;
     if (SAFE_TO_ALLOC_2(ncomponents, sizeof(unsigned char *))) {
         tbl = (unsigned char **)
@@ -1363,17 +1373,6 @@ Java_sun_awt_image_ImagingLib_lookupByteBI(JNIEnv *env, jobject thisLib,
         }
     }
 
-    nbands = setImageHints(env, srcImageP, dstImageP, FALSE, TRUE,
-                        FALSE, &hint);
-    if (nbands < 1) {
-        /* Can't handle any custom images */
-        free(tbl);
-        free(jtable);
-        awt_freeParsedImage(srcImageP, TRUE);
-        awt_freeParsedImage(dstImageP, TRUE);
-        return 0;
-    }
-
     /* Allocate the arrays */
     if (allocateArray(env, srcImageP, &src, &sdata, TRUE, FALSE, FALSE) < 0) {
         /* Must be some problem */

test/java/awt/image/mlib/MlibOpsTest.java

@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug     6556332
+ * @bug     6556332 8011992
  * @summary Test verifies that on-demnad loading of medialib library does
  *          not break imageing ops based on this library.
  * @run     main MlibOpsTest