6862968: JPEG Image Writer quantization problem

Reviewed-by: prr, hawtin

src/share/native/sun/awt/image/jpeg/imageioJPEG.c

@@ -676,6 +676,10 @@ static int setQTables(JNIEnv *env,
 #ifdef DEBUG_IIO_JPEG
     printf("in setQTables, qlen = %d, write is %d\n", qlen, write);
 #endif
+    if (qlen > NUM_QUANT_TBLS) {
+        /* Ignore extra qunterization tables. */
+        qlen = NUM_QUANT_TBLS;
+    }
     for (i = 0; i < qlen; i++) {
         table = (*env)->GetObjectArrayElement(env, qtables, i);
         qdata = (*env)->GetObjectField(env, table, JPEGQTable_tableID);
@@ -727,6 +731,11 @@ static void setHuffTable(JNIEnv *env,
     hlensBody = (*env)->GetShortArrayElements(env,
                                               huffLens,
                                               NULL);
+    if (hlensLen > 16) {
+        /* Ignore extra elements of bits array. Only 16 elements can be
+           stored. 0-th element is not used. (see jpeglib.h, line 107)  */
+        hlensLen = 16;
+    }
     for (i = 1; i <= hlensLen; i++) {
         huff_ptr->bits[i] = (UINT8)hlensBody[i-1];
     }
@@ -743,6 +752,11 @@ static void setHuffTable(JNIEnv *env,
                                               huffValues,
                                               NULL);
 
+    if (hvalsLen > 256) {
+        /* Ignore extra elements of hufval array. Only 256 elements
+           can be stored. (see jpeglib.h, line 109)                  */
+        hlensLen = 256;
+    }
     for (i = 0; i < hvalsLen; i++) {
         huff_ptr->huffval[i] = (UINT8)hvalsBody[i];
     }
@@ -763,6 +777,11 @@ static int setHTables(JNIEnv *env,
     j_compress_ptr comp;
     j_decompress_ptr decomp;
     jsize hlen = (*env)->GetArrayLength(env, DCHuffmanTables);
+
+    if (hlen > NUM_HUFF_TBLS) {
+        /* Ignore extra DC huffman tables. */
+        hlen = NUM_HUFF_TBLS;
+    }
     for (i = 0; i < hlen; i++) {
         if (cinfo->is_decompressor) {
             decomp = (j_decompress_ptr) cinfo;
@@ -784,6 +803,10 @@ static int setHTables(JNIEnv *env,
         huff_ptr->sent_table = !write;
     }
     hlen = (*env)->GetArrayLength(env, ACHuffmanTables);
+    if (hlen > NUM_HUFF_TBLS) {
+        /* Ignore extra AC huffman tables. */
+        hlen = NUM_HUFF_TBLS;
+    }
     for (i = 0; i < hlen; i++) {
         if (cinfo->is_decompressor) {
             decomp = (j_decompress_ptr) cinfo;