6710199: SJIS_0213 does not handle "unmappable" encoding operation correctly

6699038: sun/nio/cs/findencoderBugs.java fails
Summary: SJIS_0213 charset updates
Reviewed-by: okutsu

src/share/classes/sun/nio/cs/CharsetMapping.java

@@ -37,7 +37,7 @@ import java.security.*;
 
 public class CharsetMapping {
     public final static char UNMAPPABLE_DECODING = '\uFFFD';
-    public final static int  UNMAPPABLE_ENCODING = -1;
+    public final static int  UNMAPPABLE_ENCODING = 0xFFFD;
 
     char[] b2cSB;                //singlebyte b->c
     char[] b2cDB1;               //dobulebyte b->c /db1
@@ -109,9 +109,11 @@ public class CharsetMapping {
     }
 
     public int encodeSurrogate(char hi, char lo) {
-        char c = (char)Character.toCodePoint(hi, lo);
+        int cp = Character.toCodePoint(hi, lo);
+        if (cp < 0x20000 || cp >= 0x30000)
+            return UNMAPPABLE_ENCODING;
         int end = c2bSupp.length / 2;
-        int i = Arrays.binarySearch(c2bSupp, 0, end, c);
+        int i = Arrays.binarySearch(c2bSupp, 0, end, (char)cp);
         if (i >= 0)
             return c2bSupp[end + i];
         return UNMAPPABLE_ENCODING;

src/share/classes/sun/nio/cs/ext/SJIS_0213.java

@@ -274,14 +274,14 @@ public class SJIS_0213 extends Charset {
                         leftoverBase = c;
                     } else {
                         db = encodeChar(c);
-                        if (db > MAX_SINGLEBYTE) {      // DoubleByte
-                            if (dl - dp < 2)
+                        if (db <= MAX_SINGLEBYTE) {      // SingleByte
+                            if (dl <= dp)
                                 return CoderResult.OVERFLOW;
-                            da[dp++] = (byte)(db >> 8);
                             da[dp++] = (byte)db;
-                        } else if (db != UNMAPPABLE) {  // SingleByte
-                            if (dl <= dp)
+                        } else if (db != UNMAPPABLE) {   // DoubleByte
+                            if (dl - dp < 2)
                                 return CoderResult.OVERFLOW;
+                            da[dp++] = (byte)(db >> 8);
                             da[dp++] = (byte)db;
                         } else if (Character.isHighSurrogate(c)) {
                             if ((sp + 1) == sl)
@@ -297,6 +297,8 @@ public class SJIS_0213 extends Charset {
                             da[dp++] = (byte)(db >> 8);
                             da[dp++] = (byte)db;
                             sp++;
+                        } else if (Character.isLowSurrogate(c)) {
+                            return CoderResult.malformedForLength(1);
                         } else {
                             return CoderResult.unmappableForLength(1);
                         }
@@ -337,15 +339,15 @@ public class SJIS_0213 extends Charset {
                         leftoverBase = c;
                     } else {
                         db = encodeChar(c);
-                        if (db > MAX_SINGLEBYTE) {        // DoubleByte
+                        if (db <= MAX_SINGLEBYTE) {    // Single-byte
+                            if (dst.remaining() < 1)
+                                return CoderResult.OVERFLOW;
+                            dst.put((byte)db);
+                        } else if (db != UNMAPPABLE) {   // DoubleByte
                             if (dst.remaining() < 2)
                                 return CoderResult.OVERFLOW;
                             dst.put((byte)(db >> 8));
                             dst.put((byte)(db));
-                        } else if (db != UNMAPPABLE) {    // Single-byte
-                            if (dst.remaining() < 1)
-                                return CoderResult.OVERFLOW;
-                            dst.put((byte)db);
                         } else if (Character.isHighSurrogate(c)) {
                             if (!src.hasRemaining())     // Surrogates
                                 return CoderResult.UNDERFLOW;
@@ -360,6 +362,8 @@ public class SJIS_0213 extends Charset {
                             dst.put((byte)(db >> 8));
                             dst.put((byte)(db));
                             mark++;
+                        } else if (Character.isLowSurrogate(c)) {
+                            return CoderResult.malformedForLength(1);
                         } else {
                             return CoderResult.unmappableForLength(1);
                         }