6843079: Putback for the new EUC_TW is not complete

Summary: Putback the files missed in last putback
Reviewed-by: alanb

make/tools/src/build/tools/charsetmapping/GenerateEUC_TW.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+package build.tools.charsetmapping;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Scanner;
+import java.util.Formatter;
+import java.util.regex.*;
+import java.nio.charset.*;
+import static build.tools.charsetmapping.CharsetMapping.*;
+
+public class GenerateEUC_TW {
+
+    public static void genEUC_TW(String args[]) throws Exception {
+        genClass(new FileInputStream(new File(args[0], "euc_tw.map")),
+                 new PrintStream(new File(args[1], "EUC_TWMapping.java"), "ISO-8859-1"),
+                 getCopyright(new File(args[3])));
+    }
+
+    private static String getCopyright(File f) throws IOException {
+        Scanner s = new Scanner(f, "ISO-8859-1");
+        StringBuilder sb = new StringBuilder();
+        while (s.hasNextLine()) {
+            String ln = s.nextLine();
+            sb.append(ln + "\n");
+            // assume we have the copyright as the first comment
+            if (ln.matches("^\\s\\*\\/$"))
+                break;
+        }
+        s.close();
+        return sb.toString();
+    }
+
+    private static char[] toCharArray(int[] db,
+                                      int b1Min, int b1Max,
+                                      int b2Min, int b2Max)
+    {
+        char[] ca = new char[(b1Max - b1Min + 1) * (b2Max - b2Min + 1)];
+        int off = 0;
+        for (int b1 = b1Min; b1 <= b1Max; b1++) {
+            for (int b2 = b2Min; b2 <= b2Max; b2++) {
+                ca[off++] = (char)(db[b1 * 256 + b2] & 0xffff);
+            }
+        }
+        return ca;
+    }
+
+    private static void toChar(Formatter out, String fmt, char c) {
+        switch (c) {
+        case '\b':
+            out.format("\\b"); break;
+        case '\t':
+            out.format("\\t"); break;
+        case '\n':
+            out.format("\\n"); break;
+        case '\f':
+            out.format("\\f"); break;
+        case '\r':
+            out.format("\\r"); break;
+        case '\"':
+            out.format("\\\""); break;
+        case '\'':
+            out.format("\\'"); break;
+        case '\\':
+            out.format("\\\\"); break;
+        default:
+            out.format(fmt, c & 0xffff);
+        }
+    }
+
+    private static void toString(Formatter out, char[] date, String endStr)
+    {
+        int off = 0;
+        int end = date.length;
+        while (off < end) {
+            out.format("        \"");
+            for (int j = 0; j < 8 && off < end; j++) {
+                toChar(out, "\\u%04X", date[off++]);
+            }
+            if (off == end)
+               out.format("\"%s%n", endStr);
+            else
+               out.format("\" +%n");
+        }
+    }
+
+    private static char[] toCharArray(byte[] ba,
+                                      int b1Min, int b1Max,
+                                      int b2Min, int b2Max)
+    {
+        char[] ca = new char[(b1Max - b1Min + 1) * (b2Max - b2Min + 1)];
+        int off = 0;
+        for (int b1 = b1Min; b1 <= b1Max; b1++) {
+            int b2 = b2Min;
+            while (b2 <= b2Max) {
+                ca[off++] = (char)(((ba[b1 * 256 + b2++] & 0xff) << 8) |
+                                   (ba[b1 * 256 + b2++] & 0xff));
+            }
+        }
+        return ca;
+    }
+
+    private static void toCharArray(Formatter out, char[] date) {
+        int off = 0;
+        int end = date.length;
+        while (off < end) {
+            out.format("        ");
+            for (int j = 0; j < 8 && off < end; j++) {
+                toChar(out, "'\\u%04X',", date[off++]);
+            }
+            out.format("%n");
+        }
+    }
+
+    private static int initC2BIndex(char[] index) {
+        int off = 0;
+        for (int i = 0; i < index.length; i++) {
+            if (index[i] != 0) {
+                index[i] = (char)off;
+                off += 0x100;
+            } else {
+                index[i] = CharsetMapping.UNMAPPABLE_ENCODING;
+            }
+        }
+        return off;
+    }
+
+    private static Pattern euctw = Pattern.compile("(?:8ea)?(\\p{XDigit}++)\\s++(\\p{XDigit}++)?\\s*+.*");
+
+    private static void genClass(InputStream is, PrintStream ps, String copyright)
+        throws Exception
+    {
+        // ranges of byte1 and byte2, something should come from a "config" file
+        int b1Min = 0xa1;
+        int b1Max = 0xfe;
+        int b2Min = 0xa1;
+        int b2Max = 0xfe;
+
+        try {
+            int[][] db = new int[8][0x10000];        // doublebyte
+            byte[]  suppFlag = new byte[0x10000];    // doublebyte
+            char[]  indexC2B = new char[256];
+            char[]  indexC2BSupp = new char[256];
+
+            for (int i = 0; i < 8; i++)
+                for (int j = 0; j < 0x10000; j++)
+                    db[i][j] = CharsetMapping.UNMAPPABLE_DECODING;
+
+            CharsetMapping.Parser p = new CharsetMapping.Parser(is, euctw);
+            CharsetMapping.Entry  e = null;
+            while ((e = p.next()) != null) {
+                int plane = 0;
+                if (e.bs >= 0x10000) {
+                    plane = ((e.bs >> 16) & 0xff) - 1;
+                    if (plane >= 14)
+                        plane = 7;
+                    e.bs = e.bs & 0xffff;
+                }
+                db[plane][e.bs] = e.cp;
+                if (e.cp < 0x10000) {
+                    indexC2B[e.cp>>8] = 1;
+                } else {
+                    indexC2BSupp[(e.cp&0xffff)>>8] = 1;
+                    suppFlag[e.bs] |= (1 << plane);
+                }
+            }
+
+            StringBuilder out = new StringBuilder();
+            Formatter fm = new Formatter(out);
+
+            fm.format(copyright);
+            fm.format("%n// -- This file was mechanically generated: Do not edit! -- //%n");
+            fm.format("package sun.nio.cs.ext;%n%n");
+            fm.format("class EUC_TWMapping {%n%n");
+
+            // boundaries
+            fm.format("    final static int b1Min = 0x%x;%n", b1Min);
+            fm.format("    final static int b1Max = 0x%x;%n", b1Max);
+            fm.format("    final static int b2Min = 0x%x;%n", b2Min);
+            fm.format("    final static int b2Max = 0x%x;%n", b2Max);
+
+            // b2c tables
+            fm.format("%n    final static String[] b2c = {%n");
+            for (int plane = 0; plane < 8; plane++) {
+                fm.format("        // Plane %d%n", plane);
+                toString(fm, toCharArray(db[plane],
+                                         b1Min, b1Max, b2Min, b2Max),
+                         ",");
+                fm.format("%n");
+            }
+            fm.format("    };%n");
+
+            // c2bIndex
+            fm.format("%n    static final int C2BSIZE = 0x%x;%n",
+                      initC2BIndex(indexC2B));
+            fm.format("%n    static char[] c2bIndex = new char[] {%n");
+            toCharArray(fm, indexC2B);
+            fm.format("    };%n");
+
+            // c2bIndexSupp
+            fm.format("%n    static final int C2BSUPPSIZE = 0x%x;%n",
+                      initC2BIndex(indexC2BSupp));
+            fm.format("%n    static char[] c2bSuppIndex = new char[] {%n");
+            toCharArray(fm, indexC2BSupp);
+            fm.format("    };%n");
+
+            // suppFlags
+            fm.format("%n    static String b2cIsSuppStr =%n");
+            toString(fm, toCharArray(suppFlag,
+                                     b1Min, b1Max, b2Min, b2Max),
+                     ";");
+
+            fm.format("}");
+            fm.close();
+
+            ps.println(out.toString());
+            ps.close();
+        } catch (Exception x) {
+            x.printStackTrace();
+        }
+    }
+}

make/tools/src/build/tools/charsetmapping/Main.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+package build.tools.charsetmapping;
+
+import java.io.*;
+import java.util.Scanner;
+
+public class Main {
+
+    public static void main(String args[]) throws Exception {
+        if (args.length < 3 ) {
+            System.out.println("Usage: java -jar charsetmapping.jar src dst mType...");
+            System.exit(1);
+        }
+        if ("sbcs".equals(args[2]) || "extsbcs".equals(args[2])) {
+            GenerateSBCS.genSBCS(args);
+        } else if ("euctw".equals(args[2])) {
+            if (args.length != 4) {
+                System.out.println("Usage: java -jar charsetmapping.jar srcDir dstDir euctw copyrightSrc");
+                System.exit(1);
+            }
+            GenerateEUC_TW.genEUC_TW(args);
+        } else if ("sjis0213".equals(args[2])) {
+            GenerateMapping.genMapping(args);
+        }
+    }
+
+}

test/sun/nio/cs/TestEUC_TW.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 6831794 6229811
+ * @summary Test EUC_TW charset
+ */
+
+import java.nio.charset.*;
+import java.nio.*;
+import java.util.*;
+
+public class TestEUC_TW {
+
+    static class Time {
+        long t;
+    }
+    static int iteration = 1000;
+
+    static char[] decode(byte[] bb, Charset cs, boolean testDirect, Time t)
+        throws Exception {
+        String csn = cs.name();
+        CharsetDecoder dec = cs.newDecoder();
+        ByteBuffer bbf;
+        CharBuffer cbf;
+        if (testDirect) {
+            bbf = ByteBuffer.allocateDirect(bb.length);
+            cbf = ByteBuffer.allocateDirect(bb.length*2).asCharBuffer();
+            bbf.put(bb);
+        } else {
+            bbf = ByteBuffer.wrap(bb);
+            cbf = CharBuffer.allocate(bb.length);
+        }
+        CoderResult cr = null;
+        long t1 = System.nanoTime()/1000;
+        for (int i = 0; i < iteration; i++) {
+            bbf.rewind();
+            cbf.clear();
+            dec.reset();
+            cr = dec.decode(bbf, cbf, true);
+        }
+        long t2 = System.nanoTime()/1000;
+        if (t != null)
+        t.t = (t2 - t1)/iteration;
+        if (cr != CoderResult.UNDERFLOW) {
+            System.out.println("DEC-----------------");
+            int pos = bbf.position();
+            System.out.printf("  cr=%s, bbf.pos=%d, bb[pos]=%x,%x,%x,%x%n",
+                              cr.toString(), pos,
+                              bb[pos++]&0xff, bb[pos++]&0xff,bb[pos++]&0xff, bb[pos++]&0xff);
+            throw new RuntimeException("Decoding err: " + csn);
+        }
+        char[] cc = new char[cbf.position()];
+        cbf.flip(); cbf.get(cc);
+        return cc;
+
+    }
+
+    static CoderResult decodeCR(byte[] bb, Charset cs, boolean testDirect)
+        throws Exception {
+        CharsetDecoder dec = cs.newDecoder();
+        ByteBuffer bbf;
+        CharBuffer cbf;
+        if (testDirect) {
+            bbf = ByteBuffer.allocateDirect(bb.length);
+            cbf = ByteBuffer.allocateDirect(bb.length*2).asCharBuffer();
+            bbf.put(bb).flip();
+        } else {
+            bbf = ByteBuffer.wrap(bb);
+            cbf = CharBuffer.allocate(bb.length);
+        }
+        return dec.decode(bbf, cbf, true);
+    }
+
+    static byte[] encode(char[] cc, Charset cs, boolean testDirect, Time t)
+        throws Exception {
+        ByteBuffer bbf;
+        CharBuffer cbf;
+        CharsetEncoder enc = cs.newEncoder();
+        String csn = cs.name();
+        if (testDirect) {
+            bbf = ByteBuffer.allocateDirect(cc.length * 4);
+            cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();
+            cbf.put(cc).flip();
+        } else {
+            bbf = ByteBuffer.allocate(cc.length * 4);
+            cbf = CharBuffer.wrap(cc);
+        }
+        CoderResult cr = null;
+        long t1 = System.nanoTime()/1000;
+        for (int i = 0; i < iteration; i++) {
+            cbf.rewind();
+            bbf.clear();
+            enc.reset();
+            cr = enc.encode(cbf, bbf, true);
+        }
+        long t2 = System.nanoTime()/1000;
+        if (t != null)
+        t.t = (t2 - t1)/iteration;
+        if (cr != CoderResult.UNDERFLOW) {
+            System.out.println("ENC-----------------");
+            int pos = cbf.position();
+            System.out.printf("  cr=%s, cbf.pos=%d, cc[pos]=%x%n",
+                              cr.toString(), pos, cc[pos]&0xffff);
+            throw new RuntimeException("Encoding err: " + csn);
+        }
+        byte[] bb = new byte[bbf.position()];
+        bbf.flip(); bbf.get(bb);
+        return bb;
+    }
+
+    static CoderResult encodeCR(char[] cc, Charset cs, boolean testDirect)
+        throws Exception {
+        ByteBuffer bbf;
+        CharBuffer cbf;
+        CharsetEncoder enc = cs.newEncoder();
+        if (testDirect) {
+            bbf = ByteBuffer.allocateDirect(cc.length * 4);
+            cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();
+            cbf.put(cc).flip();
+        } else {
+            bbf = ByteBuffer.allocate(cc.length * 4);
+            cbf = CharBuffer.wrap(cc);
+        }
+        return enc.encode(cbf, bbf, true);
+    }
+
+    static char[] getEUC_TWChars(boolean skipNR) {
+        //CharsetEncoder encOLD = Charset.forName("EUC_TW_OLD").newEncoder();
+        CharsetEncoder encOLD = new EUC_TW_OLD().newEncoder();
+        CharsetEncoder enc = Charset.forName("EUC_TW").newEncoder();
+        char[] cc = new char[0x20000];
+        char[] c2 = new char[2];
+        int pos = 0;
+        int i = 0;
+        //bmp
+        for (i = 0; i < 0x10000; i++) {
+            //SKIP these 3 NR codepoints if compared to EUC_TW
+            if (skipNR && (i == 0x4ea0 || i == 0x51ab || i == 0x52f9))
+                continue;
+            if (encOLD.canEncode((char)i) != enc.canEncode((char)i)) {
+                System.out.printf("  Err i=%x:  old=%b new=%b%n", i,
+                                  encOLD.canEncode((char)i),
+                                  enc.canEncode((char)i));
+                throw new RuntimeException("canEncode() err!");
+            }
+
+            if (enc.canEncode((char)i)) {
+                cc[pos++] = (char)i;
+            }
+        }
+
+        //supp
+        CharBuffer cb = CharBuffer.wrap(new char[2]);
+        for (i = 0x20000; i < 0x30000; i++) {
+            Character.toChars(i, c2, 0);
+            cb.clear();cb.put(c2[0]);cb.put(c2[1]);cb.flip();
+
+            if (encOLD.canEncode(cb) != enc.canEncode(cb)) {
+                throw new RuntimeException("canEncode() err!");
+            }
+
+            if (enc.canEncode(cb)) {
+                //System.out.printf("cp=%x,  (%x, %x) %n", i, c2[0] & 0xffff, c2[1] & 0xffff);
+                cc[pos++] = c2[0];
+                cc[pos++] = c2[1];
+            }
+        }
+
+        return Arrays.copyOf(cc, pos);
+    }
+
+    static void checkRoundtrip(Charset cs) throws Exception {
+        char[] cc = getEUC_TWChars(false);
+        System.out.printf("Check roundtrip <%s>...", cs.name());
+        byte[] bb = encode(cc, cs, false, null);
+        char[] ccO = decode(bb, cs, false, null);
+
+        if (!Arrays.equals(cc, ccO)) {
+            System.out.printf("    non-direct failed");
+        }
+        bb = encode(cc, cs, true, null);
+        ccO = decode(bb, cs, true, null);
+        if (!Arrays.equals(cc, ccO)) {
+            System.out.printf("    (direct) failed");
+        }
+        System.out.println();
+    }
+
+    static void checkInit(String csn) throws Exception {
+        System.out.printf("Check init <%s>...%n", csn);
+        Charset.forName("Big5");    // load in the ExtendedCharsets
+        long t1 = System.nanoTime()/1000;
+        Charset cs = Charset.forName(csn);
+        long t2 = System.nanoTime()/1000;
+        System.out.printf("    charset     :%d%n", t2 - t1);
+        t1 = System.nanoTime()/1000;
+            cs.newDecoder();
+        t2 = System.nanoTime()/1000;
+        System.out.printf("    new Decoder :%d%n", t2 - t1);
+
+        t1 = System.nanoTime()/1000;
+            cs.newEncoder();
+        t2 = System.nanoTime()/1000;
+        System.out.printf("    new Encoder :%d%n", t2 - t1);
+    }
+
+    static void compare(Charset cs1, Charset cs2) throws Exception {
+        char[] cc = getEUC_TWChars(true);
+
+        String csn1 = cs1.name();
+        String csn2 = cs2.name();
+        System.out.printf("Diff     <%s> <%s>...%n", csn1, csn2);
+
+        Time t1 = new Time();
+        Time t2 = new Time();
+
+        byte[] bb1 = encode(cc, cs1, false, t1);
+        byte[] bb2 = encode(cc, cs2, false, t2);
+
+        System.out.printf("    Encoding TimeRatio %s/%s: %d,%d :%f%n",
+                          csn2, csn1,
+                          t2.t, t1.t,
+                          (double)(t2.t)/(t1.t));
+        if (!Arrays.equals(bb1, bb2)) {
+            System.out.printf("        encoding failed%n");
+        }
+
+        char[] cc2 = decode(bb1, cs2, false, t2);
+        char[] cc1 = decode(bb1, cs1, false, t1);
+        System.out.printf("    Decoding TimeRatio %s/%s: %d,%d :%f%n",
+                          csn2, csn1,
+                          t2.t, t1.t,
+                          (double)(t2.t)/(t1.t));
+        if (!Arrays.equals(cc1, cc2)) {
+            System.out.printf("        decoding failed%n");
+        }
+
+        bb1 = encode(cc, cs1, true, t1);
+        bb2 = encode(cc, cs2, true, t2);
+
+        System.out.printf("    Encoding(dir) TimeRatio %s/%s: %d,%d :%f%n",
+                          csn2, csn1,
+                          t2.t, t1.t,
+                          (double)(t2.t)/(t1.t));
+
+        if (!Arrays.equals(bb1, bb2))
+            System.out.printf("        encoding (direct) failed%n");
+
+        cc1 = decode(bb1, cs1, true, t1);
+        cc2 = decode(bb1, cs2, true, t2);
+        System.out.printf("    Decoding(dir) TimeRatio %s/%s: %d,%d :%f%n",
+                          csn2, csn1,
+                          t2.t, t1.t,
+                          (double)(t2.t)/(t1.t));
+        if (!Arrays.equals(cc1, cc2)) {
+            System.out.printf("        decoding (direct) failed%n");
+        }
+    }
+
+    // The first byte is the length of malformed bytes
+    static byte[][] malformed = {
+        //{5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0x80, (byte)0xC0 },
+    };
+
+    static void checkMalformed(Charset cs) throws Exception {
+        boolean failed = false;
+        String csn = cs.name();
+        System.out.printf("Check malformed <%s>...%n", csn);
+        for (boolean direct: new boolean[] {false, true}) {
+            for (byte[] bins : malformed) {
+                int mlen = bins[0];
+                byte[] bin = Arrays.copyOfRange(bins, 1, bins.length);
+                CoderResult cr = decodeCR(bin, cs, direct);
+                String ashex = "";
+                for (int i = 0; i < bin.length; i++) {
+                    if (i > 0) ashex += " ";
+                        ashex += Integer.toBinaryString((int)bin[i] & 0xff);
+                }
+                if (!cr.isMalformed()) {
+                    System.out.printf("        FAIL(direct=%b): [%s] not malformed.\n", direct, ashex);
+                    failed = true;
+                } else if (cr.length() != mlen) {
+                    System.out.printf("        FAIL(direct=%b): [%s] malformed[len=%d].\n", direct, ashex, cr.length());
+                    failed = true;
+                }
+            }
+        }
+        if (failed)
+            throw new RuntimeException("Check malformed failed " + csn);
+    }
+
+    static boolean check(CharsetDecoder dec, byte[] bytes, boolean direct, int[] flow) {
+        int inPos = flow[0];
+        int inLen = flow[1];
+        int outPos = flow[2];
+        int outLen = flow[3];
+        int expedInPos = flow[4];
+        int expedOutPos = flow[5];
+        CoderResult expedCR = (flow[6]==0)?CoderResult.UNDERFLOW
+                                          :CoderResult.OVERFLOW;
+        ByteBuffer bbf;
+        CharBuffer cbf;
+        if (direct) {
+            bbf = ByteBuffer.allocateDirect(inPos + bytes.length);
+            cbf = ByteBuffer.allocateDirect((outPos + outLen)*2).asCharBuffer();
+        } else {
+            bbf = ByteBuffer.allocate(inPos + bytes.length);
+            cbf = CharBuffer.allocate(outPos + outLen);
+        }
+        bbf.position(inPos);
+        bbf.put(bytes).flip().position(inPos).limit(inPos + inLen);
+        cbf.position(outPos);
+        dec.reset();
+        CoderResult cr = dec.decode(bbf, cbf, false);
+        if (cr != expedCR ||
+            bbf.position() != expedInPos ||
+            cbf.position() != expedOutPos) {
+            System.out.printf("Expected(direct=%5b): [", direct);
+            for (int i:flow) System.out.print(" " + i);
+            System.out.println("]  CR=" + cr +
+                               ", inPos=" + bbf.position() +
+                               ", outPos=" + cbf.position());
+            return false;
+        }
+        return true;
+    }
+
+    static void checkUnderOverflow(Charset cs) throws Exception {
+        String csn = cs.name();
+        System.out.printf("Check under/overflow <%s>...%n", csn);
+        CharsetDecoder dec = cs.newDecoder();
+        boolean failed = false;
+        //7f, a1a1, 8ea2a1a1, 8ea3a1a1, 8ea7a1a1
+        //0   1 2   3         7         11
+        byte[] bytes = new String("\u007f\u3000\u4e42\u4e28\ud840\udc55").getBytes("EUC_TW");
+        int    inlen = bytes.length;
+
+        int MAXOFF = 20;
+        for (int inoff = 0; inoff < MAXOFF; inoff++) {
+            for (int outoff = 0; outoff < MAXOFF; outoff++) {
+        int[][] Flows = {
+            //inpos, inLen, outPos,  outLen, inPosEP,    outposEP,   under(0)/over(1)
+            //overflow
+            {inoff,  inlen, outoff,  1,      inoff + 1,  outoff + 1, 1},
+            {inoff,  inlen, outoff,  2,      inoff + 3,  outoff + 2, 1},
+            {inoff,  inlen, outoff,  3,      inoff + 7,  outoff + 3, 1},
+            {inoff,  inlen, outoff,  4,      inoff + 11, outoff + 4, 1},
+            {inoff,  inlen, outoff,  5,      inoff + 11, outoff + 4, 1},
+            {inoff,  inlen, outoff,  6,      inoff + 15, outoff + 6, 0},
+            //underflow
+            {inoff,  1,     outoff,  6,      inoff + 1,  outoff + 1, 0},
+            {inoff,  2,     outoff,  6,      inoff + 1,  outoff + 1, 0},
+            {inoff,  3,     outoff,  6,      inoff + 3,  outoff + 2, 0},
+            {inoff,  4,     outoff,  6,      inoff + 3,  outoff + 2, 0},
+            {inoff,  5,     outoff,  6,      inoff + 3,  outoff + 2, 0},
+            {inoff,  8,     outoff,  6,      inoff + 7,  outoff + 3, 0},
+            {inoff,  9,     outoff,  6,      inoff + 7,  outoff + 3, 0},
+            {inoff, 10,     outoff,  6,      inoff + 7,  outoff + 3, 0},
+            {inoff, 11,     outoff,  6,      inoff +11,  outoff + 4, 0},
+            {inoff, 12,     outoff,  6,      inoff +11,  outoff + 4, 0},
+            {inoff, 15,     outoff,  6,      inoff +15,  outoff + 6, 0},
+            // 2-byte under/overflow
+            {inoff,  2,     outoff,  1,      inoff + 1,  outoff + 1, 0},
+            {inoff,  3,     outoff,  1,      inoff + 1,  outoff + 1, 1},
+            {inoff,  3,     outoff,  2,      inoff + 3,  outoff + 2, 0},
+            // 4-byte  under/overflow
+            {inoff,  4,     outoff,  2,      inoff + 3,  outoff + 2, 0},
+            {inoff,  5,     outoff,  2,      inoff + 3,  outoff + 2, 0},
+            {inoff,  6,     outoff,  2,      inoff + 3,  outoff + 2, 0},
+            {inoff,  7,     outoff,  2,      inoff + 3,  outoff + 2, 1},
+            {inoff,  7,     outoff,  3,      inoff + 7,  outoff + 3, 0},
+            // 4-byte  under/overflow
+            {inoff,  8,     outoff,  3,      inoff + 7,  outoff + 3, 0},
+            {inoff,  9,     outoff,  3,      inoff + 7,  outoff + 3, 0},
+            {inoff, 10,     outoff,  3,      inoff + 7,  outoff + 3, 0},
+            {inoff, 11,     outoff,  3,      inoff + 7,  outoff + 3, 1},
+            {inoff, 11,     outoff,  4,      inoff +11,  outoff + 4, 0},
+            // 4-byte/supp  under/overflow
+            {inoff, 11,     outoff,  4,      inoff +11,  outoff + 4, 0},
+            {inoff, 12,     outoff,  4,      inoff +11,  outoff + 4, 0},
+            {inoff, 13,     outoff,  4,      inoff +11,  outoff + 4, 0},
+            {inoff, 14,     outoff,  4,      inoff +11,  outoff + 4, 0},
+            {inoff, 15,     outoff,  4,      inoff +11,  outoff + 4, 1},
+            {inoff, 15,     outoff,  5,      inoff +11,  outoff + 4, 1},
+            {inoff, 15,     outoff,  6,      inoff +15,  outoff + 6, 0},
+        };
+        for (boolean direct: new boolean[] {false, true}) {
+            for (int[] flow: Flows) {
+                if (!check(dec, bytes, direct, flow))
+                    failed = true;
+            }
+        }}}
+        if (failed)
+            throw new RuntimeException("Check under/overflow failed " + csn);
+    }
+
+    public static void main(String[] args) throws Exception {
+        // be the first one
+        //checkInit("EUC_TW_OLD");
+        checkInit("EUC_TW");
+        Charset euctw = Charset.forName("EUC_TW");
+        checkRoundtrip(euctw);
+        compare(euctw, new EUC_TW_OLD());
+        checkMalformed(euctw);
+        checkUnderOverflow(euctw);
+    }
+}

test/sun/nio/cs/TestX11CNS.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 6831794
+ * @summary Test X11CNS charset
+ */
+
+import java.nio.charset.*;
+import java.nio.*;
+import java.util.*;
+
+public class TestX11CNS {
+    static char[] decode(byte[] bb, Charset cs)
+        throws Exception {
+        CharsetDecoder dec = cs.newDecoder();
+        ByteBuffer bbf = ByteBuffer.wrap(bb);
+        CharBuffer cbf = CharBuffer.allocate(bb.length);
+        CoderResult cr = dec.decode(bbf, cbf, true);
+        if (cr != CoderResult.UNDERFLOW) {
+            System.out.println("DEC-----------------");
+            int pos = bbf.position();
+            System.out.printf("  cr=%s, bbf.pos=%d, bb[pos]=%x,%x,%x,%x%n",
+                              cr.toString(), pos,
+                              bb[pos++]&0xff, bb[pos++]&0xff,bb[pos++]&0xff, bb[pos++]&0xff);
+            throw new RuntimeException("Decoding err: " + cs.name());
+        }
+        char[] cc = new char[cbf.position()];
+        cbf.flip(); cbf.get(cc);
+        return cc;
+
+    }
+
+    static byte[] encode(char[] cc, Charset cs)
+        throws Exception {
+        ByteBuffer bbf = ByteBuffer.allocate(cc.length * 4);
+        CharBuffer cbf = CharBuffer.wrap(cc);
+        CharsetEncoder enc = cs.newEncoder();
+
+        CoderResult cr = enc.encode(cbf, bbf, true);
+        if (cr != CoderResult.UNDERFLOW) {
+            System.out.println("ENC-----------------");
+            int pos = cbf.position();
+            System.out.printf("  cr=%s, cbf.pos=%d, cc[pos]=%x%n",
+                              cr.toString(), pos, cc[pos]&0xffff);
+            throw new RuntimeException("Encoding err: " + cs.name());
+        }
+        byte[] bb = new byte[bbf.position()];
+        bbf.flip(); bbf.get(bb);
+        return bb;
+    }
+
+    static char[] getChars(Charset newCS, Charset oldCS) {
+        CharsetEncoder enc = oldCS.newEncoder();
+        CharsetEncoder encNew = newCS.newEncoder();
+        char[] cc = new char[0x10000];
+        int pos = 0;
+        int i = 0;
+        while (i < 0x10000) {
+            if (i == 0x4ea0 || i == 0x51ab || i == 0x52f9) {
+                i++;continue;
+            }
+            if (enc.canEncode((char)i) != encNew.canEncode((char)i)) {
+                System.out.printf("  Err i=%x%n", i);
+                //throw new RuntimeException("canEncode() err!");
+            }
+            if (enc.canEncode((char)i)) {
+                cc[pos++] = (char)i;
+            }
+            i++;
+        }
+        return Arrays.copyOf(cc, pos);
+    }
+
+    static void compare(Charset newCS, Charset oldCS) throws Exception {
+        char[] cc = getChars(newCS, oldCS);
+        System.out.printf("    Diff <%s> <%s>...%n", newCS.name(), oldCS.name());
+
+        byte[] bb1 = encode(cc, newCS);
+        byte[] bb2 = encode(cc, oldCS);
+
+        if (!Arrays.equals(bb1, bb2)) {
+            System.out.printf("        encoding failed!%n");
+        }
+        char[] cc1 = decode(bb1, newCS);
+        char[] cc2 = decode(bb1, oldCS);
+        if (!Arrays.equals(cc1, cc2)) {
+            for (int i = 0; i < cc1.length; i++) {
+                if (cc1[i] != cc2[i]) {
+                    System.out.printf("i=%d, cc1=%x cc2=%x,  bb=<%x%x>%n",
+                                      i,
+                                      cc1[i]&0xffff, cc2[i]&0xffff,
+                                      bb1[i*2]&0xff, bb1[i*2+1]&0xff);
+                }
+
+            }
+
+            System.out.printf("        decoding failed%n");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        compare(new sun.awt.motif.X11CNS11643P1(),
+                new X11CNS11643P1());
+
+        compare(new sun.awt.motif.X11CNS11643P2(),
+                new X11CNS11643P2());
+
+        compare(new sun.awt.motif.X11CNS11643P3(),
+                new X11CNS11643P3());
+
+    }
+}

test/sun/nio/cs/X11CNS11643.java

+/*
+ * Copyright 2001-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+import java.nio.CharBuffer;
+import java.nio.ByteBuffer;
+import java.nio.charset.*;
+import sun.nio.cs.ext.EUC_TW;
+
+public abstract class X11CNS11643 extends Charset {
+    private final int plane;
+    public X11CNS11643 (int plane, String name) {
+        super(name, null);
+        switch (plane) {
+        case 1:
+            this.plane = 0; // CS1
+            break;
+        case 2:
+        case 3:
+            this.plane = plane;
+            break;
+        default:
+            throw new IllegalArgumentException
+                ("Only planes 1, 2, and 3 supported");
+        }
+    }
+
+    public CharsetEncoder newEncoder() {
+        return new Encoder(this, plane);
+    }
+
+    public CharsetDecoder newDecoder() {
+        return new Decoder(this, plane);
+    }
+
+    public boolean contains(Charset cs) {
+        return cs instanceof X11CNS11643;
+    }
+
+    private class Encoder extends EUC_TW_OLD.Encoder {
+        private int plane;
+        public Encoder(Charset cs, int plane) {
+            super(cs);
+            this.plane = plane;
+        }
+        public boolean canEncode(char c) {
+            if (c <= 0x7F) {
+                return false;
+            }
+            int p = getNative(c) >> 16;
+            if (p == 1 && plane == 0 ||
+                p == 2 && plane == 2 ||
+                p == 3 && plane == 3)
+                return true;
+            return false;
+        }
+
+        public boolean isLegalReplacement(byte[] repl) {
+            return true;
+        }
+
+        protected CoderResult encodeLoop(CharBuffer src, ByteBuffer dst) {
+            char[] sa = src.array();
+            int sp = src.arrayOffset() + src.position();
+            int sl = src.arrayOffset() + src.limit();
+            byte[] da = dst.array();
+            int dp = dst.arrayOffset() + dst.position();
+            int dl = dst.arrayOffset() + dst.limit();
+
+            try {
+                while (sp < sl) {
+                    char c = sa[sp];
+                    if (c >= '\uFFFE' || c <= '\u007f')
+                        return CoderResult.unmappableForLength(1);
+                    int cns = getNative(c);
+                    int p = cns >> 16;
+                    if (p == 1 && plane == 0 ||
+                        p == 2 && plane == 2 ||
+                        p == 3 && plane == 3) {
+                        if (dl - dp < 2)
+                            return CoderResult.OVERFLOW;
+                        da[dp++] = (byte) ((cns  >> 8) & 0x7f);
+                        da[dp++] = (byte) (cns & 0x7f);
+                        sp++;
+                        continue;
+                    }
+                    return CoderResult.unmappableForLength(1);
+                }
+                return CoderResult.UNDERFLOW;
+            } finally {
+                src.position(sp - src.arrayOffset());
+                dst.position(dp - dst.arrayOffset());
+            }
+        }
+    }
+
+    private class Decoder extends EUC_TW_OLD.Decoder {
+        private String table;
+        protected Decoder(Charset cs, int plane) {
+            super(cs);
+            switch (plane) {
+            case 0:
+                table = unicodeCNS1;
+                break;
+            case 2:
+                table = unicodeCNS2;
+                break;
+            case 3:
+                table = unicodeCNS3;
+                break;
+            default:
+                throw new IllegalArgumentException
+                    ("Only planes 1, 2, and 3 supported");
+            }
+        }
+
+        //we only work on array backed buffer.
+        protected CoderResult decodeLoop(ByteBuffer src, CharBuffer dst) {
+            byte[] sa = src.array();
+            int sp = src.arrayOffset() + src.position();
+            int sl = src.arrayOffset() + src.limit();
+            assert (sp <= sl);
+            sp = (sp <= sl ? sp : sl);
+
+            char[] da = dst.array();
+            int dp = dst.arrayOffset() + dst.position();
+            int dl = dst.arrayOffset() + dst.limit();
+            assert (dp <= dl);
+            dp = (dp <= dl ? dp : dl);
+
+            try {
+                while (sp < sl) {
+                    if ( sl - sp < 2) {
+                        return CoderResult.UNDERFLOW;
+                    }
+                    byte b1 = sa[sp];
+                    byte b2 = sa[sp + 1];
+                    char c = replacement().charAt(0);
+
+                    if (table == unicodeCNS3) {
+                        char[] cc = convToSurrogate((byte)(b1 | 0x80),
+                                                    (byte)(b2 | 0x80),
+                                                    table);
+                        if (cc != null && cc[0] == '\u0000')
+                            c = cc[1];
+                    } else {
+                        c = convToUnicode((byte)(b1 | 0x80),
+                                           (byte)(b2 | 0x80),
+                                           table);
+                    }
+                    if (c == replacement().charAt(0)
+                        //to keep the compatibility with b2cX11CNS11643
+                        /*|| c == '\u0000'*/) {
+                        return CoderResult.unmappableForLength(2);
+                    }
+                    if (dl - dp < 1)
+                        return CoderResult.OVERFLOW;
+                    da[dp++] = c;
+                    sp +=2;
+                }
+                return CoderResult.UNDERFLOW;
+            } finally {
+                src.position(sp - src.arrayOffset());
+                dst.position(dp - dst.arrayOffset());
+            }
+        }
+    }
+}

test/sun/nio/cs/X11CNS11643P1.java

+/*
+ * Copyright 1996-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+public class X11CNS11643P1 extends X11CNS11643 {
+    public X11CNS11643P1() {
+        super(1, "X11CNS11643P1");
+    }
+}

test/sun/nio/cs/X11CNS11643P2.java

+/*
+ * Copyright 1996-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+public class X11CNS11643P2 extends X11CNS11643 {
+    public X11CNS11643P2() {
+        super(2, "X11CNS11643P2");
+    }
+}

test/sun/nio/cs/X11CNS11643P3.java

+/*
+ * Copyright 1997-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+public class X11CNS11643P3 extends X11CNS11643 {
+    public X11CNS11643P3() {
+        super(3, "X11CNS11643P3");
+    }
+}