7144981: javac should ignore ignorable characters in input

Reviewed-by: jjg, mcimadamore

src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

@@ -348,8 +348,8 @@ public class JavaTokenizer {
     private void scanIdent() {
         boolean isJavaIdentifierPart;
         char high;
+        reader.putChar(true);
         do {
-            reader.putChar(true);
             switch (reader.ch) {
             case 'A': case 'B': case 'C': case 'D': case 'E':
             case 'F': case 'G': case 'H': case 'I': case 'J':
@@ -366,6 +366,7 @@ public class JavaTokenizer {
             case '$': case '_':
             case '0': case '1': case '2': case '3': case '4':
             case '5': case '6': case '7': case '8': case '9':
+                break;
             case '\u0000': case '\u0001': case '\u0002': case '\u0003':
             case '\u0004': case '\u0005': case '\u0006': case '\u0007':
             case '\u0008': case '\u000E': case '\u000F': case '\u0010':
@@ -373,26 +374,33 @@ public class JavaTokenizer {
             case '\u0015': case '\u0016': case '\u0017':
             case '\u0018': case '\u0019': case '\u001B':
             case '\u007F':
-                break;
+                reader.scanChar();
+                continue;
             case '\u001A': // EOI is also a legal identifier part
                 if (reader.bp >= reader.buflen) {
                     name = reader.name();
                     tk = tokens.lookupKind(name);
                     return;
                 }
-                break;
+                reader.scanChar();
+                continue;
             default:
                 if (reader.ch < '\u0080') {
                     // all ASCII range chars already handled, above
                     isJavaIdentifierPart = false;
                 } else {
-                    high = reader.scanSurrogates();
-                    if (high != 0) {
-                        reader.putChar(high);
-                        isJavaIdentifierPart = Character.isJavaIdentifierPart(
-                            Character.toCodePoint(high, reader.ch));
+                    if (Character.isIdentifierIgnorable(reader.ch)) {
+                        reader.scanChar();
+                        continue;
                     } else {
-                        isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+                        high = reader.scanSurrogates();
+                        if (high != 0) {
+                            reader.putChar(high);
+                            isJavaIdentifierPart = Character.isJavaIdentifierPart(
+                                Character.toCodePoint(high, reader.ch));
+                        } else {
+                            isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+                        }
                     }
                 }
                 if (!isJavaIdentifierPart) {
@@ -401,6 +409,7 @@ public class JavaTokenizer {
                     return;
                 }
             }
+            reader.putChar(true);
         } while (true);
     }
 

test/tools/javac/7144981/IgnoreIgnorableCharactersInInput.java

+
+/*
+ * @test  /nodynamiccopyright/
+ * @bug 7144981
+ * @summary javac should ignore ignorable characters in input
+ * @run main IgnoreIgnorableCharactersInInput
+ */
+
+import com.sun.source.util.JavacTask;
+import java.io.File;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.ToolProvider;
+
+public class IgnoreIgnorableCharactersInInput {
+
+    public static void main(String... args) throws Exception {
+        new IgnoreIgnorableCharactersInInput().run();
+    }
+
+    void run() throws Exception {
+        JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
+        File classesDir = new File(System.getProperty("user.dir"), "classes");
+        classesDir.mkdirs();
+        JavaSource[] sources = new JavaSource[]{
+            new JavaSource("TestOneIgnorableChar", "AA\\u0000BB"),
+            new JavaSource("TestMultipleIgnorableChar", "AA\\u0000\\u0000\\u0000BB")};
+        JavacTask ct = (JavacTask)comp.getTask(null, null, null,
+                Arrays.asList("-d", classesDir.getPath()),
+                null, Arrays.asList(sources));
+        try {
+            if (!ct.call()) {
+                throw new AssertionError("Error thrown when compiling test cases");
+            }
+        } catch (Throwable ex) {
+            throw new AssertionError("Error thrown when compiling test cases");
+        }
+        check(classesDir,
+                "TestOneIgnorableChar.class",
+                "TestOneIgnorableChar$AABB.class",
+                "TestMultipleIgnorableChar.class",
+                "TestMultipleIgnorableChar$AABB.class");
+        if (errors > 0)
+            throw new AssertionError("There are some errors in the test check the error output");
+    }
+
+    /**
+     *  Check that a directory contains the expected files.
+     */
+    void check(File dir, String... paths) {
+        Set<String> found = new TreeSet<String>(Arrays.asList(dir.list()));
+        Set<String> expect = new TreeSet<String>(Arrays.asList(paths));
+        if (found.equals(expect))
+            return;
+        for (String f: found) {
+            if (!expect.contains(f))
+                error("Unexpected file found: " + f);
+        }
+        for (String e: expect) {
+            if (!found.contains(e))
+                error("Expected file not found: " + e);
+        }
+    }
+
+    int errors;
+
+    void error(String msg) {
+        System.err.println(msg);
+        errors++;
+    }
+
+    class JavaSource extends SimpleJavaFileObject {
+
+        String internalSource =
+            "public class #O {public class #I {} }";
+        public JavaSource(String outerClassName, String innerClassName) {
+            super(URI.create(outerClassName + ".java"), JavaFileObject.Kind.SOURCE);
+            internalSource =
+                    internalSource.replace("#O", outerClassName).replace("#I", innerClassName);
+        }
+
+        @Override
+        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+            return internalSource;
+        }
+    }
+}