7096014: Javac tokens should retain state

Summary: Refactor javac tokens from enum constants to stateful instances (to keep track of position, comments, etc.)
Reviewed-by: jjg

src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java

@@ -42,7 +42,6 @@ import com.sun.tools.javac.tree.JCTree.*;
 import com.sun.tools.apt.comp.*;
 import com.sun.tools.apt.util.Bark;
 import com.sun.mirror.apt.AnnotationProcessorFactory;
-import com.sun.tools.javac.parser.DocCommentScanner;
 
 /**
  *  <p><b>This is NOT part of any supported API.

src/share/classes/com/sun/tools/javac/parser/EndPosParser.java

@@ -67,14 +67,14 @@ public class EndPosParser extends JavacParser {
     /** {@inheritDoc} */
     @Override
     protected <T extends JCTree> T to(T t) {
-        storeEnd(t, S.endPos());
+        storeEnd(t, token.endPos);
         return t;
     }
 
     /** {@inheritDoc} */
     @Override
     protected <T extends JCTree> T toP(T t) {
-        storeEnd(t, S.prevEndPos());
+        storeEnd(t, S.prevToken().endPos);
         return t;
     }
 
@@ -88,7 +88,7 @@ public class EndPosParser extends JavacParser {
     /** {@inheritDoc} */
     @Override
     JCExpression parExpression() {
-        int pos = S.pos();
+        int pos = token.pos;
         JCExpression t = super.parExpression();
         return toP(F.at(pos).Parens(t));
     }

src/share/classes/com/sun/tools/javac/parser/DocCommentScanner.java → src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java

 /*
- * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,9 +25,12 @@
 
 package com.sun.tools.javac.parser;
 
+import com.sun.tools.javac.file.JavacFileManager;
+import com.sun.tools.javac.parser.Tokens.Token;
+import com.sun.tools.javac.util.*;
+
 import java.nio.*;
 
-import com.sun.tools.javac.util.*;
 import static com.sun.tools.javac.util.LayoutCharacters.*;
 
 /** An extension to the base lexical analyzer that captures
@@ -40,26 +43,22 @@ import static com.sun.tools.javac.util.LayoutCharacters.*;
  *  This code and its internal interfaces are subject to change or
  *  deletion without notice.</b>
  */
-public class DocCommentScanner extends Scanner {
+public class JavadocTokenizer extends JavaTokenizer {
 
     /** Create a scanner from the input buffer.  buffer must implement
      *  array() and compact(), and remaining() must be less than limit().
      */
-    protected DocCommentScanner(ScannerFactory fac, CharBuffer buffer) {
+    protected JavadocTokenizer(ScannerFactory fac, CharBuffer buffer) {
         super(fac, buffer);
     }
 
     /** Create a scanner from the input array.  The array must have at
      *  least a single character of extra space.
      */
-    protected DocCommentScanner(ScannerFactory fac, char[] input, int inputLength) {
+    protected JavadocTokenizer(ScannerFactory fac, char[] input, int inputLength) {
         super(fac, input, inputLength);
     }
 
-    /** Starting position of the comment in original source
-     */
-    private int pos;
-
     /** The comment input buffer, index of next chacter to be read,
      *  index of one past last character in buffer.
      */
@@ -178,6 +177,14 @@ public class DocCommentScanner extends Scanner {
         }
     }
 
+    @Override
+    public Token readToken() {
+        docComment = null;
+        Token tk = super.readToken();
+        tk.docComment = docComment;
+        return tk;
+    }
+
     /**
      * Read next character in doc comment, skipping over double '\' characters.
      * If a double '\' is skipped, put in the buffer and update buffer count.
@@ -196,32 +203,17 @@ public class DocCommentScanner extends Scanner {
         }
     }
 
-    /* Reset doc comment before reading each new token
-     */
-    public void nextToken() {
-        docComment = null;
-        super.nextToken();
-    }
-
-    /**
-     * Returns the documentation string of the current token.
-     */
-    public String docComment() {
-        return docComment;
-    }
-
     /**
      * Process a doc comment and make the string content available.
      * Strips leading whitespace and stars.
      */
     @SuppressWarnings("fallthrough")
-    protected void processComment(CommentStyle style) {
+    protected void processComment(int pos, int endPos, CommentStyle style) {
         if (style != CommentStyle.JAVADOC) {
             return;
         }
 
-        pos = pos();
-        buf = getRawCharacters(pos, endPos());
+        buf = reader.getRawCharacters(pos, endPos);
         buflen = buf.length;
         bp = 0;
         col = 0;
@@ -414,7 +406,7 @@ public class DocCommentScanner extends Scanner {
      *
      * @return a LineMap */
     public Position.LineMap getLineMap() {
-        char[] buf = getRawCharacters();
+        char[] buf = reader.getRawCharacters();
         return Position.makeLineMap(buf, buf.length, true);
     }
 }

src/share/classes/com/sun/tools/javac/parser/Keywords.java

-/*
- * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import com.sun.tools.javac.util.Context;
-import com.sun.tools.javac.util.Log;
-import com.sun.tools.javac.util.Name;
-import com.sun.tools.javac.util.Names;
-
-import static com.sun.tools.javac.parser.Token.*;
-
-/**
- * Map from Name to Token and Token to String.
- *
- * <p><b>This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.</b>
- */
-public class Keywords {
-    public static final Context.Key<Keywords> keywordsKey =
-        new Context.Key<Keywords>();
-
-    public static Keywords instance(Context context) {
-        Keywords instance = context.get(keywordsKey);
-        if (instance == null)
-            instance = new Keywords(context);
-        return instance;
-    }
-
-    private final Names names;
-
-    protected Keywords(Context context) {
-        context.put(keywordsKey, this);
-        names = Names.instance(context);
-
-        for (Token t : Token.values()) {
-            if (t.name != null)
-                enterKeyword(t.name, t);
-            else
-                tokenName[t.ordinal()] = null;
-        }
-
-        key = new Token[maxKey+1];
-        for (int i = 0; i <= maxKey; i++) key[i] = IDENTIFIER;
-        for (Token t : Token.values()) {
-            if (t.name != null)
-                key[tokenName[t.ordinal()].getIndex()] = t;
-        }
-    }
-
-
-    public Token key(Name name) {
-        return (name.getIndex() > maxKey) ? IDENTIFIER : key[name.getIndex()];
-    }
-
-    /**
-     * Keyword array. Maps name indices to Token.
-     */
-    private final Token[] key;
-
-    /**  The number of the last entered keyword.
-     */
-    private int maxKey = 0;
-
-    /** The names of all tokens.
-     */
-    private Name[] tokenName = new Name[Token.values().length];
-
-    private void enterKeyword(String s, Token token) {
-        Name n = names.fromString(s);
-        tokenName[token.ordinal()] = n;
-        if (n.getIndex() > maxKey) maxKey = n.getIndex();
-    }
-}

src/share/classes/com/sun/tools/javac/parser/Lexer.java

@@ -25,7 +25,7 @@
 
 package com.sun.tools.javac.parser;
 
-import com.sun.tools.javac.util.*;
+import com.sun.tools.javac.parser.Tokens.*;
 import com.sun.tools.javac.util.Position.LineMap;
 
 /**
@@ -40,22 +40,26 @@ import com.sun.tools.javac.util.Position.LineMap;
 public interface Lexer {
 
     /**
-     * Has a @deprecated been encountered in last doc comment?
-     * This needs to be reset by client with resetDeprecatedFlag.
+     * Consume the next token.
      */
-    boolean deprecatedFlag();
+    void nextToken();
 
-    void resetDeprecatedFlag();
+    /**
+     * Return current token.
+     */
+    Token token();
 
     /**
-     * Returns the documentation string of the current token.
+     * Return the last character position of the previous token.
      */
-    String docComment();
+    Token prevToken();
 
     /**
-     * Return the last character position of the current token.
+     * Splits the current token in two and return the first (splitted) token.
+     * For instance '<<<' is splitted into two tokens '<' and '<<' respectively,
+     * and the latter is returned.
      */
-    int endPos();
+    Token split();
 
     /**
      * Return the position where a lexical error occurred;
@@ -74,69 +78,4 @@ public interface Lexer {
      * @return a LineMap
      */
     LineMap getLineMap();
-
-    /**
-     * Returns a copy of the input buffer, up to its inputLength.
-     * Unicode escape sequences are not translated.
-     */
-    char[] getRawCharacters();
-
-    /**
-     * Returns a copy of a character array subset of the input buffer.
-     * The returned array begins at the <code>beginIndex</code> and
-     * extends to the character at index <code>endIndex - 1</code>.
-     * Thus the length of the substring is <code>endIndex-beginIndex</code>.
-     * This behavior is like
-     * <code>String.substring(beginIndex, endIndex)</code>.
-     * Unicode escape sequences are not translated.
-     *
-     * @param beginIndex the beginning index, inclusive.
-     * @param endIndex the ending index, exclusive.
-     * @throws IndexOutOfBounds if either offset is outside of the
-     *         array bounds
-     */
-    char[] getRawCharacters(int beginIndex, int endIndex);
-
-    /**
-     * Return the name of an identifier or token for the current token.
-     */
-    Name name();
-
-    /**
-     * Read token.
-     */
-    void nextToken();
-
-    /**
-     * Return the current token's position: a 0-based
-     *  offset from beginning of the raw input stream
-     *  (before unicode translation)
-     */
-    int pos();
-
-    /**
-     * Return the last character position of the previous token.
-     */
-    int prevEndPos();
-
-    /**
-     * Return the radix of a numeric literal token.
-     */
-    int radix();
-
-    /**
-     * The value of a literal token, recorded as a string.
-     *  For integers, leading 0x and 'l' suffixes are suppressed.
-     */
-    String stringVal();
-
-    /**
-     * Return the current token, set by nextToken().
-     */
-    Token token();
-
-    /**
-     * Sets the current token.
-     */
-    void token(Token token);
 }

src/share/classes/com/sun/tools/javac/parser/ParserFactory.java

@@ -55,7 +55,7 @@ public class ParserFactory {
 
     final TreeMaker F;
     final Log log;
-    final Keywords keywords;
+    final Tokens tokens;
     final Source source;
     final Names names;
     final Options options;
@@ -67,7 +67,7 @@ public class ParserFactory {
         this.F = TreeMaker.instance(context);
         this.log = Log.instance(context);
         this.names = Names.instance(context);
-        this.keywords = Keywords.instance(context);
+        this.tokens = Tokens.instance(context);
         this.source = Source.instance(context);
         this.options = Options.instance(context);
         this.scannerFactory = ScannerFactory.instance(context);

src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java

@@ -57,7 +57,7 @@ public class ScannerFactory {
     final Log log;
     final Names names;
     final Source source;
-    final Keywords keywords;
+    final Tokens tokens;
 
     /** Create a new scanner factory. */
     protected ScannerFactory(Context context) {
@@ -65,14 +65,14 @@ public class ScannerFactory {
         this.log = Log.instance(context);
         this.names = Names.instance(context);
         this.source = Source.instance(context);
-        this.keywords = Keywords.instance(context);
+        this.tokens = Tokens.instance(context);
     }
 
     public Scanner newScanner(CharSequence input, boolean keepDocComments) {
         if (input instanceof CharBuffer) {
             CharBuffer buf = (CharBuffer) input;
             if (keepDocComments)
-                return new DocCommentScanner(this, buf);
+                return new Scanner(this, new JavadocTokenizer(this, buf));
             else
                 return new Scanner(this, buf);
         } else {
@@ -83,7 +83,7 @@ public class ScannerFactory {
 
     public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
         if (keepDocComments)
-            return new DocCommentScanner(this, input, inputLength);
+            return new Scanner(this, new JavadocTokenizer(this, input, inputLength));
         else
             return new Scanner(this, input, inputLength);
     }

src/share/classes/com/sun/tools/javac/parser/Token.java

-/*
- * Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import java.util.Locale;
-
-import com.sun.tools.javac.api.Formattable;
-import com.sun.tools.javac.api.Messages;
-
-/** An interface that defines codes for Java source tokens
- *  returned from lexical analysis.
- *
- *  <p><b>This is NOT part of any supported API.
- *  If you write code that depends on this, you do so at your own risk.
- *  This code and its internal interfaces are subject to change or
- *  deletion without notice.</b>
- */
-public enum Token implements Formattable {
-    EOF,
-    ERROR,
-    IDENTIFIER,
-    ABSTRACT("abstract"),
-    ASSERT("assert"),
-    BOOLEAN("boolean"),
-    BREAK("break"),
-    BYTE("byte"),
-    CASE("case"),
-    CATCH("catch"),
-    CHAR("char"),
-    CLASS("class"),
-    CONST("const"),
-    CONTINUE("continue"),
-    DEFAULT("default"),
-    DO("do"),
-    DOUBLE("double"),
-    ELSE("else"),
-    ENUM("enum"),
-    EXTENDS("extends"),
-    FINAL("final"),
-    FINALLY("finally"),
-    FLOAT("float"),
-    FOR("for"),
-    GOTO("goto"),
-    IF("if"),
-    IMPLEMENTS("implements"),
-    IMPORT("import"),
-    INSTANCEOF("instanceof"),
-    INT("int"),
-    INTERFACE("interface"),
-    LONG("long"),
-    NATIVE("native"),
-    NEW("new"),
-    PACKAGE("package"),
-    PRIVATE("private"),
-    PROTECTED("protected"),
-    PUBLIC("public"),
-    RETURN("return"),
-    SHORT("short"),
-    STATIC("static"),
-    STRICTFP("strictfp"),
-    SUPER("super"),
-    SWITCH("switch"),
-    SYNCHRONIZED("synchronized"),
-    THIS("this"),
-    THROW("throw"),
-    THROWS("throws"),
-    TRANSIENT("transient"),
-    TRY("try"),
-    VOID("void"),
-    VOLATILE("volatile"),
-    WHILE("while"),
-    INTLITERAL,
-    LONGLITERAL,
-    FLOATLITERAL,
-    DOUBLELITERAL,
-    CHARLITERAL,
-    STRINGLITERAL,
-    TRUE("true"),
-    FALSE("false"),
-    NULL("null"),
-    LPAREN("("),
-    RPAREN(")"),
-    LBRACE("{"),
-    RBRACE("}"),
-    LBRACKET("["),
-    RBRACKET("]"),
-    SEMI(";"),
-    COMMA(","),
-    DOT("."),
-    ELLIPSIS("..."),
-    EQ("="),
-    GT(">"),
-    LT("<"),
-    BANG("!"),
-    TILDE("~"),
-    QUES("?"),
-    COLON(":"),
-    EQEQ("=="),
-    LTEQ("<="),
-    GTEQ(">="),
-    BANGEQ("!="),
-    AMPAMP("&&"),
-    BARBAR("||"),
-    PLUSPLUS("++"),
-    SUBSUB("--"),
-    PLUS("+"),
-    SUB("-"),
-    STAR("*"),
-    SLASH("/"),
-    AMP("&"),
-    BAR("|"),
-    CARET("^"),
-    PERCENT("%"),
-    LTLT("<<"),
-    GTGT(">>"),
-    GTGTGT(">>>"),
-    PLUSEQ("+="),
-    SUBEQ("-="),
-    STAREQ("*="),
-    SLASHEQ("/="),
-    AMPEQ("&="),
-    BAREQ("|="),
-    CARETEQ("^="),
-    PERCENTEQ("%="),
-    LTLTEQ("<<="),
-    GTGTEQ(">>="),
-    GTGTGTEQ(">>>="),
-    MONKEYS_AT("@"),
-    CUSTOM;
-
-    Token() {
-        this(null);
-    }
-    Token(String name) {
-        this.name = name;
-    }
-
-    public final String name;
-
-    public String toString() {
-        switch (this) {
-        case IDENTIFIER:
-            return "token.identifier";
-        case CHARLITERAL:
-            return "token.character";
-        case STRINGLITERAL:
-            return "token.string";
-        case INTLITERAL:
-            return "token.integer";
-        case LONGLITERAL:
-            return "token.long-integer";
-        case FLOATLITERAL:
-            return "token.float";
-        case DOUBLELITERAL:
-            return "token.double";
-        case ERROR:
-            return "token.bad-symbol";
-        case EOF:
-            return "token.end-of-input";
-        case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
-        case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
-            return "'" + name + "'";
-        default:
-            return name;
-        }
-    }
-
-    public String getKind() {
-        return "Token";
-    }
-
-    public String toString(Locale locale, Messages messages) {
-        return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
-    }
-}

src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java

@@ -1072,9 +1072,9 @@ public class JavacProcessingEnvironment implements ProcessingEnvironment, Closea
             Assert.checkNonNull(names);
             next.put(Names.namesKey, names);
 
-            Keywords keywords = Keywords.instance(context);
-            Assert.checkNonNull(keywords);
-            next.put(Keywords.keywordsKey, keywords);
+            Tokens tokens = Tokens.instance(context);
+            Assert.checkNonNull(tokens);
+            next.put(Tokens.tokensKey, tokens);
 
             JavaCompiler oldCompiler = JavaCompiler.instance(context);
             JavaCompiler nextCompiler = JavaCompiler.instance(next);

src/share/classes/com/sun/tools/javadoc/JavadocTool.java

@@ -39,7 +39,6 @@ import javax.tools.StandardLocation;
 
 import com.sun.tools.javac.code.Symbol.CompletionFailure;
 import com.sun.tools.javac.comp.Annotate;
-import com.sun.tools.javac.parser.DocCommentScanner;
 import com.sun.tools.javac.tree.JCTree;
 import com.sun.tools.javac.tree.JCTree.JCClassDecl;
 import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;