Comments in the new macro system, reflecting conversation with pauls.

src/libsyntax/ast.rs

@@ -376,6 +376,47 @@ enum token_tree {
 type matcher = spanned<matcher_>;
 
 #[auto_serialize]
+//
+// Matchers are nodes defined-by and recognized-by the main rust parser and
+// language, but they're only ever found inside syntax-extension invocations.
+// They represent a small sub-language for pattern-matching token-trees, and
+// are thus primarily used by the macro-defining extension itself.
+//
+// mtc_tok   ===>   A matcher that matches a single token,
+//                  denoted by the token itself. So long as
+//                  there's no $ involved.
+//
+//
+// mtc_rep   ===>   A matcher that matches a sequence of
+//                  sub-matchers, denoted various ways:
+//
+//             $(M)*       zero or more Ms
+//             $(M)+       one or more Ms
+//             $(M),+      one or more comma-separated Ms
+//             $(A B C);*  zero or more semi-separated 'A B C' seqs
+//
+//
+// mtc_bb ===> A matcher that matches one of a few interesting named rust
+//             nonterminals, such as types, expressions, items, or raw
+//             token-trees. A black-box matcher on expr, for example, binds an
+//             expr to a given ident, and that ident can re-occur as an
+//             interpolation in the RHS of a macro-by-example rule. For
+//             example:
+//
+//        $foo:expr   =>     1 + $foo    // interpolate an expr
+//        $foo:tt     =>     $foo        // interpolate a token-tree
+//        $foo:tt     =>     bar! $foo   // only other valid interpolation
+//                                       // is in arg position for another macro
+//
+// As a final, horrifying aside, note that macro-by-example's input is
+// also matched by one of these matchers. Holy self-referential! It is matched
+// by an mtc_rep, specifically this one:
+//
+//                   $( $lhs:mtcs => $rhs:tt );+
+//
+// If you understand that, you have closed to loop and understand the whole
+// macro system. Congratulations.
+//
 enum matcher_ {
     /* match one token */
     mtc_tok(token::token),
@@ -401,10 +442,11 @@ enum matcher_ {
 #[auto_serialize]
 enum mac_ {
     mac_invoc(@path, mac_arg, mac_body),
-    mac_invoc_tt(@path,~[token_tree]),//will kill mac_invoc and steal its name
-    mac_embed_type(@ty),
-    mac_embed_block(blk),
-    mac_ellipsis,
+    mac_invoc_tt(@path,~[token_tree]), // will kill mac_invoc and steal its name
+    mac_embed_type(@ty),  // obsolete quoter
+    mac_embed_block(blk), // obsolete quoter
+    mac_ellipsis,         // obsolete pattern-match terminal
+
     // the span is used by the quoter/anti-quoter ...
     mac_aq(span /* span of quote */, @expr), // anti-quote
     mac_var(uint)

src/libsyntax/ext/base.rs

@@ -4,6 +4,31 @@
 import codemap::{codemap, span, expn_info, expanded_from};
 import std::map::str_hash;
 
+
+// Nomenclature / abbreviations in the ext modules:
+//
+//     ms: matcher span, wraps a matcher with fake span
+//    mtc: matcher
+//   mtcs: matchers
+//     tt: token tree
+//     bt: backtrace
+//     cx: expansion context
+//     mr: macro result
+//
+
+// obsolete old-style #macro code:
+//
+//    syntax_expander, normal, macro_defining, macro_definer,
+//    builtin
+//
+// new-style macro! tt code:
+//
+//    syntax_expander_tt, syntax_expander_tt_item, mac_result,
+//    expr_tt, item_tt
+//
+// also note that ast::mac has way too many cases and can probably
+// be trimmed down substantially.
+
 // second argument is the span to blame for general argument problems
 type syntax_expander_ =
     fn@(ext_ctxt, span, ast::mac_arg, ast::mac_body) -> @ast::expr;
@@ -11,8 +36,11 @@
 type syntax_expander = {expander: syntax_expander_, span: option<span>};
 
 type macro_def = {ident: ast::ident, ext: syntax_extension};
+
+// macro_definer is obsolete, remove when #old_macros go away.
 type macro_definer =
     fn@(ext_ctxt, span, ast::mac_arg, ast::mac_body) -> macro_def;
+
 type item_decorator =
     fn@(ext_ctxt, span, ast::meta_item, ~[@ast::item]) -> ~[@ast::item];
 
@@ -32,10 +60,17 @@ enum mac_result {
 }
 
 enum syntax_extension {
+
+    // normal() is obsolete, remove when #old_macros go away.
     normal(syntax_expander),
+
+    // macro_defining() is obsolete, remove when #old_macros go away.
     macro_defining(macro_definer),
+
+    // #[auto_serialize] and such. will probably survive death of #old_macros
     item_decorator(item_decorator),
 
+    // Token-tree expanders
     expr_tt(syntax_expander_tt),
     item_tt(syntax_expander_tt_item),
 }
@@ -87,6 +122,10 @@ fn builtin_item_tt(f: syntax_expander_tt_item_) -> syntax_extension {
     ret syntax_expanders;
 }
 
+
+// One of these is made during expansion and incrementally updated as we go;
+// when a macro expansion occurs, the resulting nodes have the backtrace()
+// -> expn_info of their expansion context stored into their span.
 iface ext_ctxt {
     fn codemap() -> codemap;
     fn parse_sess() -> parse::parse_sess;
@@ -244,6 +283,9 @@ fn get_mac_body(cx: ext_ctxt, sp: span, args: ast::mac_body)
     }
 }
 
+// Massage syntactic form of new-style arguments to internal representation
+// of old-style macro args, such that old-style macro can be run and invoked
+// using new syntax. This will be obsolete when #old_macros go away.
 fn tt_args_to_original_flavor(cx: ext_ctxt, sp: span, arg: ~[ast::token_tree])
     -> ast::mac_arg {
     import ast::{matcher, matcher_, mtc_tok, mtc_rep, mtc_bb};

src/libsyntax/ext/expand.rs

@@ -16,7 +16,12 @@ fn expand_expr(exts: hashmap<~str, syntax_extension>, cx: ext_ctxt,
     -> (expr_, span)
 {
     ret alt e {
+      // expr_mac should really be expr_ext or something; it's the
+      // entry-point for all syntax extensions.
           expr_mac(mac) {
+
+            // Old-style macros, for compatibility, will erase this whole
+            // block once we've transitioned.
             alt mac.node {
               mac_invoc(pth, args, body) {
                 assert (vec::len(pth.idents) > 0u);
@@ -58,6 +63,9 @@ fn expand_expr(exts: hashmap<~str, syntax_extension>, cx: ext_ctxt,
                   }
                 }
               }
+
+              // Token-tree macros, these will be the only case when we're
+              // finished transitioning.
               mac_invoc_tt(pth, tts) {
                 assert (vec::len(pth.idents) == 1u);
                 let extname = pth.idents[0];
@@ -111,6 +119,15 @@ fn expand_expr(exts: hashmap<~str, syntax_extension>, cx: ext_ctxt,
         };
 }
 
+// This is a secondary mechanism for invoking syntax extensions on items:
+// "decorator" attributes, such as #[auto_serialize]. These are invoked by an
+// attribute prefixing an item, and are interpreted by feeding the item
+// through the named attribute _as a syntax extension_ and splicing in the
+// resulting item vec into place in favour of the decorator. Note that
+// these do _not_ work for macro extensions, just item_decorator ones.
+//
+// NB: there is some redundancy between this and expand_item, below, and
+// they might benefit from some amount of semantic and language-UI merger.
 fn expand_mod_items(exts: hashmap<~str, syntax_extension>, cx: ext_ctxt,
                     module: ast::_mod, fld: ast_fold,
                     orig: fn@(ast::_mod, ast_fold) -> ast::_mod)
@@ -145,7 +162,8 @@ fn expand_mod_items(exts: hashmap<~str, syntax_extension>, cx: ext_ctxt,
     ret {items: new_items with module};
 }
 
-/* record module we enter for `#mod` */
+// Support for item-position macro invocations, exactly the same
+// logic as for expression-position macro invocations.
 fn expand_item(exts: hashmap<~str, syntax_extension>,
                cx: ext_ctxt, &&it: @ast::item, fld: ast_fold,
                orig: fn@(&&@ast::item, ast_fold) -> option<@ast::item>)

src/libsyntax/ext/tt/earley_parser.rs

@@ -77,7 +77,24 @@ fn initial_matcher_pos(ms: ~[matcher], sep: option<token>, lo: uint)
       match_lo: 0u, match_hi: match_idx_hi, sp_lo: lo}
 }
 
-/* logically, an arb_depth should contain only one kind of nonterminal */
+// arb_depth is a pattern-match result for a single black-box matcher
+// (ast::mtc_bb): so it is associated with a single ident in a parse, and all
+// leaves in the arb_depth have the same nonterminal type (expr, item,
+// etc). All the leaves in a single arb_depth correspond to a single mtc_bb in
+// the ast::matcher that produced it.
+//
+// It should probably be renamed, it has more or less exact correspondence to
+// ast::match nodes, and the in-memory structure of a particular arb_depth
+// represents the match that occurred when a particular subset of an
+// ast::match -- those ast::matcher nodes leading to a single mtc_bb -- was
+// applied to a particular token tree.
+//
+// The width of each seq in the arb_depth, and the identity of the leaf nodes,
+// will depend on the token tree it was applied to: each seq corresponds to a
+// single mtc_rep in the originating ast::matcher. The depth of the arb_depth
+// structure will therefore depend only on the nesting depth of mtc_reps in
+// the originating ast::matcher it was derived from.
+
 enum arb_depth { leaf(whole_nt), seq(~[@arb_depth], codemap::span) }
 
 type earley_item = matcher_pos;