Adds a `ProcMacro` form of syntax extension

This commit adds syntax extension forms matching the types for procedural macros 2.0 (RFC #1566), these still require the usual syntax extension boiler plate, but this is a first step towards proper implementation and should be useful for macros 1.1 stuff too.

Supports both attribute-like and function-like macros.

src/librustc_plugin/registry.rs

@@ -111,6 +111,8 @@ pub fn register_syntax_extension(&mut self, name: ast::Name, extension: SyntaxEx
             }
             MultiDecorator(ext) => MultiDecorator(ext),
             MultiModifier(ext) => MultiModifier(ext),
+            SyntaxExtension::ProcMacro(ext) => SyntaxExtension::ProcMacro(ext),
+            SyntaxExtension::AttrProcMacro(ext) => SyntaxExtension::AttrProcMacro(ext),
         }));
     }
 

src/libsyntax/ext/base.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-pub use self::SyntaxExtension::*;
+pub use self::SyntaxExtension::{MultiDecorator, MultiModifier, NormalTT, IdentTT, MacroRulesTT};
 
 use ast::{self, Attribute, Name, PatKind};
 use attr::HasAttrs;
@@ -19,7 +19,7 @@
 use ext::hygiene::Mark;
 use ext::tt::macro_rules;
 use parse;
-use parse::parser;
+use parse::parser::{self, Parser};
 use parse::token;
 use parse::token::{InternedString, str_to_ident};
 use ptr::P;
@@ -31,7 +31,8 @@
 use std::collections::HashMap;
 use std::path::PathBuf;
 use std::rc::Rc;
-use tokenstream;
+use std::default::Default;
+use tokenstream::{self, TokenStream};
 
 
 #[derive(Debug,Clone)]
@@ -60,6 +61,14 @@ fn map_attrs<F: FnOnce(Vec<Attribute>) -> Vec<Attribute>>(self, f: F) -> Self {
 }
 
 impl Annotatable {
+    pub fn span(&self) -> Span {
+        match *self {
+            Annotatable::Item(ref item) => item.span,
+            Annotatable::TraitItem(ref trait_item) => trait_item.span,
+            Annotatable::ImplItem(ref impl_item) => impl_item.span,
+        }
+    }
+
     pub fn expect_item(self) -> P<ast::Item> {
         match self {
             Annotatable::Item(i) => i,
@@ -146,6 +155,173 @@ fn into(self) -> Vec<Annotatable> {
     }
 }
 
+pub trait ProcMacro {
+    fn expand<'cx>(&self,
+                   ecx: &'cx mut ExtCtxt,
+                   span: Span,
+                   ts: TokenStream)
+                   -> Box<MacResult+'cx>;
+}
+
+impl<F> ProcMacro for F
+    where F: Fn(TokenStream) -> TokenStream
+{
+    fn expand<'cx>(&self,
+                   ecx: &'cx mut ExtCtxt,
+                   span: Span,
+                   ts: TokenStream)
+                   -> Box<MacResult+'cx> {
+        let result = (*self)(ts);
+        // FIXME setup implicit context in TLS before calling self.
+        let parser = ecx.new_parser_from_tts(&result.to_tts());
+        Box::new(TokResult { parser: parser, span: span })
+    }
+}
+
+pub trait AttrProcMacro {
+    fn expand<'cx>(&self,
+                   ecx: &'cx mut ExtCtxt,
+                   span: Span,
+                   annotation: TokenStream,
+                   annotated: TokenStream)
+                   -> Box<MacResult+'cx>;
+}
+
+impl<F> AttrProcMacro for F
+    where F: Fn(TokenStream, TokenStream) -> TokenStream
+{
+    fn expand<'cx>(&self,
+                   ecx: &'cx mut ExtCtxt,
+                   span: Span,
+                   annotation: TokenStream,
+                   annotated: TokenStream)
+                   -> Box<MacResult+'cx> {
+        // FIXME setup implicit context in TLS before calling self.
+        let parser = ecx.new_parser_from_tts(&(*self)(annotation, annotated).to_tts());
+        Box::new(TokResult { parser: parser, span: span })
+    }
+}
+
+struct TokResult<'a> {
+    parser: Parser<'a>,
+    span: Span,
+}
+
+impl<'a> MacResult for TokResult<'a> {
+    fn make_items(mut self: Box<Self>) -> Option<SmallVector<P<ast::Item>>> {
+        if self.parser.sess.span_diagnostic.has_errors() {
+            return None;
+        }
+
+        let mut items = SmallVector::zero();
+        loop {
+            match self.parser.parse_item() {
+                Ok(Some(item)) => {
+                    // FIXME better span info.
+                    let mut item = item.unwrap();
+                    item.span = self.span;
+                    items.push(P(item));
+                }
+                Ok(None) => {
+                    return Some(items);
+                }
+                Err(mut e) => {
+                    e.emit();
+                    return None;
+                }
+            }
+        }
+    }
+
+    fn make_impl_items(mut self: Box<Self>) -> Option<SmallVector<ast::ImplItem>> {
+        let mut items = SmallVector::zero();
+        loop {
+            match self.parser.parse_impl_item() {
+                Ok(mut item) => {
+                    // FIXME better span info.
+                    item.span = self.span;
+                    items.push(item);
+
+                    return Some(items);
+                }
+                Err(mut e) => {
+                    e.emit();
+                    return None;
+                }
+            }
+        }
+    }
+
+    fn make_trait_items(mut self: Box<Self>) -> Option<SmallVector<ast::TraitItem>> {
+        let mut items = SmallVector::zero();
+        loop {
+            match self.parser.parse_trait_item() {
+                Ok(mut item) => {
+                    // FIXME better span info.
+                    item.span = self.span;
+                    items.push(item);
+
+                    return Some(items);
+                }
+                Err(mut e) => {
+                    e.emit();
+                    return None;
+                }
+            }
+        }
+    }
+
+    fn make_expr(mut self: Box<Self>) -> Option<P<ast::Expr>> {
+        match self.parser.parse_expr() {
+            Ok(e) => Some(e),
+            Err(mut e) => {
+                e.emit();
+                return None;
+            }
+        }
+    }
+
+    fn make_pat(mut self: Box<Self>) -> Option<P<ast::Pat>> {
+        match self.parser.parse_pat() {
+            Ok(e) => Some(e),
+            Err(mut e) => {
+                e.emit();
+                return None;
+            }
+        }
+    }
+
+    fn make_stmts(mut self: Box<Self>) -> Option<SmallVector<ast::Stmt>> {
+        let mut stmts = SmallVector::zero();
+        loop {
+            if self.parser.token == token::Eof {
+                return Some(stmts);
+            }
+            match self.parser.parse_full_stmt(true) {
+                Ok(Some(mut stmt)) => {
+                    stmt.span = self.span;
+                    stmts.push(stmt);
+                }
+                Ok(None) => { /* continue */ }
+                Err(mut e) => {
+                    e.emit();
+                    return None;
+                }
+            }
+        }
+    }
+
+    fn make_ty(mut self: Box<Self>) -> Option<P<ast::Ty>> {
+        match self.parser.parse_ty() {
+            Ok(e) => Some(e),
+            Err(mut e) => {
+                e.emit();
+                return None;
+            }
+        }
+    }
+}
+
 /// Represents a thing that maps token trees to Macro Results
 pub trait TTMacroExpander {
     fn expand<'cx>(&self,
@@ -439,11 +615,22 @@ pub enum SyntaxExtension {
     /// based upon it.
     ///
     /// `#[derive(...)]` is a `MultiItemDecorator`.
-    MultiDecorator(Box<MultiItemDecorator + 'static>),
+    ///
+    /// Prefer ProcMacro or MultiModifier since they are more flexible.
+    MultiDecorator(Box<MultiItemDecorator>),
 
     /// A syntax extension that is attached to an item and modifies it
-    /// in-place. More flexible version than Modifier.
-    MultiModifier(Box<MultiItemModifier + 'static>),
+    /// in-place. Also allows decoration, i.e., creating new items.
+    MultiModifier(Box<MultiItemModifier>),
+
+    /// A function-like procedural macro. TokenStream -> TokenStream.
+    ProcMacro(Box<ProcMacro>),
+
+    /// An attribute-like procedural macro. TokenStream, TokenStream -> TokenStream.
+    /// The first TokenSteam is the attribute, the second is the annotated item.
+    /// Allows modification of the input items and adding new items, similar to
+    /// MultiModifier, but uses TokenStreams, rather than AST nodes.
+    AttrProcMacro(Box<AttrProcMacro>),
 
     /// A normal, function-like syntax extension.
     ///
@@ -451,12 +638,12 @@ pub enum SyntaxExtension {
     ///
     /// The `bool` dictates whether the contents of the macro can
     /// directly use `#[unstable]` things (true == yes).
-    NormalTT(Box<TTMacroExpander + 'static>, Option<Span>, bool),
+    NormalTT(Box<TTMacroExpander>, Option<Span>, bool),
 
     /// A function-like syntax extension that has an extra ident before
     /// the block.
     ///
-    IdentTT(Box<IdentMacroExpander + 'static>, Option<Span>, bool),
+    IdentTT(Box<IdentMacroExpander>, Option<Span>, bool),
 }
 
 pub type NamedSyntaxExtension = (Name, SyntaxExtension);

src/libsyntax/ext/expand.rs

@@ -22,8 +22,9 @@
 use fold;
 use fold::*;
 use parse::token::{intern, keywords};
+use parse::span_to_tts;
 use ptr::P;
-use tokenstream::TokenTree;
+use tokenstream::{TokenTree, TokenStream};
 use util::small_vector::SmallVector;
 use visit::Visitor;
 
@@ -308,6 +309,31 @@ fn expand_attr_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) ->
                 items.push(item);
                 kind.expect_from_annotatables(items)
             }
+            SyntaxExtension::AttrProcMacro(ref mac) => {
+                let attr_toks = TokenStream::from_tts(span_to_tts(&fld.cx.parse_sess,
+                                                                  attr.span));
+                let item_toks = TokenStream::from_tts(span_to_tts(&fld.cx.parse_sess,
+                                                                  item.span()));
+                let result = mac.expand(self.cx, attr.span, attr_toks, item_toks);
+                let items = match item {
+                    Annotatable::Item(_) => result.make_items()
+                                                  .unwrap_or(SmallVector::zero())
+                                                  .into_iter()
+                                                  .map(|i| Annotatable::Item(i))
+                                                  .collect(),
+                    Annotatable::TraitItem(_) => result.make_trait_items()
+                                                       .unwrap_or(SmallVector::zero())
+                                                       .into_iter()
+                                                       .map(|i| Annotatable::TraitItem(P(i)))
+                                                       .collect(),
+                    Annotatable::ImplItem(_) => result.make_impl_items()
+                                                      .unwrap_or(SmallVector::zero())
+                                                      .into_iter()
+                                                      .map(|i| Annotatable::ImplItem(P(i)))
+                                                      .collect(),
+                };
+                kind.expect_from_annotatables(items)
+            }
             _ => unreachable!(),
         }
     }
@@ -377,11 +403,34 @@ fn expand_bang_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) ->
                 kind.make_from(expander.expand(self.cx, span, ident, marked_tts, attrs))
             }
 
-            MultiDecorator(..) | MultiModifier(..) => {
+            MultiDecorator(..) | MultiModifier(..) | SyntaxExtension::AttrProcMacro(..) => {
                 self.cx.span_err(path.span,
                                  &format!("`{}` can only be used in attributes", extname));
                 return kind.dummy(span);
             }
+
+            SyntaxExtension::ProcMacro(ref expandfun) => {
+                if ident.name != keywords::Invalid.name() {
+                    let msg =
+                        format!("macro {}! expects no ident argument, given '{}'", extname, ident);
+                    fld.cx.span_err(path.span, &msg);
+                    return None;
+                }
+
+                fld.cx.bt_push(ExpnInfo {
+                    call_site: call_site,
+                    callee: NameAndSpan {
+                        format: MacroBang(extname),
+                        // FIXME procedural macros do not have proper span info
+                        // yet, when they do, we should use it here.
+                        span: None,
+                        // FIXME probably want to follow macro_rules macros here.
+                        allow_internal_unstable: false,
+                    },
+                });
+
+                Some(expandfun.expand(fld.cx, call_site, TokenStream::from_tts(marked_tts)))
+            }
         };
 
         let expanded = if let Some(expanded) = opt_expanded {

src/libsyntax/ext/proc_macro_shim.rs

@@ -24,7 +24,9 @@
 
 /// Take a `ExtCtxt`, `Span`, and `TokenStream`, and produce a Macro Result that parses
 /// the TokenStream as a block and returns it as an `Expr`.
-pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStream)
+pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt,
+                                sp: Span,
+                                output: TokenStream)
                                 -> Box<MacResult + 'cx> {
     let parser = cx.new_parser_from_tts(&output.to_tts());
 
@@ -60,7 +62,7 @@ fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
 }
 
 pub mod prelude {
-    pub use ext::proc_macro_shim::build_block_emitter;
+    pub use super::build_block_emitter;
     pub use ast::Ident;
     pub use codemap::{DUMMY_SP, Span};
     pub use ext::base::{ExtCtxt, MacResult};

src/libsyntax/parse/lexer/mod.rs

@@ -85,6 +85,12 @@ pub struct StringReader<'a> {
     /// The last character to be read
     pub curr: Option<char>,
     pub filemap: Rc<syntax_pos::FileMap>,
+    /// If Some, stop reading the source at this position (inclusive).
+    pub terminator: Option<BytePos>,
+    /// Whether to record new-lines in filemap. This is only necessary the first
+    /// time a filemap is lexed. If part of a filemap is being re-lexed, this
+    /// should be set to false.
+    pub save_new_lines: bool,
     // cached:
     pub peek_tok: token::Token,
     pub peek_span: Span,
@@ -96,7 +102,15 @@ pub struct StringReader<'a> {
 
 impl<'a> Reader for StringReader<'a> {
     fn is_eof(&self) -> bool {
-        self.curr.is_none()
+        if self.curr.is_none() {
+            return true;
+        }
+
+        match self.terminator {
+            Some(t) => self.pos > t,
+            None => false,
+        }
+
     }
     /// Return the next token. EFFECT: advances the string_reader.
     fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> {
@@ -164,6 +178,14 @@ impl<'a> StringReader<'a> {
     pub fn new_raw<'b>(span_diagnostic: &'b Handler,
                        filemap: Rc<syntax_pos::FileMap>)
                        -> StringReader<'b> {
+        let mut sr = StringReader::new_raw_internal(span_diagnostic, filemap);
+        sr.bump();
+        sr
+    }
+
+    fn new_raw_internal<'b>(span_diagnostic: &'b Handler,
+                            filemap: Rc<syntax_pos::FileMap>)
+                            -> StringReader<'b> {
         if filemap.src.is_none() {
             span_diagnostic.bug(&format!("Cannot lex filemap \
                                           without source: {}",
@@ -172,21 +194,21 @@ pub fn new_raw<'b>(span_diagnostic: &'b Handler,
 
         let source_text = (*filemap.src.as_ref().unwrap()).clone();
 
-        let mut sr = StringReader {
+        StringReader {
             span_diagnostic: span_diagnostic,
             pos: filemap.start_pos,
             last_pos: filemap.start_pos,
             col: CharPos(0),
             curr: Some('\n'),
             filemap: filemap,
+            terminator: None,
+            save_new_lines: true,
             // dummy values; not read
             peek_tok: token::Eof,
             peek_span: syntax_pos::DUMMY_SP,
             source_text: source_text,
             fatal_errs: Vec::new(),
-        };
-        sr.bump();
-        sr
+        }
     }
 
     pub fn new<'b>(span_diagnostic: &'b Handler,
@@ -200,6 +222,28 @@ pub fn new<'b>(span_diagnostic: &'b Handler,
         sr
     }
 
+    pub fn from_span<'b>(span_diagnostic: &'b Handler,
+                         span: Span,
+                         codemap: &CodeMap)
+                         -> StringReader<'b> {
+        let start_pos = codemap.lookup_byte_offset(span.lo);
+        let last_pos = codemap.lookup_byte_offset(span.hi);
+        assert!(start_pos.fm.name == last_pos.fm.name, "Attempt to lex span which crosses files");
+        let mut sr = StringReader::new_raw_internal(span_diagnostic, start_pos.fm.clone());
+        sr.pos = span.lo;
+        sr.last_pos = span.lo;
+        sr.terminator = Some(span.hi);
+        sr.save_new_lines = false;
+
+        sr.bump();
+
+        if let Err(_) = sr.advance_token() {
+            sr.emit_fatal_errors();
+            panic!(FatalError);
+        }
+        sr
+    }
+
     pub fn curr_is(&self, c: char) -> bool {
         self.curr == Some(c)
     }
@@ -405,7 +449,9 @@ pub fn bump(&mut self) {
             self.curr = Some(ch);
             self.col = self.col + CharPos(1);
             if last_char == '\n' {
-                self.filemap.next_line(self.last_pos);
+                if self.save_new_lines {
+                    self.filemap.next_line(self.last_pos);
+                }
                 self.col = CharPos(0);
             }
 

src/libsyntax/parse/mod.rs

@@ -258,6 +258,13 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>)
     }
 }
 
+pub fn span_to_tts(sess: &ParseSess, span: Span) -> Vec<tokenstream::TokenTree> {
+    let cfg = Vec::new();
+    let srdr = lexer::StringReader::from_span(&sess.span_diagnostic, span, &sess.code_map);
+    let mut p1 = Parser::new(sess, cfg, Box::new(srdr));
+    panictry!(p1.parse_all_token_trees())
+}
+
 /// Given a filemap, produce a sequence of token-trees
 pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>)
     -> Vec<tokenstream::TokenTree> {

src/libsyntax/tokenstream.rs

@@ -33,6 +33,7 @@
 use parse::lexer;
 use parse;
 use parse::token::{self, Token, Lit, Nonterminal};
+use print::pprust;
 
 use std::fmt;
 use std::iter::*;
@@ -781,6 +782,12 @@ pub fn as_delimited_stream(tts: Vec<TokenTree>, delim: token::DelimToken) -> Tok
     }
 }
 
+impl fmt::Display for TokenStream {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str(&pprust::tts_to_string(&self.to_tts()))
+    }
+}
+
 // FIXME Reimplement this iterator to hold onto a slice iterator for a leaf, getting the
 // next leaf's iterator when the current one is exhausted.
 pub struct Iter<'a> {

src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs

+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(plugin, plugin_registrar, rustc_private)]
+
+extern crate proc_macro;
+extern crate rustc_plugin;
+extern crate syntax;
+
+use proc_macro::prelude::*;
+use rustc_plugin::Registry;
+use syntax::ext::base::SyntaxExtension;
+use syntax::ext::proc_macro_shim::prelude::*;
+
+#[plugin_registrar]
+pub fn plugin_registrar(reg: &mut Registry) {
+    reg.register_syntax_extension(token::intern("attr_tru"),
+                                  SyntaxExtension::AttrProcMacro(Box::new(attr_tru)));
+    reg.register_syntax_extension(token::intern("attr_identity"),
+                                  SyntaxExtension::AttrProcMacro(Box::new(attr_identity)));
+    reg.register_syntax_extension(token::intern("tru"),
+                                  SyntaxExtension::ProcMacro(Box::new(tru)));
+    reg.register_syntax_extension(token::intern("ret_tru"),
+                                  SyntaxExtension::ProcMacro(Box::new(ret_tru)));
+    reg.register_syntax_extension(token::intern("identity"),
+                                  SyntaxExtension::ProcMacro(Box::new(identity)));
+}
+
+fn attr_tru(_attr: TokenStream, _item: TokenStream) -> TokenStream {
+    lex("fn f1() -> bool { true }")
+}
+
+fn attr_identity(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let source = item.to_string();
+    lex(&source)
+}
+
+fn tru(_ts: TokenStream) -> TokenStream {
+    lex("true")
+}
+
+fn ret_tru(_ts: TokenStream) -> TokenStream {
+    lex("return true;")
+}
+
+fn identity(ts: TokenStream) -> TokenStream {
+    let source = ts.to_string();
+    lex(&source)
+}

src/test/run-pass-fulldeps/proc_macro.rs

+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:proc_macro_def.rs
+// ignore-stage1
+// ignore-cross-compile
+
+#![feature(plugin, custom_attribute)]
+#![feature(type_macros)]
+
+#![plugin(proc_macro_def)]
+
+#[attr_tru]
+fn f1() -> bool {
+    return false;
+}
+
+#[attr_identity]
+fn f2() -> bool {
+    return identity!(true);
+}
+
+fn f3() -> identity!(bool) {
+    ret_tru!();
+}
+
+fn f4(x: bool) -> bool {
+    match x {
+        identity!(true) => false,
+        identity!(false) => true,
+    }
+}
+
+fn main() {
+    assert!(f1());
+    assert!(f2());
+    assert!(tru!());
+    assert!(f3());
+    assert!(identity!(5 == 5));
+    assert!(f4(false));
+}