Use TokenTrees in lhs of macros

Author: pczarn

src/librustdoc/html/highlight.rs

@@ -163,7 +163,8 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
 
             token::Lifetime(..) => "lifetime",
             token::DocComment(..) => "doccomment",
-            token::Underscore | token::Eof | token::Interpolated(..) => "",
+            token::Underscore | token::Eof | token::Interpolated(..) |
+                token::MatchNt(..) | token::SubstNt(..) => "",
         };
 
         // as mentioned above, use the original source code instead of

src/libsyntax/ast.rs

@@ -10,7 +10,7 @@
 
 // The Rust abstract syntax tree.
 
-use codemap::{Span, Spanned, DUMMY_SP, ExpnId};
+use codemap::{Span, Spanned, DUMMY_SP, ExpnId, respan};
 use abi::Abi;
 use ast_util;
 use owned_slice::OwnedSlice;
@@ -657,23 +657,55 @@ pub enum TokenTree {
     /// A delimited sequence of token trees
     TtDelimited(Span, Rc<Delimited>),
 
-    // These only make sense for right-hand-sides of MBE macros:
+    // This only makes sense for right-hand-sides of MBE macros:
 
     /// A Kleene-style repetition sequence with an optional separator.
     // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST.
-    TtSequence(Span, Rc<Vec<TokenTree>>, Option<token::Token>, KleeneOp),
-    /// A syntactic variable that will be filled in by macro expansion.
-    TtNonterminal(Span, Ident)
+    TtSequence(Span, Rc<Vec<TokenTree>>, Option<::parse::token::Token>, KleeneOp, uint),
 }
 
 impl TokenTree {
+    pub fn expand_into_tts(self) -> Rc<Vec<TokenTree>> {
+        match self {
+            TtToken(sp, token::DocComment(name)) => {
+                let doc = MetaNameValue(token::intern_and_get_ident("doc"),
+                                        respan(sp, LitStr(token::get_name(name), CookedStr)));
+                let doc = token::NtMeta(P(respan(sp, doc)));
+                let delimed = Delimited {
+                    delim: token::Bracket,
+                    open_span: sp,
+                    tts: vec![TtToken(sp, token::Interpolated(doc))],
+                    close_span: sp,
+                };
+                Rc::new(vec![TtToken(sp, token::Pound),
+                             TtDelimited(sp, Rc::new(delimed))])
+            }
+            TtDelimited(_, ref delimed) => {
+                let mut tts = Vec::with_capacity(1 + delimed.tts.len() + 1);
+                tts.push(delimed.open_tt());
+                tts.extend(delimed.tts.iter().map(|tt| tt.clone()));
+                tts.push(delimed.close_tt());
+                Rc::new(tts)
+            }
+            TtToken(sp, token::SubstNt(name, namep)) => {
+                Rc::new(vec![TtToken(sp, token::Dollar),
+                             TtToken(sp, token::Ident(name, namep))])
+            }
+            TtToken(sp, token::MatchNt(name, kind, namep, kindp)) => {
+                Rc::new(vec![TtToken(sp, token::SubstNt(name, namep)),
+                             TtToken(sp, token::Colon),
+                             TtToken(sp, token::Ident(kind, kindp))])
+            }
+            _ => panic!("Cannot expand a token")
+        }
+    }
+
     /// Returns the `Span` corresponding to this token tree.
     pub fn get_span(&self) -> Span {
         match *self {
-            TtToken(span, _)           => span,
-            TtDelimited(span, _)       => span,
-            TtSequence(span, _, _, _)  => span,
-            TtNonterminal(span, _)     => span,
+            TtToken(span, _)              => span,
+            TtDelimited(span, _)          => span,
+            TtSequence(span, _, _, _, _)  => span,
         }
     }
 }

src/libsyntax/ext/quote.rs

@@ -616,6 +616,20 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
                                 vec!(mk_name(cx, sp, ident.ident())));
         }
 
+        token::MatchNt(name, kind, name_style, kind_style) => {
+            return cx.expr_call(sp,
+                                mk_token_path(cx, sp, "MatchNt"),
+                                vec![mk_ident(cx, sp, name),
+                                     mk_ident(cx, sp, kind),
+                                     match name_style {
+                                         ModName => mk_token_path(cx, sp, "ModName"),
+                                         Plain   => mk_token_path(cx, sp, "Plain"),
+                                     },
+                                     match kind_style {
+                                         ModName => mk_token_path(cx, sp, "ModName"),
+                                         Plain   => mk_token_path(cx, sp, "Plain"),
+                                     }]);
+        }
         token::Interpolated(_) => panic!("quote! with interpolated token"),
 
         _ => ()
@@ -654,6 +668,25 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
 
 fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
     match *tt {
+        ast::TtToken(sp, SubstNt(ident, _)) => {
+            // tt.extend($ident.to_tokens(ext_cx).into_iter())
+
+            let e_to_toks =
+                cx.expr_method_call(sp,
+                                    cx.expr_ident(sp, ident),
+                                    id_ext("to_tokens"),
+                                    vec!(cx.expr_ident(sp, id_ext("ext_cx"))));
+            let e_to_toks =
+                cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]);
+
+            let e_push =
+                cx.expr_method_call(sp,
+                                    cx.expr_ident(sp, id_ext("tt")),
+                                    id_ext("extend"),
+                                    vec!(e_to_toks));
+
+            vec!(cx.stmt_expr(e_push))
+        }
         ast::TtToken(sp, ref tok) => {
             let e_sp = cx.expr_ident(sp, id_ext("_sp"));
             let e_tok = cx.expr_call(sp,
@@ -673,25 +706,6 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
                 .collect()
         },
         ast::TtSequence(..) => panic!("TtSequence in quote!"),
-        ast::TtNonterminal(sp, ident) => {
-            // tt.extend($ident.to_tokens(ext_cx).into_iter())
-
-            let e_to_toks =
-                cx.expr_method_call(sp,
-                                    cx.expr_ident(sp, ident),
-                                    id_ext("to_tokens"),
-                                    vec!(cx.expr_ident(sp, id_ext("ext_cx"))));
-            let e_to_toks =
-                cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]);
-
-            let e_push =
-                cx.expr_method_call(sp,
-                                    cx.expr_ident(sp, id_ext("tt")),
-                                    id_ext("extend"),
-                                    vec!(e_to_toks));
-
-            vec!(cx.stmt_expr(e_push))
-        },
     }
 }