syntax: switch tt quoter to emit ~[tt], not tt.

Author: graydon

src/libsyntax/ext/build.rs

@@ -79,6 +79,11 @@ fn mk_uniq_vec_e(cx: ext_ctxt, sp: span, exprs: ~[@ast::expr]) ->
    @ast::expr {
     mk_vstore_e(cx, sp, mk_base_vec_e(cx, sp, exprs), ast::expr_vstore_uniq)
 }
+fn mk_slice_vec_e(cx: ext_ctxt, sp: span, exprs: ~[@ast::expr]) ->
+   @ast::expr {
+    mk_vstore_e(cx, sp, mk_base_vec_e(cx, sp, exprs),
+                ast::expr_vstore_slice)
+}
 fn mk_fixed_vec_e(cx: ext_ctxt, sp: span, exprs: ~[@ast::expr]) ->
    @ast::expr {
     mk_vstore_e(cx, sp, mk_base_vec_e(cx, sp, exprs),

src/libsyntax/ext/expand.rs

@@ -5,7 +5,7 @@ use ast::{crate, expr_, expr_mac, mac_invoc, mac_invoc_tt,
 use fold::*;
 use ext::base::*;
 use ext::qquote::{qq_helper};
-use parse::{parser, parse_expr_from_source_str, new_parser_from_tt};
+use parse::{parser, parse_expr_from_source_str, new_parser_from_tts};
 
 
 use codemap::{span, ExpandedFrom};

src/libsyntax/ext/pipes/pipec.rs

@@ -473,6 +473,7 @@ trait ext_ctxt_parse_utils {
     fn parse_item(s: ~str) -> @ast::item;
     fn parse_expr(s: ~str) -> @ast::expr;
     fn parse_stmt(s: ~str) -> @ast::stmt;
+    fn parse_tts(s: ~str) -> ~[ast::token_tree];
 }
 
 impl ext_ctxt: ext_ctxt_parse_utils {
@@ -508,4 +509,12 @@ impl ext_ctxt: ext_ctxt_parse_utils {
             self.cfg(),
             self.parse_sess())
     }
+
+    fn parse_tts(s: ~str) -> ~[ast::token_tree] {
+        parse::parse_tts_from_source_str(
+            ~"***protocol expansion***",
+            @(copy s),
+            self.cfg(),
+            self.parse_sess())
+    }
 }

src/libsyntax/ext/quote.rs

@@ -19,15 +19,15 @@ use token::*;
 pub mod rt {
     pub use ast::*;
     pub use parse::token::*;
-    pub use parse::new_parser_from_tt;
+    pub use parse::new_parser_from_tts;
     pub use codemap::BytePos;
     pub use codemap::span;
 }
 
 pub fn expand_quote_tokens(cx: ext_ctxt,
                            sp: span,
                            tts: ~[ast::token_tree]) -> base::mac_result {
-    base::mr_expr(expand_tt(cx, sp, tts))
+    base::mr_expr(expand_tts(cx, sp, tts))
 }
 
 pub fn expand_quote_expr(cx: ext_ctxt,
@@ -121,8 +121,7 @@ fn mk_span(cx: ext_ctxt, qsp: span, sp: span) -> @ast::expr {
         }
     };
 
-    let span_path = ids_ext(
-        cx, ~[~"syntax", ~"ext", ~"quote", ~"rt", ~"span"]);
+    let span_path = ids_ext(cx, ~[~"span"]);
 
     build::mk_struct_e(cx, qsp,
                        span_path,
@@ -150,7 +149,7 @@ fn mk_ident(cx: ext_ctxt, sp: span, ident: ast::ident) -> @ast::expr {
 }
 
 fn mk_bytepos(cx: ext_ctxt, sp: span, bpos: BytePos) -> @ast::expr {
-    let path = ids_ext(cx, ~[~"syntax", ~"ext", ~"quote", ~"rt", ~"BytePos"]);
+    let path = ids_ext(cx, ~[~"BytePos"]);
     let arg = build::mk_uint(cx, sp, bpos.to_uint());
     build::mk_call(cx, sp, path, ~[arg])
 }
@@ -318,17 +317,21 @@ fn mk_token(cx: ext_ctxt, sp: span, tok: token::Token) -> @ast::expr {
 
 fn mk_tt(cx: ext_ctxt, sp: span, tt: &ast::token_tree) -> @ast::expr {
     match *tt {
-        ast::tt_tok(sp, tok) =>
-        build::mk_call(cx, sp,
-                       ids_ext(cx, ~[~"tt_tok"]),
-                       ~[mk_span(cx, sp, sp),
-                         mk_token(cx, sp, tok)]),
+        ast::tt_tok(sp, tok) => {
+            let e_tok =
+                build::mk_call(cx, sp,
+                               ids_ext(cx, ~[~"tt_tok"]),
+                               ~[mk_span(cx, sp, sp),
+                                 mk_token(cx, sp, tok)]);
+            build::mk_uniq_vec_e(cx, sp, ~[e_tok])
+        }
 
         ast::tt_delim(tts) => {
-            let e_tts = tts.map(|tt| mk_tt(cx, sp, tt));
-            build::mk_call(cx, sp,
-                           ids_ext(cx, ~[~"tt_delim"]),
-                           ~[build::mk_uniq_vec_e(cx, sp, e_tts)])
+            let e_delim =
+                build::mk_call(cx, sp,
+                               ids_ext(cx, ~[~"tt_delim"]),
+                               ~[mk_tts(cx, sp, tts)]);
+            build::mk_uniq_vec_e(cx, sp, ~[e_delim])
         }
 
         ast::tt_seq(*) => fail ~"tt_seq in quote!",
@@ -338,39 +341,41 @@ fn mk_tt(cx: ext_ctxt, sp: span, tt: &ast::token_tree) -> @ast::expr {
     }
 }
 
+fn mk_tts(cx: ext_ctxt, sp: span, tts: &[ast::token_tree]) -> @ast::expr {
+    let e_tts = tts.map(|tt| mk_tt(cx, sp, tt));
+    build::mk_call(cx, sp,
+                   ids_ext(cx, ~[~"vec", ~"concat"]),
+                   ~[build::mk_slice_vec_e(cx, sp, e_tts)])
+}
 
-fn expand_tt(cx: ext_ctxt,
-             sp: span,
-             tts: ~[ast::token_tree]) -> @ast::expr {
+fn expand_tts(cx: ext_ctxt,
+              sp: span,
+              tts: ~[ast::token_tree]) -> @ast::expr {
     // NB: It appears that the main parser loses its mind if we consider
     // $foo as a tt_nonterminal during the main parse, so we have to re-parse
     // under quote_depth > 0. This is silly and should go away; the _guess_ is
     // it has to do with transition away from supporting old-style macros, so
     // try removing it when enough of them are gone.
-    let p = parse::new_parser_from_tt(cx.parse_sess(), cx.cfg(), tts);
+    let p = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts);
     p.quote_depth += 1u;
-    let tq = dvec::DVec();
-    while p.token != token::EOF {
-        tq.push(p.parse_token_tree());
-    }
-    let tts = tq.get();
+    let tts = p.parse_all_token_trees();
+    p.abort_if_errors();
 
     // We want to emit a block expression that does a sequence of 'use's to
     // import the runtime module, followed by a tt expression.
     let uses = ~[ build::mk_glob_use(cx, sp, ids_ext(cx, ~[~"syntax",
                                                            ~"ext",
                                                            ~"quote",
                                                            ~"rt"])) ];
-    build::mk_block(cx, sp, uses, ~[],
-                    Some(mk_tt(cx, sp, &ast::tt_delim(tts))))
+    build::mk_block(cx, sp, uses, ~[], Some(mk_tts(cx, sp, tts)))
 }
 
 fn expand_parse_call(cx: ext_ctxt,
                      sp: span,
                      parse_method: ~str,
                      arg_exprs: ~[@ast::expr],
                      tts: ~[ast::token_tree]) -> @ast::expr {
-    let tt_expr = expand_tt(cx, sp, tts);
+    let tts_expr = expand_tts(cx, sp, tts);
 
     let cfg_call = || build::mk_call_(
         cx, sp, build::mk_access(cx, sp, ids_ext(cx, ~[~"ext_cx"]),
@@ -386,10 +391,10 @@ fn expand_parse_call(cx: ext_ctxt,
                                      ~"ext",
                                      ~"quote",
                                      ~"rt",
-                                     ~"new_parser_from_tt"]),
+                                     ~"new_parser_from_tts"]),
                        ~[parse_sess_call(),
                          cfg_call(),
-                         build::mk_uniq_vec_e(cx, sp, ~[tt_expr])]);
+                         tts_expr]);
 
     build::mk_call_(cx, sp,
                     build::mk_access_(cx, sp, new_parser_call,

src/libsyntax/parse.rs

@@ -5,12 +5,13 @@ export new_parse_sess, new_parse_sess_special_handler;
 export next_node_id;
 export new_parser_from_file, new_parser_etc_from_file;
 export new_parser_from_source_str;
-export new_parser_from_tt;
+export new_parser_from_tts;
 export new_sub_parser_from_file;
 export parse_crate_from_file, parse_crate_from_crate_file;
 export parse_crate_from_source_str;
 export parse_expr_from_source_str, parse_item_from_source_str;
 export parse_stmt_from_source_str;
+export parse_tts_from_source_str;
 export parse_from_source_str;
 
 use parser::Parser;
@@ -127,6 +128,16 @@ fn parse_stmt_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg,
     return r;
 }
 
+fn parse_tts_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg,
+                             sess: parse_sess) -> ~[ast::token_tree] {
+    let p = new_parser_from_source_str(sess, cfg, name,
+                                       codemap::FssNone, source);
+    p.quote_depth += 1u;
+    let r = p.parse_all_token_trees();
+    p.abort_if_errors();
+    return r;
+}
+
 fn parse_from_source_str<T>(f: fn (p: Parser) -> T,
                             name: ~str, ss: codemap::FileSubstr,
                             source: @~str, cfg: ast::crate_cfg,
@@ -199,9 +210,9 @@ fn new_sub_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg,
     }
 }
 
-fn new_parser_from_tt(sess: parse_sess, cfg: ast::crate_cfg,
-                      tt: ~[ast::token_tree]) -> Parser {
+fn new_parser_from_tts(sess: parse_sess, cfg: ast::crate_cfg,
+                       tts: ~[ast::token_tree]) -> Parser {
     let trdr = lexer::new_tt_reader(sess.span_diagnostic, sess.interner,
-                                    None, tt);
+                                    None, tts);
     return Parser(sess, cfg, trdr as reader)
 }

src/libsyntax/parse/parser.rs

@@ -1321,6 +1321,14 @@ impl Parser {
         };
     }
 
+    fn parse_all_token_trees() -> ~[token_tree] {
+        let tts = DVec();
+        while self.token != token::EOF {
+            tts.push(self.parse_token_tree());
+        }
+        tts.get()
+    }
+
     fn parse_matchers() -> ~[matcher] {
         // unification of matchers and token_trees would vastly improve
         // the interpolation of matchers

src/test/run-pass-fulldeps/quote-tokens.rs

@@ -1,10 +1,12 @@
+#[allow(non_implicitly_copyable_typarams)];
+
 extern mod syntax;
 
 use syntax::ext::base::ext_ctxt;
 
 fn syntax_extension(ext_cx: @ext_ctxt) {
-    let e_toks : syntax::ast::token_tree = quote_tokens!(1 + 2);
-    let p_toks : syntax::ast::token_tree = quote_tokens!((x, 1 .. 4, *));
+    let e_toks : ~[syntax::ast::token_tree] = quote_tokens!(1 + 2);
+    let p_toks : ~[syntax::ast::token_tree] = quote_tokens!((x, 1 .. 4, *));
 
     let _a: @syntax::ast::expr = quote_expr!(1 + 2);
     let _b: Option<@syntax::ast::item> = quote_item!( const foo : int = $e_toks; );
@@ -14,6 +16,6 @@ fn syntax_extension(ext_cx: @ext_ctxt) {
 }
 
 fn main() {
-    let _x: syntax::ast::token_tree = quote_tokens!(a::Foo::foo());
+    let _x: ~[syntax::ast::token_tree] = quote_tokens!(a::Foo::foo());
 }