From 29accaf57611341beff24facf90d25241d5dd21b Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 22:52:34 -0700 Subject: [PATCH 01/67] disallow ident equality checks when contexts are not equal --- src/libsyntax/ast.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 470e05223db16..8bbd7ef106438 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -24,9 +24,22 @@ use extra::serialize::{Encodable, Decodable, Encoder, Decoder}; // table) and a SyntaxContext to track renaming and // macro expansion per Flatt et al., "Macros // That Work Together" -#[deriving(Clone, Eq, IterBytes)] +#[deriving(Clone, IterBytes)] pub struct ident { name: Name, ctxt: SyntaxContext } +impl Eq for ident { + fn eq(&self, other: &ident) -> bool { + if (self.ctxt == other.ctxt) { + self.name == other.name + } else { + fail!(fmt!("not allowed to compare these idents: %?, %?", self, other)); + } + } + fn ne(&self, other: &ident) -> bool { + ! self.eq(other) + } +} + /// Construct an identifier with the given name and an empty context: pub fn new_ident(name: Name) -> ident { ident {name: name, ctxt: empty_ctxt}} From 3ca295c7ee52af11b9e9aebc95ee72afcb2ebd99 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 14:09:30 -0700 Subject: [PATCH 02/67] rename resolve to mtwt_resolve --- src/libsyntax/ast_util.rs | 2 +- src/libsyntax/ext/expand.rs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index 37112a533c816..3d38e527829bc 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -722,7 +722,7 @@ fn idx_push(vec: &mut ~[T], val: T) -> uint { } /// Resolve a syntax object to a name, per MTWT. -pub fn resolve(id : ident) -> Name { +pub fn mtwt_resolve(id : ident) -> Name { resolve_internal(id, get_sctable()) } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index af05f72686060..13759450c3c71 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -12,7 +12,7 @@ use ast::{Block, Crate, expr_, expr_mac, mac_invoc_tt}; use ast::{item_mac, stmt_, stmt_mac, stmt_expr, stmt_semi}; use ast::{illegal_ctxt}; use ast; -use ast_util::{new_rename, new_mark, resolve}; +use ast_util::{new_rename, new_mark, mtwt_resolve}; use attr; use attr::AttrMetaMethods; use codemap; @@ -784,12 +784,11 @@ pub fn new_ident_resolver() -> @fn(ast::ident)->ast::ident { |id : ast::ident| ast::ident { - name : resolve(id), + name : mtwt_resolve(id), ctxt : illegal_ctxt } } - #[cfg(test)] mod test { use super::*; From a5df81dc57dd2dd90eda4160a3baaf1acc0f9b3e Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 5 Jun 2013 19:49:41 -0700 Subject: [PATCH 03/67] ident->name --- src/librustc/middle/resolve.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/librustc/middle/resolve.rs b/src/librustc/middle/resolve.rs index e2d2ac9cf238a..a9951b4eebaa2 100644 --- a/src/librustc/middle/resolve.rs +++ b/src/librustc/middle/resolve.rs @@ -28,7 +28,7 @@ use syntax::ast_util::{Privacy, Public, Private}; use syntax::ast_util::{variant_visibility_to_privacy, visibility_to_privacy}; use syntax::attr; use syntax::parse::token; -use syntax::parse::token::ident_interner; +use syntax::parse::token::{ident_interner, interner_get}; use syntax::parse::token::special_idents; use syntax::print::pprust::path_to_str; use syntax::codemap::{span, dummy_sp, BytePos}; @@ -284,7 +284,7 @@ pub enum DuplicateCheckingMode { /// One local scope. pub struct Rib { - bindings: @mut HashMap, + bindings: @mut HashMap, self_binding: @mut Option, kind: RibKind, } @@ -3435,7 +3435,7 @@ impl Resolver { let mut i = ribs.len(); while i != 0 { i -= 1; - match ribs[i].bindings.find(&name) { + match ribs[i].bindings.find(&name.name) { Some(&def_like) => { return self.upvarify(ribs, i, def_like, span, allow_capturing_self); @@ -3531,7 +3531,7 @@ impl Resolver { // Create a new rib for the self type. let self_type_rib = @Rib(NormalRibKind); self.type_ribs.push(self_type_rib); - self_type_rib.bindings.insert(self.type_self_ident, + self_type_rib.bindings.insert(self.type_self_ident.name, dl_def(def_self_ty(item.id))); // Create a new rib for the trait-wide type parameters. @@ -3671,7 +3671,7 @@ impl Resolver { // the item that bound it self.record_def(type_parameter.id, def_typaram_binder(node_id)); - function_type_rib.bindings.insert(name, def_like); + function_type_rib.bindings.insert(name.name, def_like); } } @@ -4290,7 +4290,7 @@ impl Resolver { let this = &mut *self; let last_rib = this.value_ribs[ this.value_ribs.len() - 1]; - last_rib.bindings.insert(ident, + last_rib.bindings.insert(ident.name, dl_def(def)); bindings_list.insert(ident, pat_id); } @@ -4311,7 +4311,7 @@ impl Resolver { let this = &mut *self; let last_rib = this.value_ribs[ this.value_ribs.len() - 1]; - last_rib.bindings.insert(ident, + last_rib.bindings.insert(ident.name, dl_def(def)); } } @@ -4839,7 +4839,7 @@ impl Resolver { while j != 0 { j -= 1; for this.value_ribs[j].bindings.each_key |&k| { - maybes.push(this.session.str_of(k)); + maybes.push(interner_get(k)); values.push(uint::max_value); } } @@ -5009,7 +5009,7 @@ impl Resolver { let this = &mut *self; let def_like = dl_def(def_label(expr.id)); let rib = this.label_ribs[this.label_ribs.len() - 1]; - rib.bindings.insert(label, def_like); + rib.bindings.insert(label.name, def_like); } visit_expr(expr, ((), visitor)); @@ -5417,3 +5417,4 @@ pub fn resolve_crate(session: Session, trait_map: resolver.trait_map.clone(), } } + From 85ef8342641fda44ed61f4c147b60b4484c001ce Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 15:10:45 -0700 Subject: [PATCH 04/67] compare macro tokens hygienically (commented out) --- src/libsyntax/ext/tt/macro_parser.rs | 16 ++++++++++++++- src/libsyntax/parse/parser.rs | 5 ++++- src/libsyntax/parse/token.rs | 30 +++++++++++++++++++++++----- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 9d9155ff0d521..a60feb066ec75 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -234,6 +234,19 @@ pub fn parse_or_else( } } +// temporary for testing +pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { + if (*t1 == *t2) { + true + } else { + match (t1,t2) { + (&token::IDENT(id1,_),&token::IDENT(id2,_)) => + id1.name == id2.name, + _ => false + } + } +} + pub fn parse( sess: @mut ParseSess, cfg: ast::CrateConfig, @@ -343,7 +356,8 @@ pub fn parse( match_nonterminal(_,_,_) => { bb_eis.push(ei) } match_tok(ref t) => { let mut ei_t = ei.clone(); - if (*t) == tok { + if (token_name_eq(t,&tok)) { + //if (token::mtwt_token_eq(t,&tok)) { ei_t.idx += 1; next_eis.push(ei_t); } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index dc42b4bdb80b2..61617b67e7767 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -3393,7 +3393,10 @@ impl Parser { } fn is_self_ident(&self) -> bool { - *self.token == token::IDENT(special_idents::self_, false) + match *self.token { + token::IDENT(id, false) => id.name == special_idents::self_.name, + _ => false + } } fn expect_self_ident(&self) { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 52b6d4459bf48..44f479d0e49b7 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -679,14 +679,34 @@ pub fn is_reserved_keyword(tok: &Token) -> bool { } } +pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool { + if (*t1 == *t2) { + true + } else { + match (t1,t2) { + (&IDENT(id1,_),&IDENT(id2,_)) => + ast_util::mtwt_resolve(id1) == ast_util::mtwt_resolve(id2), + _ => false + } + } +} + #[cfg(test)] mod test { + use ast; + use ast_util; use super::*; - use std::io; - #[test] fn t1() { - let a = fresh_name("ghi"); - printfln!("interned name: %u,\ntextual name: %s\n", - a, interner_get(a)); + + fn mark_ident(id : ast::ident, m : ast::Mrk) -> ast::ident { + ast::ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)} + } + + #[test] fn mtwt_token_eq_test() { + assert!(mtwt_token_eq(>,>)); + let a = str_to_ident("bac"); + let a1 = mark_ident(a,92); + assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false))); } + } From f08967e5ca4195a479555f7762cc92238d599079 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 26 Jun 2013 11:16:09 -0700 Subject: [PATCH 05/67] try removing basically dead code in resolve --- src/librustc/middle/resolve.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/librustc/middle/resolve.rs b/src/librustc/middle/resolve.rs index a9951b4eebaa2..1cd4df91e4d06 100644 --- a/src/librustc/middle/resolve.rs +++ b/src/librustc/middle/resolve.rs @@ -4605,25 +4605,13 @@ impl Resolver { return NoNameDefinition; } - pub fn intern_module_part_of_path(@mut self, path: &Path) -> ~[ident] { - let mut module_path_idents = ~[]; - for path.idents.iter().enumerate().advance |(index, ident)| { - if index == path.idents.len() - 1 { - break; - } - - module_path_idents.push(*ident); - } - - return module_path_idents; - } - + // resolve a "module-relative" path, e.g. a::b::c pub fn resolve_module_relative_path(@mut self, path: &Path, xray: XrayFlag, namespace: Namespace) -> Option { - let module_path_idents = self.intern_module_part_of_path(path); + let module_path_idents = path.idents.init(); let containing_module; match self.resolve_module_path(self.current_module, @@ -4689,7 +4677,7 @@ impl Resolver { xray: XrayFlag, namespace: Namespace) -> Option { - let module_path_idents = self.intern_module_part_of_path(path); + let module_path_idents = path.idents.init(); let root_module = self.graph_root.get_module(); From 844a30347a8b53fb26a9fc83746a6af4fc0f88f5 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 25 Jun 2013 11:40:51 -0700 Subject: [PATCH 06/67] add hygiene support fns, move them around. also adds test cases --- src/libsyntax/ast.rs | 2 +- src/libsyntax/ast_util.rs | 2 +- src/libsyntax/ext/expand.rs | 210 +++++++++++++++++++++------ src/libsyntax/parse/token.rs | 4 +- src/libsyntax/util/parser_testing.rs | 15 +- 5 files changed, 179 insertions(+), 54 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 8bbd7ef106438..e17d9192ddee5 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -546,7 +546,7 @@ pub enum token_tree { // These only make sense for right-hand-sides of MBE macros: // a kleene-style repetition sequence with a span, a tt_forest, - // an optional separator (?), and a boolean where true indicates + // an optional separator, and a boolean where true indicates // zero or more (*), and false indicates one or more (+). tt_seq(span, @mut ~[token_tree], Option<::parse::token::Token>, bool), diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index 3d38e527829bc..676424508588d 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -633,7 +633,7 @@ pub fn pat_is_ident(pat: @ast::pat) -> bool { // HYGIENE FUNCTIONS /// Construct an identifier with the given name and an empty context: -pub fn new_ident(name: Name) -> ident { ident {name: name, ctxt: 0}} +pub fn new_ident(name: Name) -> ident { ident {name: name, ctxt: empty_ctxt}} /// Extend a syntax context with a given mark pub fn new_mark(m:Mrk, tail:SyntaxContext) -> SyntaxContext { diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 13759450c3c71..f9f16ba312823 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -8,20 +8,20 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Block, Crate, expr_, expr_mac, mac_invoc_tt}; -use ast::{item_mac, stmt_, stmt_mac, stmt_expr, stmt_semi}; -use ast::{illegal_ctxt}; +use ast::{Block, Crate, decl_local, expr_, expr_mac, Local, mac_invoc_tt}; +use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi}; +use ast::{SCTable, token_tree, illegal_ctxt}; use ast; use ast_util::{new_rename, new_mark, mtwt_resolve}; use attr; use attr::AttrMetaMethods; use codemap; -use codemap::{span, ExpnInfo, NameAndSpan}; +use codemap::{span, spanned, ExpnInfo, NameAndSpan}; use ext::base::*; use fold::*; use parse; use parse::{parse_item_from_source_str}; -use parse::token::{ident_to_str, intern}; +use parse::token::{fresh_name, ident_to_str, intern}; use visit; use visit::Visitor; @@ -362,6 +362,74 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, } + +// expand a non-macro stmt. this is essentially the fallthrough for +// expand_stmt, above. +fn expand_non_macro_stmt (exts: SyntaxEnv, + s: &stmt_, + sp: span, + fld: @ast_fold, + orig: @fn(&stmt_, span, @ast_fold) -> (Option, span)) + -> (Option,span) { + // is it a let? + match *s { + stmt_decl(@spanned{node: decl_local(ref local), span: stmt_span}, node_id) => { + let block_info = get_block_info(exts); + let pending_renames = block_info.pending_renames; + // there's no need to have a lot of these, we could + // just pass one of them around... + let name_finder = new_name_finder(); + + // take it apart: + let @Local{is_mutbl:is_mutbl, + ty:_, + pat:pat, + init:init, + id:id, + span:span + } = *local; + // types can't be copied automatically because of the owned ptr in ty_tup... + let ty = local.ty.clone(); + // expand the pat (it might contain exprs... #:(o)> + let expanded_pat = fld.fold_pat(pat); + // find the pat_idents in the pattern: + // oh dear heaven... this is going to include the enum names, as well.... + let idents = @mut ~[]; + ((*name_finder).visit_pat) (expanded_pat, + (idents, + visit::mk_vt(name_finder))); + // generate fresh names, push them to a new pending list + let new_pending_renames = @mut ~[]; + for idents.iter().advance |ident| { + let new_name = fresh_name(ident); + new_pending_renames.push((*ident,new_name)); + } + let mut rename_fld = renames_to_fold(new_pending_renames); + // rewrite the pattern using the new names (the old ones + // have already been applied): + let rewritten_pat = rename_fld.fold_pat(expanded_pat); + // add them to the existing pending renames: + for new_pending_renames.iter().advance |pr| {pending_renames.push(*pr)} + // also, don't forget to expand the init: + let new_init_opt = init.map(|e| fld.fold_expr(*e)); + let rewritten_local = + @Local{is_mutbl:is_mutbl, + ty:ty, + pat:rewritten_pat, + init:new_init_opt, + id:id, + span:span}; + (Some(stmt_decl(@spanned{node:decl_local(rewritten_local), + span: stmt_span},node_id)), + sp) + }, + _ => { + orig(s, sp, fld) + } + } +} + + // return a visitor that extracts the pat_ident paths // from a given pattern and puts them in a mutable // array (passed in to the traversal) @@ -393,30 +461,10 @@ pub fn new_name_finder() -> @Visitor<@mut ~[ast::ident]> { } } -pub fn expand_block(extsbox: @mut SyntaxEnv, - _cx: @ExtCtxt, - blk: &Block, - fld: @ast_fold, - orig: @fn(&Block, @ast_fold) -> Block) - -> Block { - // see note below about treatment of exts table - with_exts_frame!(extsbox,false,orig(blk,fld)) -} - - -// get the (innermost) BlockInfo from an exts stack -fn get_block_info(exts : SyntaxEnv) -> BlockInfo { - match exts.find_in_topmost_frame(&intern(special_block_name)) { - Some(@BlockInfo(bi)) => bi, - _ => fail!(fmt!("special identifier %? was bound to a non-BlockInfo", - @" block")) - } -} - - // given a mutable list of renames, return a tree-folder that applies those // renames. -fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @ast_fold { +// FIXME #4536: currently pub to allow testing +pub fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @ast_fold { let afp = default_ast_fold(); let f_pre = @AstFoldFns { fold_ident: |id,_| { @@ -432,15 +480,57 @@ fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @ast_fold { make_fold(f_pre) } -// perform a bunch of renames -fn apply_pending_renames(folder : @ast_fold, stmt : ast::stmt) -> @ast::stmt { - match folder.fold_stmt(&stmt) { - Some(s) => s, - None => fail!(fmt!("renaming of stmt produced None")) + +pub fn expand_block(extsbox: @mut SyntaxEnv, + _cx: @ExtCtxt, + blk: &Block, + fld: @ast_fold, + orig: @fn(&Block, @ast_fold) -> Block) + -> Block { + // see note below about treatment of exts table + with_exts_frame!(extsbox,false,orig(blk,fld)) +} + + +pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block { + let block_info = get_block_info(exts); + let pending_renames = block_info.pending_renames; + let mut rename_fld = renames_to_fold(pending_renames); + let new_view_items = b.view_items.map(|x| fld.fold_view_item(x)); + let mut new_stmts = ~[]; + for b.stmts.iter().advance |x| { + match fld.fold_stmt(mustbesome(rename_fld.fold_stmt(*x))) { + Some(s) => new_stmts.push(s), + None => () + } + } + let new_expr = b.expr.map(|x| fld.fold_expr(rename_fld.fold_expr(*x))); + Block{ + view_items: new_view_items, + stmts: new_stmts, + expr: new_expr, + id: fld.new_id(b.id), + rules: b.rules, + span: b.span, } } +// rename_fold should never return "None". +fn mustbesome(val : Option) -> T { + match val { + Some(v) => v, + None => fail!("rename_fold returned None") + } +} +// get the (innermost) BlockInfo from an exts stack +fn get_block_info(exts : SyntaxEnv) -> BlockInfo { + match exts.find_in_topmost_frame(&intern(special_block_name)) { + Some(@BlockInfo(bi)) => bi, + _ => fail!(fmt!("special identifier %? was bound to a non-BlockInfo", + @" block")) + } +} pub fn new_span(cx: @ExtCtxt, sp: span) -> span { /* this discards information in the case of macro-defining macros */ @@ -794,12 +884,14 @@ mod test { use super::*; use ast; use ast::{Attribute_, AttrOuter, MetaWord, empty_ctxt}; + use ast_util::{get_sctable, new_ident, new_rename}; use codemap; use codemap::spanned; use parse; - use parse::token::{intern, get_ident_interner}; + use parse::token::{gensym, intern, get_ident_interner}; use print::pprust; - use util::parser_testing::{string_to_item, string_to_pat, strs_to_idents}; + use std; + use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat, strs_to_idents}; use visit::{mk_vt}; // make sure that fail! is present @@ -900,26 +992,60 @@ mod test { #[test] fn renaming () { - let maybe_item_ast = string_to_item(@"fn a() -> int { let b = 13; b }"); - let item_ast = match maybe_item_ast { - Some(x) => x, - None => fail!("test case fail") - }; + let item_ast = string_to_item(@"fn a() -> int { let b = 13; b }").get(); let a_name = intern("a"); - let a2_name = intern("a2"); + let a2_name = gensym("a2"); let renamer = new_ident_renamer(ast::ident{name:a_name,ctxt:empty_ctxt}, a2_name); let renamed_ast = fun_to_ident_folder(renamer).fold_item(item_ast).get(); let resolver = new_ident_resolver(); - let resolved_ast = fun_to_ident_folder(resolver).fold_item(renamed_ast).get(); + let resolver_fold = fun_to_ident_folder(resolver); + let resolved_ast = resolver_fold.fold_item(renamed_ast).get(); let resolved_as_str = pprust::item_to_str(resolved_ast, get_ident_interner()); assert_eq!(resolved_as_str,~"fn a2() -> int { let b = 13; b }"); + // try a double-rename, with pending_renames. + let a3_name = gensym("a3"); + let ctxt2 = new_rename(new_ident(a_name),a2_name,empty_ctxt); + let pending_renames = @mut ~[(new_ident(a_name),a2_name), + (ast::ident{name:a_name,ctxt:ctxt2},a3_name)]; + let double_renamed = renames_to_fold(pending_renames).fold_item(item_ast).get(); + let resolved_again = resolver_fold.fold_item(double_renamed).get(); + let double_renamed_as_str = pprust::item_to_str(resolved_again, + get_ident_interner()); + assert_eq!(double_renamed_as_str,~"fn a3() -> int { let b = 13; b }"); + + } + + fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) { + pprust::print_mod(s, &crate.module, crate.attrs); + } + + // "fn a() -> int { let b = 13; let c = b; b+c }" --> b & c should get new names, in the expr too. + // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" --> one should + // be renamed, one should not. + fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str { + let resolver = new_ident_resolver(); + let resolver_fold = fun_to_ident_folder(resolver); + let (crate_ast,ps) = string_to_crate_and_sess(crate_str); + // the cfg argument actually does matter, here... + let expanded_ast = expand_crate(ps,~[],crate_ast); + // std::io::println(fmt!("expanded: %?\n",expanded_ast)); + let resolved_ast = resolver_fold.fold_crate(expanded_ast); + pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner()) } - // sigh... it looks like I have two different renaming mechanisms, now... + #[test] + fn automatic_renaming () { + let teststrs = + ~[@"fn a() -> int { let b = 13; let c = b; b+c }", + @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"]; + for teststrs.iter().advance |s| { + std::io::println(expand_and_resolve_and_pretty_print(*s)); + } + } #[test] fn pat_idents(){ diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 44f479d0e49b7..8da67c3e1c9f1 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -543,9 +543,9 @@ pub fn gensym_ident(str : &str) -> ast::ident { // by using a gensym with a name that has a random number // at the end. So, the gensym guarantees the uniqueness, // and the int helps to avoid confusion. -pub fn fresh_name(src_name : &str) -> Name { +pub fn fresh_name(src_name : &ast::ident) -> Name { let num = rand::rng().gen_uint_range(0,0xffff); - gensym(fmt!("%s_%u",src_name,num)) + gensym(fmt!("%s_%u",ident_to_str(src_name),num)) } /** diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs index 9d286f1759eff..2d967d6ac3e94 100644 --- a/src/libsyntax/util/parser_testing.rs +++ b/src/libsyntax/util/parser_testing.rs @@ -40,12 +40,19 @@ fn with_error_checking_parse(s: @str, f: &fn(&mut Parser) -> T) -> T { x } +// parse a string, return a crate. pub fn string_to_crate (source_str : @str) -> @ast::Crate { do with_error_checking_parse(source_str) |p| { p.parse_crate_mod() } } +// parse a string, return a crate and the ParseSess +pub fn string_to_crate_and_sess (source_str : @str) -> (@ast::Crate,@mut ParseSess) { + let (p,ps) = string_to_parser_and_sess(source_str); + (p.parse_crate_mod(),ps) +} + // parse a string, return an expr pub fn string_to_expr (source_str : @str) -> @ast::expr { do with_error_checking_parse(source_str) |p| { @@ -60,14 +67,6 @@ pub fn string_to_item (source_str : @str) -> Option<@ast::item> { } } -// parse a string, return an item and the ParseSess -pub fn string_to_item_and_sess (source_str : @str) -> (Option<@ast::item>,@mut ParseSess) { - let (p,ps) = string_to_parser_and_sess(source_str); - let io = p.parse_item(~[]); - p.abort_if_errors(); - (io,ps) -} - // parse a string, return a stmt pub fn string_to_stmt(source_str : @str) -> @ast::stmt { do with_error_checking_parse(source_str) |p| { From 77fe4710114639181e51650570cdce8bd7ea179c Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 30 May 2013 17:46:25 -0700 Subject: [PATCH 07/67] one-line comment --- src/librustc/middle/resolve.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/librustc/middle/resolve.rs b/src/librustc/middle/resolve.rs index 1cd4df91e4d06..7df19f8a296ff 100644 --- a/src/librustc/middle/resolve.rs +++ b/src/librustc/middle/resolve.rs @@ -4507,6 +4507,7 @@ impl Resolver { return unqualified_def; } + // resolve a single identifier (used as a varref) pub fn resolve_identifier(@mut self, identifier: ident, namespace: Namespace, From 1f41b8e86e3395f30f93d616fb45ead72514bcd7 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 26 Jun 2013 15:56:13 -0700 Subject: [PATCH 08/67] update librustc to use name comparison in most cases, and mtwt_resolve comparison in others --- src/librustc/middle/resolve.rs | 168 +++++++++++---------- src/librustc/middle/trans/_match.rs | 4 +- src/librustc/middle/trans/consts.rs | 4 +- src/librustc/middle/trans/expr.rs | 5 +- src/librustc/middle/trans/meth.rs | 3 +- src/librustc/middle/ty.rs | 10 +- src/librustc/middle/typeck/check/method.rs | 13 +- src/librustc/middle/typeck/check/mod.rs | 32 ++-- src/librustc/middle/typeck/coherence.rs | 4 +- src/librustc/middle/typeck/collect.rs | 2 +- 10 files changed, 127 insertions(+), 118 deletions(-) diff --git a/src/librustc/middle/resolve.rs b/src/librustc/middle/resolve.rs index 7df19f8a296ff..6ec5badcdf7a4 100644 --- a/src/librustc/middle/resolve.rs +++ b/src/librustc/middle/resolve.rs @@ -22,7 +22,7 @@ use middle::pat_util::pat_bindings; use syntax::ast::*; use syntax::ast; -use syntax::ast_util::{def_id_of_def, local_def}; +use syntax::ast_util::{def_id_of_def, local_def}; // mtwt_resolve use syntax::ast_util::{path_to_ident, walk_pat, trait_method_to_ty_method}; use syntax::ast_util::{Privacy, Public, Private}; use syntax::ast_util::{variant_visibility_to_privacy, visibility_to_privacy}; @@ -53,7 +53,7 @@ pub struct binding_info { } // Map from the name in a pattern to its binding mode. -pub type BindingMap = HashMap; +pub type BindingMap = HashMap; // Trait method resolution pub type TraitMap = HashMap; @@ -297,7 +297,6 @@ pub fn Rib(kind: RibKind) -> Rib { } } - /// One import directive. pub struct ImportDirective { privacy: Privacy, @@ -413,12 +412,12 @@ pub struct Module { def_id: Option, kind: ModuleKind, - children: @mut HashMap, + children: @mut HashMap, imports: @mut ~[@ImportDirective], // The external module children of this node that were declared with // `extern mod`. - external_module_children: @mut HashMap, + external_module_children: @mut HashMap, // The anonymous children of this node. Anonymous children are pseudo- // modules that are implicitly created around items contained within @@ -437,7 +436,7 @@ pub struct Module { anonymous_children: @mut HashMap, // The status of resolving each import in this module. - import_resolutions: @mut HashMap, + import_resolutions: @mut HashMap, // The number of unresolved globs that this module exports. glob_count: uint, @@ -715,15 +714,14 @@ pub fn NameBindings() -> NameBindings { /// Interns the names of the primitive types. pub struct PrimitiveTypeTable { - primitive_types: HashMap, + primitive_types: HashMap, } impl PrimitiveTypeTable { pub fn intern(&mut self, string: &str, primitive_type: prim_ty) { - let ident = token::str_to_ident(string); - self.primitive_types.insert(ident, primitive_type); + self.primitive_types.insert(token::intern(string), primitive_type); } } @@ -827,7 +825,7 @@ pub struct Resolver { graph_root: @mut NameBindings, - method_map: @mut HashMap>, + method_map: @mut HashMap>, structs: HashSet, // The number of imports that are currently unresolved. @@ -964,10 +962,10 @@ impl Resolver { // Add or reuse the child. let new_parent = ModuleReducedGraphParent(module_); - match module_.children.find(&name) { + match module_.children.find(&name.name) { None => { let child = @mut NameBindings(); - module_.children.insert(name, child); + module_.children.insert(name.name, child); return (child, new_parent); } Some(&child) => { @@ -1233,7 +1231,7 @@ impl Resolver { } if path.idents.len() == 1 => { let name = path_to_ident(path); - let new_parent = match parent.children.find(&name) { + let new_parent = match parent.children.find(&name.name) { // It already exists Some(&child) if child.get_module_if_available() .is_some() && @@ -1347,7 +1345,7 @@ impl Resolver { match ty_m.explicit_self.node { sty_static => {} _ => { - method_names.insert(ident, ()); + method_names.insert(ident.name, ()); } } } @@ -1499,7 +1497,7 @@ impl Resolver { NormalModuleKind); parent.external_module_children.insert( - name, + name.name, external_module); self.build_reduced_graph_for_external_crate( @@ -1619,7 +1617,7 @@ impl Resolver { } ModuleParentLink(parent_module, ident) => { let name_bindings = parent_module.children.get( - &ident); + &ident.name); resolution.type_target = Some(Target(parent_module, *name_bindings)); } @@ -1628,7 +1626,7 @@ impl Resolver { debug!("(building reduced graph for external crate) \ ... creating import resolution"); - new_parent.import_resolutions.insert(ident, resolution); + new_parent.import_resolutions.insert(ident.name, resolution); } } } @@ -1669,7 +1667,7 @@ impl Resolver { // Add it to the trait info if not static. if explicit_self != sty_static { - interned_method_names.insert(method_name); + interned_method_names.insert(method_name.name); } } for interned_method_names.iter().advance |name| { @@ -1919,7 +1917,7 @@ impl Resolver { self.idents_to_str(directive.module_path), self.session.str_of(target)); - match module_.import_resolutions.find(&target) { + match module_.import_resolutions.find(&target.name) { Some(&resolution) => { debug!("(building import directive) bumping \ reference"); @@ -1934,7 +1932,7 @@ impl Resolver { debug!("(building import directive) creating new"); let resolution = @mut ImportResolution(privacy, id); resolution.outstanding_references = 1; - module_.import_resolutions.insert(target, resolution); + module_.import_resolutions.insert(target.name, resolution); } } } @@ -2212,7 +2210,7 @@ impl Resolver { let mut type_result = UnknownResult; // Search for direct children of the containing module. - match containing_module.children.find(&source) { + match containing_module.children.find(&source.name) { None => { // Continue. } @@ -2246,7 +2244,7 @@ impl Resolver { // Now search the exported imports within the containing // module. - match containing_module.import_resolutions.find(&source) { + match containing_module.import_resolutions.find(&source.name) { None => { // The containing module definitely doesn't have an // exported import with the name in question. We can @@ -2317,7 +2315,7 @@ impl Resolver { BoundResult(*) => {} _ => { match containing_module.external_module_children - .find(&source) { + .find(&source.name) { None => {} // Continue. Some(module) => { let name_bindings = @@ -2331,8 +2329,8 @@ impl Resolver { } // We've successfully resolved the import. Write the results in. - assert!(module_.import_resolutions.contains_key(&target)); - let import_resolution = module_.import_resolutions.get(&target); + assert!(module_.import_resolutions.contains_key(&target.name)); + let import_resolution = module_.import_resolutions.get(&target.name); match value_result { BoundResult(target_module, name_bindings) => { @@ -2495,15 +2493,15 @@ impl Resolver { } } - let merge_import_resolution = |ident, + let merge_import_resolution = |name, name_bindings: @mut NameBindings| { let dest_import_resolution; - match module_.import_resolutions.find(&ident) { + match module_.import_resolutions.find(&name) { None => { // Create a new import resolution from this child. dest_import_resolution = @mut ImportResolution(privacy, id); module_.import_resolutions.insert - (ident, dest_import_resolution); + (name, dest_import_resolution); } Some(&existing_import_resolution) => { dest_import_resolution = existing_import_resolution; @@ -2512,7 +2510,7 @@ impl Resolver { debug!("(resolving glob import) writing resolution `%s` in `%s` \ to `%s`, privacy=%?", - self.session.str_of(ident), + interner_get(name), self.module_to_str(containing_module), self.module_to_str(module_), dest_import_resolution.privacy); @@ -2531,16 +2529,16 @@ impl Resolver { }; // Add all children from the containing module. - for containing_module.children.iter().advance |(&ident, name_bindings)| { - merge_import_resolution(ident, *name_bindings); + for containing_module.children.iter().advance |(&name, name_bindings)| { + merge_import_resolution(name, *name_bindings); } // Add external module children from the containing module. for containing_module.external_module_children.iter().advance - |(&ident, module)| { + |(&name, module)| { let name_bindings = @mut Resolver::create_name_bindings_from_module(*module); - merge_import_resolution(ident, name_bindings); + merge_import_resolution(name, name_bindings); } debug!("(resolving glob import) successfully resolved import"); @@ -2766,7 +2764,7 @@ impl Resolver { // The current module node is handled specially. First, check for // its immediate children. - match module_.children.find(&name) { + match module_.children.find(&name.name) { Some(name_bindings) if name_bindings.defined_in_namespace(namespace) => { return Success(Target(module_, *name_bindings)); @@ -2778,7 +2776,7 @@ impl Resolver { // all its imports in the usual way; this is because chains of // adjacent import statements are processed as though they mutated the // current scope. - match module_.import_resolutions.find(&name) { + match module_.import_resolutions.find(&name.name) { None => { // Not found; continue. } @@ -2802,7 +2800,7 @@ impl Resolver { // Search for external modules. if namespace == TypeNS { - match module_.external_module_children.find(&name) { + match module_.external_module_children.find(&name.name) { None => {} Some(module) => { let name_bindings = @@ -2964,8 +2962,9 @@ impl Resolver { } } - /// Resolves a "module prefix". A module prefix is one of (a) `self::`; + /// Resolves a "module prefix". A module prefix is one or both of (a) `self::`; /// (b) some chain of `super::`. + /// grammar: (SELF MOD_SEP ) ? (SUPER MOD_SEP) * pub fn resolve_module_prefix(@mut self, module_: @mut Module, module_path: &[ident]) @@ -3020,7 +3019,7 @@ impl Resolver { self.module_to_str(module_)); // First, check the direct children of the module. - match module_.children.find(&name) { + match module_.children.find(&name.name) { Some(name_bindings) if name_bindings.defined_in_namespace(namespace) => { debug!("(resolving name in module) found node as child"); @@ -3041,7 +3040,7 @@ impl Resolver { } // Check the list of resolved imports. - match module_.import_resolutions.find(&name) { + match module_.import_resolutions.find(&name.name) { Some(import_resolution) => { if import_resolution.privacy == Public && import_resolution.outstanding_references != 0 { @@ -3076,7 +3075,7 @@ impl Resolver { // Finally, search through external children. if namespace == TypeNS { - match module_.external_module_children.find(&name) { + match module_.external_module_children.find(&name.name) { None => {} Some(module) => { let name_bindings = @@ -3200,7 +3199,7 @@ impl Resolver { pub fn add_exports_of_namebindings(@mut self, exports2: &mut ~[Export2], - ident: ident, + name: Name, namebindings: @mut NameBindings, ns: Namespace, reexport: bool) { @@ -3209,11 +3208,11 @@ impl Resolver { (Some(d), Some(Public)) => { debug!("(computing exports) YES: %s '%s' => %?", if reexport { ~"reexport" } else { ~"export"}, - self.session.str_of(ident), + interner_get(name), def_id_of_def(d)); exports2.push(Export2 { reexport: reexport, - name: self.session.str_of(ident), + name: interner_get(name), def_id: def_id_of_def(d) }); } @@ -3229,11 +3228,11 @@ impl Resolver { pub fn add_exports_for_module(@mut self, exports2: &mut ~[Export2], module_: @mut Module) { - for module_.import_resolutions.iter().advance |(ident, + for module_.import_resolutions.iter().advance |(name, importresolution)| { if importresolution.privacy != Public { debug!("(computing exports) not reexporting private `%s`", - self.session.str_of(*ident)); + interner_get(*name)); loop; } let xs = [TypeNS, ValueNS]; @@ -3241,9 +3240,9 @@ impl Resolver { match importresolution.target_for_namespace(*ns) { Some(target) => { debug!("(computing exports) maybe reexport '%s'", - self.session.str_of(*ident)); + interner_get(*name)); self.add_exports_of_namebindings(&mut *exports2, - *ident, + *name, target.bindings, *ns, true) @@ -3281,7 +3280,7 @@ impl Resolver { // Nothing to do. } Some(name) => { - match orig_module.children.find(&name) { + match orig_module.children.find(&name.name) { None => { debug!("!!! (with scope) didn't find `%s` in `%s`", self.session.str_of(name), @@ -3425,7 +3424,7 @@ impl Resolver { pub fn search_ribs(@mut self, ribs: &mut ~[@Rib], - name: ident, + name: Name, span: span, allow_capturing_self: AllowCapturingSelfFlag) -> Option { @@ -3435,7 +3434,7 @@ impl Resolver { let mut i = ribs.len(); while i != 0 { i -= 1; - match ribs[i].bindings.find(&name.name) { + match ribs[i].bindings.find(&name) { Some(&def_like) => { return self.upvarify(ribs, i, def_like, span, allow_capturing_self); @@ -3531,7 +3530,9 @@ impl Resolver { // Create a new rib for the self type. let self_type_rib = @Rib(NormalRibKind); self.type_ribs.push(self_type_rib); - self_type_rib.bindings.insert(self.type_self_ident.name, + // plain insert (no renaming) + let name = self.type_self_ident.name; + self_type_rib.bindings.insert(name, dl_def(def_self_ty(item.id))); // Create a new rib for the trait-wide type parameters. @@ -3671,6 +3672,7 @@ impl Resolver { // the item that bound it self.record_def(type_parameter.id, def_typaram_binder(node_id)); + // plain insert (no renaming) function_type_rib.bindings.insert(name.name, def_like); } } @@ -4007,8 +4009,8 @@ impl Resolver { pub fn binding_mode_map(@mut self, pat: @pat) -> BindingMap { let mut result = HashMap::new(); do pat_bindings(self.def_map, pat) |binding_mode, _id, sp, path| { - let ident = path_to_ident(path); - result.insert(ident, + let name = path_to_ident(path).name; // mtwt_resolve(path_to_ident(path)); + result.insert(name, binding_info {span: sp, binding_mode: binding_mode}); } @@ -4028,7 +4030,7 @@ impl Resolver { p.span, fmt!("variable `%s` from pattern #1 is \ not bound in pattern #%u", - self.session.str_of(key), i + 1)); + interner_get(key), i + 1)); } Some(binding_i) => { if binding_0.binding_mode != binding_i.binding_mode { @@ -4036,7 +4038,7 @@ impl Resolver { binding_i.span, fmt!("variable `%s` is bound with different \ mode in pattern #%u than in pattern #1", - self.session.str_of(key), i + 1)); + interner_get(key), i + 1)); } } } @@ -4048,7 +4050,7 @@ impl Resolver { binding.span, fmt!("variable `%s` from pattern #%u is \ not bound in pattern #1", - self.session.str_of(key), i + 1)); + interner_get(key), i + 1)); } } } @@ -4110,11 +4112,11 @@ impl Resolver { // First, check to see whether the name is a primitive type. if path.idents.len() == 1 { - let name = *path.idents.last(); + let id = *path.idents.last(); match self.primitive_type_table .primitive_types - .find(&name) { + .find(&id.name) { Some(&primitive_type) => { result_def = @@ -4192,7 +4194,7 @@ impl Resolver { mutability: Mutability, // Maps idents to the node ID for the (outermost) // pattern that binds them - bindings_list: Option<@mut HashMap>, + bindings_list: Option<@mut HashMap>, visitor: ResolveVisitor) { let pat_id = pattern.id; for walk_pat(pattern) |pattern| { @@ -4210,13 +4212,14 @@ impl Resolver { // what you want). let ident = path.idents[0]; + let renamed = ident.name; // mtwt_resolve(ident); match self.resolve_bare_identifier_pattern(ident) { FoundStructOrEnumVariant(def) if mode == RefutableMode => { debug!("(resolving pattern) resolving `%s` to \ struct or enum variant", - self.session.str_of(ident)); + interner_get(renamed)); self.enforce_default_binding_mode( pattern, @@ -4230,13 +4233,12 @@ impl Resolver { shadows an enum \ variant or unit-like \ struct in scope", - self.session - .str_of(ident))); + interner_get(renamed))); } FoundConst(def) if mode == RefutableMode => { debug!("(resolving pattern) resolving `%s` to \ constant", - self.session.str_of(ident)); + interner_get(renamed)); self.enforce_default_binding_mode( pattern, @@ -4251,7 +4253,7 @@ impl Resolver { } BareIdentifierPatternUnresolved => { debug!("(resolving pattern) binding `%s`", - self.session.str_of(ident)); + interner_get(renamed)); let is_mutable = mutability == Mutable; @@ -4286,16 +4288,16 @@ impl Resolver { match bindings_list { Some(bindings_list) - if !bindings_list.contains_key(&ident) => { + if !bindings_list.contains_key(&renamed) => { let this = &mut *self; let last_rib = this.value_ribs[ this.value_ribs.len() - 1]; - last_rib.bindings.insert(ident.name, + last_rib.bindings.insert(renamed, dl_def(def)); - bindings_list.insert(ident, pat_id); + bindings_list.insert(renamed, pat_id); } Some(b) => { - if b.find(&ident) == Some(&pat_id) { + if b.find(&renamed) == Some(&pat_id) { // Then this is a duplicate variable // in the same disjunct, which is an // error @@ -4311,7 +4313,7 @@ impl Resolver { let this = &mut *self; let last_rib = this.value_ribs[ this.value_ribs.len() - 1]; - last_rib.bindings.insert(ident.name, + last_rib.bindings.insert(renamed, dl_def(def)); } } @@ -4539,7 +4541,7 @@ impl Resolver { xray: XrayFlag) -> NameDefinition { // First, search children. - match containing_module.children.find(&name) { + match containing_module.children.find(&name.name) { Some(child_name_bindings) => { match (child_name_bindings.def_for_namespace(namespace), child_name_bindings.privacy_for_namespace(namespace)) { @@ -4562,7 +4564,7 @@ impl Resolver { } // Next, search import resolutions. - match containing_module.import_resolutions.find(&name) { + match containing_module.import_resolutions.find(&name.name) { Some(import_resolution) if import_resolution.privacy == Public || xray == Xray => { match (*import_resolution).target_for_namespace(namespace) { @@ -4590,7 +4592,7 @@ impl Resolver { // Finally, search through external children. if namespace == TypeNS { - match containing_module.external_module_children.find(&name) { + match containing_module.external_module_children.find(&name.name) { None => {} Some(module) => { match module.def_id { @@ -4637,9 +4639,9 @@ impl Resolver { } } - let name = *path.idents.last(); + let ident = *path.idents.last(); let def = match self.resolve_definition_of_name_in_module(containing_module, - name, + ident, namespace, xray) { NoNameDefinition => { @@ -4652,7 +4654,7 @@ impl Resolver { }; match containing_module.kind { TraitModuleKind | ImplModuleKind => { - match self.method_map.find(&name) { + match self.method_map.find(&ident.name) { Some(s) => { match containing_module.def_id { Some(def_id) if s.contains(&def_id) => { @@ -4729,12 +4731,14 @@ impl Resolver { let search_result; match namespace { ValueNS => { - search_result = self.search_ribs(self.value_ribs, ident, + let renamed = ident.name; // mtwt_resolve(ident); + search_result = self.search_ribs(self.value_ribs, renamed, span, DontAllowCapturingSelf); } TypeNS => { - search_result = self.search_ribs(self.type_ribs, ident, + let name = ident.name; + search_result = self.search_ribs(self.type_ribs, name, span, AllowCapturingSelf); } } @@ -4998,6 +5002,7 @@ impl Resolver { let this = &mut *self; let def_like = dl_def(def_label(expr.id)); let rib = this.label_ribs[this.label_ribs.len() - 1]; + // plain insert (no renaming) rib.bindings.insert(label.name, def_like); } @@ -5006,7 +5011,8 @@ impl Resolver { } expr_break(Some(label)) | expr_again(Some(label)) => { - match self.search_ribs(self.label_ribs, label, expr.span, + let name = label.name; + match self.search_ribs(self.label_ribs, name, expr.span, DontAllowCapturingSelf) { None => self.session.span_err(expr.span, @@ -5134,7 +5140,7 @@ impl Resolver { let mut found_traits = ~[]; let mut search_module = self.current_module; - match self.method_map.find(&name) { + match self.method_map.find(&name.name) { Some(candidate_traits) => loop { // Look for the current trait. match self.current_trait_refs { @@ -5358,7 +5364,7 @@ impl Resolver { debug!("Children:"); for module_.children.each_key |&name| { - debug!("* %s", self.session.str_of(name)); + debug!("* %s", interner_get(name)); } debug!("Import resolutions:"); @@ -5381,7 +5387,7 @@ impl Resolver { } } - debug!("* %s:%s%s", self.session.str_of(*name), + debug!("* %s:%s%s", interner_get(*name), value_repr, type_repr); } } diff --git a/src/librustc/middle/trans/_match.rs b/src/librustc/middle/trans/_match.rs index 648dd0332876b..d4c4a89704d13 100644 --- a/src/librustc/middle/trans/_match.rs +++ b/src/librustc/middle/trans/_match.rs @@ -1344,7 +1344,7 @@ fn compile_submatch_continue(mut bcx: @mut Block, let pat_repr = adt::represent_type(bcx.ccx(), pat_ty); do expr::with_field_tys(tcx, pat_ty, None) |discr, field_tys| { let rec_vals = rec_fields.map(|field_name| { - let ix = ty::field_idx_strict(tcx, *field_name, field_tys); + let ix = ty::field_idx_strict(tcx, field_name.name, field_tys); adt::trans_field_ptr(bcx, pat_repr, val, discr, ix) }); compile_submatch( @@ -2010,7 +2010,7 @@ fn bind_irrefutable_pat(bcx: @mut Block, let pat_repr = adt::represent_type(bcx.ccx(), pat_ty); do expr::with_field_tys(tcx, pat_ty, None) |discr, field_tys| { for fields.iter().advance |f| { - let ix = ty::field_idx_strict(tcx, f.ident, field_tys); + let ix = ty::field_idx_strict(tcx, f.ident.name, field_tys); let fldptr = adt::trans_field_ptr(bcx, pat_repr, val, discr, ix); bcx = bind_irrefutable_pat(bcx, f.pat, fldptr, binding_mode); diff --git a/src/librustc/middle/trans/consts.rs b/src/librustc/middle/trans/consts.rs index 853b3a3c28f1c..5c191ccf46bff 100644 --- a/src/librustc/middle/trans/consts.rs +++ b/src/librustc/middle/trans/consts.rs @@ -371,7 +371,7 @@ fn const_expr_unadjusted(cx: @mut CrateContext, e: &ast::expr) -> ValueRef { let brepr = adt::represent_type(cx, bt); let bv = const_expr(cx, base); do expr::with_field_tys(cx.tcx, bt, None) |discr, field_tys| { - let ix = ty::field_idx_strict(cx.tcx, field, field_tys); + let ix = ty::field_idx_strict(cx.tcx, field.name, field_tys); adt::const_get_field(cx, brepr, bv, discr, ix) } } @@ -492,7 +492,7 @@ fn const_expr_unadjusted(cx: @mut CrateContext, e: &ast::expr) -> ValueRef { do expr::with_field_tys(tcx, ety, Some(e.id)) |discr, field_tys| { let cs = field_tys.map(|field_ty| { - match fs.iter().find_(|f| field_ty.ident == f.ident) { + match fs.iter().find_(|f| field_ty.ident.name == f.ident.name) { Some(f) => const_expr(cx, (*f).expr), None => { cx.tcx.sess.span_bug(e.span, "missing struct field"); diff --git a/src/librustc/middle/trans/expr.rs b/src/librustc/middle/trans/expr.rs index c038ca710aa4f..ba58c94eefdeb 100644 --- a/src/librustc/middle/trans/expr.rs +++ b/src/librustc/middle/trans/expr.rs @@ -852,7 +852,7 @@ fn trans_lvalue_unadjusted(bcx: @mut Block, expr: @ast::expr) -> DatumBlock { let base_datum = unpack_datum!(bcx, trans_to_datum(bcx, base)); let repr = adt::represent_type(bcx.ccx(), base_datum.ty); do with_field_tys(bcx.tcx(), base_datum.ty, None) |discr, field_tys| { - let ix = ty::field_idx_strict(bcx.tcx(), field, field_tys); + let ix = ty::field_idx_strict(bcx.tcx(), field.name, field_tys); DatumBlock { datum: do base_datum.get_element(bcx, field_tys[ix].mt.ty, @@ -1139,7 +1139,8 @@ fn trans_rec_or_struct(bcx: @mut Block, let mut need_base = vec::from_elem(field_tys.len(), true); let numbered_fields = do fields.map |field| { - let opt_pos = field_tys.iter().position(|field_ty| field_ty.ident == field.ident); + let opt_pos = field_tys.iter().position(|field_ty| + field_ty.ident.name == field.ident.name); match opt_pos { Some(i) => { need_base[i] = false; diff --git a/src/librustc/middle/trans/meth.rs b/src/librustc/middle/trans/meth.rs index 9228f20513bb6..50a2fb28ff421 100644 --- a/src/librustc/middle/trans/meth.rs +++ b/src/librustc/middle/trans/meth.rs @@ -280,6 +280,7 @@ pub fn trans_static_method_callee(bcx: @mut Block, pub fn method_with_name(ccx: &mut CrateContext, impl_id: ast::def_id, name: ast::ident) -> ast::def_id { + // NOTE : SHOULDN'T THIS USE NAME? let meth_id_opt = ccx.impl_method_cache.find_copy(&(impl_id, name)); match meth_id_opt { Some(m) => return m, @@ -288,7 +289,7 @@ pub fn method_with_name(ccx: &mut CrateContext, let imp = ccx.tcx.impls.find(&impl_id) .expect("could not find impl while translating"); - let meth = imp.methods.iter().find_(|m| m.ident == name) + let meth = imp.methods.iter().find_(|m| m.ident.name == name.name) .expect("could not find method while translating"); ccx.impl_method_cache.insert((impl_id, name), meth.def_id); diff --git a/src/librustc/middle/ty.rs b/src/librustc/middle/ty.rs index 9b3df349f1e0c..da8116758462e 100644 --- a/src/librustc/middle/ty.rs +++ b/src/librustc/middle/ty.rs @@ -3269,19 +3269,19 @@ pub fn stmt_node_id(s: &ast::stmt) -> ast::node_id { } } -pub fn field_idx(id: ast::ident, fields: &[field]) -> Option { +pub fn field_idx(id: ast::Name, fields: &[field]) -> Option { let mut i = 0u; - for fields.iter().advance |f| { if f.ident == id { return Some(i); } i += 1u; } + for fields.iter().advance |f| { if f.ident.name == id { return Some(i); } i += 1u; } return None; } -pub fn field_idx_strict(tcx: ty::ctxt, id: ast::ident, fields: &[field]) +pub fn field_idx_strict(tcx: ty::ctxt, id: ast::Name, fields: &[field]) -> uint { let mut i = 0u; - for fields.iter().advance |f| { if f.ident == id { return i; } i += 1u; } + for fields.iter().advance |f| { if f.ident.name == id { return i; } i += 1u; } tcx.sess.bug(fmt!( "No field named `%s` found in the list of fields `%?`", - tcx.sess.str_of(id), + token::interner_get(id), fields.map(|f| tcx.sess.str_of(f.ident)))); } diff --git a/src/librustc/middle/typeck/check/method.rs b/src/librustc/middle/typeck/check/method.rs index a4d93586327d7..94257da10571b 100644 --- a/src/librustc/middle/typeck/check/method.rs +++ b/src/librustc/middle/typeck/check/method.rs @@ -105,6 +105,7 @@ use syntax::ast::{sty_uniq, sty_static, node_id}; use syntax::ast::{m_const, m_mutbl, m_imm}; use syntax::ast; use syntax::ast_map; +use syntax::parse::token; #[deriving(Eq)] pub enum CheckTraitsFlag { @@ -126,7 +127,7 @@ pub fn lookup( self_expr: @ast::expr, // The expression `a`. callee_id: node_id, /* Where to store `a.b`'s type, * also the scope of the call */ - m_name: ast::ident, // The ident `b`. + m_name: ast::Name, // The name `b`. self_ty: ty::t, // The type of `a`. supplied_tps: &[ty::t], // The list of types X, Y, ... . deref_args: check::DerefArgs, // Whether we autopointer first. @@ -158,7 +159,7 @@ pub struct LookupContext<'self> { expr: @ast::expr, self_expr: @ast::expr, callee_id: node_id, - m_name: ast::ident, + m_name: ast::Name, supplied_tps: &'self [ty::t], impl_dups: @mut HashSet, inherent_candidates: @mut ~[Candidate], @@ -341,7 +342,7 @@ impl<'self> LookupContext<'self> { let tcx = self.tcx(); let ms = ty::trait_methods(tcx, did); - let index = match ms.iter().position(|m| m.ident == self.m_name) { + let index = match ms.iter().position(|m| m.ident.name == self.m_name) { Some(i) => i, None => { return; } // no method with the right name }; @@ -431,7 +432,7 @@ impl<'self> LookupContext<'self> { let pos = { match trait_methods.iter().position(|m| { m.explicit_self != ast::sty_static && - m.ident == self.m_name }) + m.ident.name == self.m_name }) { Some(pos) => pos, None => { @@ -479,12 +480,12 @@ impl<'self> LookupContext<'self> { return; // already visited } debug!("push_candidates_from_impl: %s %s %s", - self.m_name.repr(self.tcx()), + token::interner_get(self.m_name), impl_info.ident.repr(self.tcx()), impl_info.methods.map(|m| m.ident).repr(self.tcx())); let idx = { - match impl_info.methods.iter().position(|m| m.ident == self.m_name) { + match impl_info.methods.iter().position(|m| m.ident.name == self.m_name) { Some(idx) => idx, None => { return; } // No method with the right name. } diff --git a/src/librustc/middle/typeck/check/mod.rs b/src/librustc/middle/typeck/check/mod.rs index cffceee88d3e7..ae6b3a85c2694 100644 --- a/src/librustc/middle/typeck/check/mod.rs +++ b/src/librustc/middle/typeck/check/mod.rs @@ -1135,10 +1135,10 @@ pub fn impl_self_ty(vcx: &VtableContext, pub fn lookup_field_ty(tcx: ty::ctxt, class_id: ast::def_id, items: &[ty::field_ty], - fieldname: ast::ident, + fieldname: ast::Name, substs: &ty::substs) -> Option { - let o_field = items.iter().find_(|f| f.ident == fieldname); + let o_field = items.iter().find_(|f| f.ident.name == fieldname); do o_field.map() |f| { ty::lookup_field_type(tcx, class_id, f.id, substs) } @@ -1419,7 +1419,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, expr, rcvr, callee_id, - method_name, + method_name.name, expr_t, tps, DontDerefArgs, @@ -1503,7 +1503,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, op_ex: @ast::expr, self_ex: @ast::expr, self_t: ty::t, - opname: ast::ident, + opname: ast::Name, args: ~[@ast::expr], deref_args: DerefArgs, autoderef_receiver: AutoderefReceiverFlag, @@ -1643,7 +1643,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, lhs_resolved_t, None) }; return lookup_op_method(fcx, callee_id, ex, lhs_expr, lhs_resolved_t, - fcx.tcx().sess.ident_of(*name), + token::intern(*name), ~[rhs], DoDerefArgs, DontAutoderefReceiver, if_op_unbound, expected_result); } @@ -1677,7 +1677,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, -> ty::t { lookup_op_method( fcx, callee_id, ex, rhs_expr, rhs_t, - fcx.tcx().sess.ident_of(mname), ~[], + token::intern(mname), ~[], DoDerefArgs, DontAutoderefReceiver, || { fcx.type_error_message(ex.span, |actual| { @@ -1803,7 +1803,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, fn check_field(fcx: @mut FnCtxt, expr: @ast::expr, base: @ast::expr, - field: ast::ident, + field: ast::Name, tys: &[ast::Ty]) { let tcx = fcx.ccx.tcx; let bot = check_expr(fcx, base); @@ -1851,7 +1851,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, |actual| { fmt!("attempted to take value of method `%s` on type `%s` \ (try writing an anonymous function)", - tcx.sess.str_of(field), actual) + token::interner_get(field), actual) }, expr_t, None); } @@ -1862,7 +1862,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, |actual| { fmt!("attempted access of field `%s` on type `%s`, \ but no field with that name was found", - tcx.sess.str_of(field), actual) + token::interner_get(field), actual) }, expr_t, None); } @@ -1884,7 +1884,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, let mut class_field_map = HashMap::new(); let mut fields_found = 0; for field_types.iter().advance |field| { - class_field_map.insert(field.ident, (field.id, false)); + class_field_map.insert(field.ident.name, (field.id, false)); } let mut error_happened = false; @@ -1893,7 +1893,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, for ast_fields.iter().advance |field| { let mut expected_field_type = ty::mk_err(); - let pair = class_field_map.find(&field.ident). + let pair = class_field_map.find(&field.ident.name). map_consume(|x| *x); match pair { None => { @@ -1915,7 +1915,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, ty::lookup_field_type( tcx, class_id, field_id, &substitutions); class_field_map.insert( - field.ident, (field_id, true)); + field.ident.name, (field_id, true)); fields_found += 1; } } @@ -1937,11 +1937,11 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, if fields_found < field_types.len() { let mut missing_fields = ~[]; for field_types.iter().advance |class_field| { - let name = class_field.ident; + let name = class_field.ident.name; let (_, seen) = *class_field_map.get(&name); if !seen { missing_fields.push( - ~"`" + tcx.sess.str_of(name) + "`"); + ~"`" + token::interner_get(name) + "`"); } } @@ -2822,7 +2822,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, } } ast::expr_field(base, field, ref tys) => { - check_field(fcx, expr, base, field, *tys); + check_field(fcx, expr, base, field.name, *tys); } ast::expr_index(callee_id, base, idx) => { check_expr(fcx, base); @@ -2862,7 +2862,7 @@ pub fn check_expr_with_unifier(fcx: @mut FnCtxt, expr, base, resolved, - index_ident, + index_ident.name, ~[idx], DoDerefArgs, AutoderefReceiver, diff --git a/src/librustc/middle/typeck/coherence.rs b/src/librustc/middle/typeck/coherence.rs index 4298f043e935d..fda1a1cfe1fdf 100644 --- a/src/librustc/middle/typeck/coherence.rs +++ b/src/librustc/middle/typeck/coherence.rs @@ -552,13 +552,13 @@ impl CoherenceChecker { let mut provided_names = HashSet::new(); // Implemented methods for uint::range(0, all_methods.len()) |i| { - provided_names.insert(all_methods[i].ident); + provided_names.insert(all_methods[i].ident.name); } let r = ty::trait_methods(tcx, trait_did); for r.iter().advance |method| { debug!("checking for %s", method.ident.repr(tcx)); - if provided_names.contains(&method.ident) { loop; } + if provided_names.contains(&method.ident.name) { loop; } tcx.sess.span_err(trait_ref_span, fmt!("missing method `%s`", diff --git a/src/librustc/middle/typeck/collect.rs b/src/librustc/middle/typeck/collect.rs index 98b4de9d719a8..71a73b6a5ff8a 100644 --- a/src/librustc/middle/typeck/collect.rs +++ b/src/librustc/middle/typeck/collect.rs @@ -675,7 +675,7 @@ pub fn check_methods_against_trait(ccx: &CrateCtxt, // we'll catch it in coherence let trait_ms = ty::trait_methods(tcx, trait_ref.def_id); for impl_ms.iter().advance |impl_m| { - match trait_ms.iter().find_(|trait_m| trait_m.ident == impl_m.mty.ident) { + match trait_ms.iter().find_(|trait_m| trait_m.ident.name == impl_m.mty.ident.name) { Some(trait_m) => { let num_impl_tps = generics.ty_params.len(); compare_impl_method( From 0b85e4c2249f84ad014f20ba30da920b72c43472 Mon Sep 17 00:00:00 2001 From: John Clements Date: Mon, 8 Jul 2013 15:52:51 -0700 Subject: [PATCH 09/67] make ifn macro non-capturing --- src/librustc/middle/trans/base.rs | 130 +++++++++++++++--------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/src/librustc/middle/trans/base.rs b/src/librustc/middle/trans/base.rs index b9318b62a2754..e77b884211f19 100644 --- a/src/librustc/middle/trans/base.rs +++ b/src/librustc/middle/trans/base.rs @@ -2675,10 +2675,10 @@ pub fn p2i(ccx: &CrateContext, v: ValueRef) -> ValueRef { } macro_rules! ifn ( - ($name:expr, $args:expr, $ret:expr) => ({ + ($intrinsics:ident, $name:expr, $args:expr, $ret:expr) => ({ let name = $name; let f = decl_cdecl_fn(llmod, name, Type::func($args, &$ret)); - intrinsics.insert(name, f); + $intrinsics.insert(name, f); }) ) @@ -2686,81 +2686,81 @@ pub fn declare_intrinsics(llmod: ModuleRef) -> HashMap<&'static str, ValueRef> { let i8p = Type::i8p(); let mut intrinsics = HashMap::new(); - ifn!("llvm.memcpy.p0i8.p0i8.i32", + ifn!(intrinsics, "llvm.memcpy.p0i8.p0i8.i32", [i8p, i8p, Type::i32(), Type::i32(), Type::i1()], Type::void()); - ifn!("llvm.memcpy.p0i8.p0i8.i64", + ifn!(intrinsics, "llvm.memcpy.p0i8.p0i8.i64", [i8p, i8p, Type::i64(), Type::i32(), Type::i1()], Type::void()); - ifn!("llvm.memmove.p0i8.p0i8.i32", + ifn!(intrinsics, "llvm.memmove.p0i8.p0i8.i32", [i8p, i8p, Type::i32(), Type::i32(), Type::i1()], Type::void()); - ifn!("llvm.memmove.p0i8.p0i8.i64", + ifn!(intrinsics, "llvm.memmove.p0i8.p0i8.i64", [i8p, i8p, Type::i64(), Type::i32(), Type::i1()], Type::void()); - ifn!("llvm.memset.p0i8.i32", + ifn!(intrinsics, "llvm.memset.p0i8.i32", [i8p, Type::i8(), Type::i32(), Type::i32(), Type::i1()], Type::void()); - ifn!("llvm.memset.p0i8.i64", + ifn!(intrinsics, "llvm.memset.p0i8.i64", [i8p, Type::i8(), Type::i64(), Type::i32(), Type::i1()], Type::void()); - ifn!("llvm.trap", [], Type::void()); - ifn!("llvm.frameaddress", [Type::i32()], i8p); - - ifn!("llvm.powi.f32", [Type::f32(), Type::i32()], Type::f32()); - ifn!("llvm.powi.f64", [Type::f64(), Type::i32()], Type::f64()); - ifn!("llvm.pow.f32", [Type::f32(), Type::f32()], Type::f32()); - ifn!("llvm.pow.f64", [Type::f64(), Type::f64()], Type::f64()); - - ifn!("llvm.sqrt.f32", [Type::f32()], Type::f32()); - ifn!("llvm.sqrt.f64", [Type::f64()], Type::f64()); - ifn!("llvm.sin.f32", [Type::f32()], Type::f32()); - ifn!("llvm.sin.f64", [Type::f64()], Type::f64()); - ifn!("llvm.cos.f32", [Type::f32()], Type::f32()); - ifn!("llvm.cos.f64", [Type::f64()], Type::f64()); - ifn!("llvm.exp.f32", [Type::f32()], Type::f32()); - ifn!("llvm.exp.f64", [Type::f64()], Type::f64()); - ifn!("llvm.exp2.f32", [Type::f32()], Type::f32()); - ifn!("llvm.exp2.f64", [Type::f64()], Type::f64()); - ifn!("llvm.log.f32", [Type::f32()], Type::f32()); - ifn!("llvm.log.f64", [Type::f64()], Type::f64()); - ifn!("llvm.log10.f32",[Type::f32()], Type::f32()); - ifn!("llvm.log10.f64",[Type::f64()], Type::f64()); - ifn!("llvm.log2.f32", [Type::f32()], Type::f32()); - ifn!("llvm.log2.f64", [Type::f64()], Type::f64()); - - ifn!("llvm.fma.f32", [Type::f32(), Type::f32(), Type::f32()], Type::f32()); - ifn!("llvm.fma.f64", [Type::f64(), Type::f64(), Type::f64()], Type::f64()); - - ifn!("llvm.fabs.f32", [Type::f32()], Type::f32()); - ifn!("llvm.fabs.f64", [Type::f64()], Type::f64()); - ifn!("llvm.floor.f32",[Type::f32()], Type::f32()); - ifn!("llvm.floor.f64",[Type::f64()], Type::f64()); - ifn!("llvm.ceil.f32", [Type::f32()], Type::f32()); - ifn!("llvm.ceil.f64", [Type::f64()], Type::f64()); - ifn!("llvm.trunc.f32",[Type::f32()], Type::f32()); - ifn!("llvm.trunc.f64",[Type::f64()], Type::f64()); - - ifn!("llvm.ctpop.i8", [Type::i8()], Type::i8()); - ifn!("llvm.ctpop.i16",[Type::i16()], Type::i16()); - ifn!("llvm.ctpop.i32",[Type::i32()], Type::i32()); - ifn!("llvm.ctpop.i64",[Type::i64()], Type::i64()); - - ifn!("llvm.ctlz.i8", [Type::i8() , Type::i1()], Type::i8()); - ifn!("llvm.ctlz.i16", [Type::i16(), Type::i1()], Type::i16()); - ifn!("llvm.ctlz.i32", [Type::i32(), Type::i1()], Type::i32()); - ifn!("llvm.ctlz.i64", [Type::i64(), Type::i1()], Type::i64()); - - ifn!("llvm.cttz.i8", [Type::i8() , Type::i1()], Type::i8()); - ifn!("llvm.cttz.i16", [Type::i16(), Type::i1()], Type::i16()); - ifn!("llvm.cttz.i32", [Type::i32(), Type::i1()], Type::i32()); - ifn!("llvm.cttz.i64", [Type::i64(), Type::i1()], Type::i64()); - - ifn!("llvm.bswap.i16",[Type::i16()], Type::i16()); - ifn!("llvm.bswap.i32",[Type::i32()], Type::i32()); - ifn!("llvm.bswap.i64",[Type::i64()], Type::i64()); + ifn!(intrinsics, "llvm.trap", [], Type::void()); + ifn!(intrinsics, "llvm.frameaddress", [Type::i32()], i8p); + + ifn!(intrinsics, "llvm.powi.f32", [Type::f32(), Type::i32()], Type::f32()); + ifn!(intrinsics, "llvm.powi.f64", [Type::f64(), Type::i32()], Type::f64()); + ifn!(intrinsics, "llvm.pow.f32", [Type::f32(), Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.pow.f64", [Type::f64(), Type::f64()], Type::f64()); + + ifn!(intrinsics, "llvm.sqrt.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.sqrt.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.sin.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.sin.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.cos.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.cos.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.exp.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.exp.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.exp2.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.exp2.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.log.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.log.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.log10.f32",[Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.log10.f64",[Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.log2.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.log2.f64", [Type::f64()], Type::f64()); + + ifn!(intrinsics, "llvm.fma.f32", [Type::f32(), Type::f32(), Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.fma.f64", [Type::f64(), Type::f64(), Type::f64()], Type::f64()); + + ifn!(intrinsics, "llvm.fabs.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.fabs.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.floor.f32",[Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.floor.f64",[Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.ceil.f32", [Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.ceil.f64", [Type::f64()], Type::f64()); + ifn!(intrinsics, "llvm.trunc.f32",[Type::f32()], Type::f32()); + ifn!(intrinsics, "llvm.trunc.f64",[Type::f64()], Type::f64()); + + ifn!(intrinsics, "llvm.ctpop.i8", [Type::i8()], Type::i8()); + ifn!(intrinsics, "llvm.ctpop.i16",[Type::i16()], Type::i16()); + ifn!(intrinsics, "llvm.ctpop.i32",[Type::i32()], Type::i32()); + ifn!(intrinsics, "llvm.ctpop.i64",[Type::i64()], Type::i64()); + + ifn!(intrinsics, "llvm.ctlz.i8", [Type::i8() , Type::i1()], Type::i8()); + ifn!(intrinsics, "llvm.ctlz.i16", [Type::i16(), Type::i1()], Type::i16()); + ifn!(intrinsics, "llvm.ctlz.i32", [Type::i32(), Type::i1()], Type::i32()); + ifn!(intrinsics, "llvm.ctlz.i64", [Type::i64(), Type::i1()], Type::i64()); + + ifn!(intrinsics, "llvm.cttz.i8", [Type::i8() , Type::i1()], Type::i8()); + ifn!(intrinsics, "llvm.cttz.i16", [Type::i16(), Type::i1()], Type::i16()); + ifn!(intrinsics, "llvm.cttz.i32", [Type::i32(), Type::i1()], Type::i32()); + ifn!(intrinsics, "llvm.cttz.i64", [Type::i64(), Type::i1()], Type::i64()); + + ifn!(intrinsics, "llvm.bswap.i16",[Type::i16()], Type::i16()); + ifn!(intrinsics, "llvm.bswap.i32",[Type::i32()], Type::i32()); + ifn!(intrinsics, "llvm.bswap.i64",[Type::i64()], Type::i64()); return intrinsics; } pub fn declare_dbg_intrinsics(llmod: ModuleRef, intrinsics: &mut HashMap<&'static str, ValueRef>) { - ifn!("llvm.dbg.declare", [Type::metadata(), Type::metadata()], Type::void()); - ifn!("llvm.dbg.value", [Type::metadata(), Type::i64(), Type::metadata()], Type::void()); + ifn!(intrinsics, "llvm.dbg.declare", [Type::metadata(), Type::metadata()], Type::void()); + ifn!(intrinsics, "llvm.dbg.value", [Type::metadata(), Type::i64(), Type::metadata()], Type::void()); } pub fn trap(bcx: @mut Block) { From 9511cad16f3f702456786cc9658e6f10086e1f72 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 26 Jun 2013 10:37:25 -0700 Subject: [PATCH 10/67] resolve test case resolve must ignore syntax context when comparing module names --- src/test/run-pass/hygiene-dodging-1.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/test/run-pass/hygiene-dodging-1.rs diff --git a/src/test/run-pass/hygiene-dodging-1.rs b/src/test/run-pass/hygiene-dodging-1.rs new file mode 100644 index 0000000000000..20ffa74e68775 --- /dev/null +++ b/src/test/run-pass/hygiene-dodging-1.rs @@ -0,0 +1,11 @@ +mod x { + pub fn g() -> uint {14} +} + +fn main(){ + // should *not* shadow the module x: + let x = 9; + // use it to avoid warnings: + x+3; + assert_eq!(x::g(),14); +} From 868e03259ad0337cb1bb5389691d57d1b1c7e314 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 6 Jun 2013 11:42:34 -0700 Subject: [PATCH 11/67] flip the switch on let renaming --- src/libsyntax/ext/expand.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index f9f16ba312823..d5abd0c7965a5 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -303,7 +303,7 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, } } } - _ => return orig(s, sp, fld) + _ => return expand_non_macro_stmt(*extsbox,s,sp,fld,orig) }; if (pth.idents.len() > 1u) { cx.span_fatal( From 858b15cd7cdb18782af02c88974500cba984989d Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 6 Jun 2013 11:21:02 -0700 Subject: [PATCH 12/67] renaming test cases --- src/libsyntax/ext/expand.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index d5abd0c7965a5..8a32c498f2211 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1022,10 +1022,6 @@ mod test { pprust::print_mod(s, &crate.module, crate.attrs); } - // "fn a() -> int { let b = 13; let c = b; b+c }" --> b & c should get new names, in the expr too. - // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" --> one should - // be renamed, one should not. - fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str { let resolver = new_ident_resolver(); let resolver_fold = fun_to_ident_folder(resolver); @@ -1039,10 +1035,20 @@ mod test { #[test] fn automatic_renaming () { + // "fn a() -> int { let b = 13; let c = b; b+c }" + // --> b & c should get new names, in the expr too. + // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" + // --> one should be renamed, one should not. + let teststrs = - ~[@"fn a() -> int { let b = 13; let c = b; b+c }", - @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"]; + ~[// b & c should get new names throughout, in the expr too: + @"fn a() -> int { let b = 13; let c = b; b+c }", + // the use of b before the + should be renamed, the other one not: + @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}", + // the b before the plus should not be renamed (requires marks) + @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}"]; for teststrs.iter().advance |s| { + // we need regexps to test these! std::io::println(expand_and_resolve_and_pretty_print(*s)); } } From baab2e0f85a9cd999780fa127c2bbb896876be88 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 6 Jun 2013 11:14:29 -0700 Subject: [PATCH 13/67] comments in ast.rs --- src/libsyntax/ast.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index e17d9192ddee5..8e1c901540fe0 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -543,6 +543,7 @@ pub enum token_tree { // a delimited sequence (the delimiters appear as the first // and last elements of the vector) tt_delim(@mut ~[token_tree]), + // These only make sense for right-hand-sides of MBE macros: // a kleene-style repetition sequence with a span, a tt_forest, @@ -621,6 +622,10 @@ pub enum matcher_ { pub type mac = spanned; +// represents a macro invocation. The Path indicates which macro +// is being invoked, and the vector of token-trees contains the source +// of the macro invocation. +// There's only one flavor, now, so this could presumably be simplified. #[deriving(Clone, Eq, Encodable, Decodable, IterBytes)] pub enum mac_ { mac_invoc_tt(Path,~[token_tree]), // new macro-invocation From 5279ec62cc182b9982af5053fbf30c832e67064c Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 28 May 2013 14:53:38 -0700 Subject: [PATCH 14/67] removed unneccessary SyntaxExpander structs --- src/libsyntax/ext/base.rs | 21 +++++---------------- src/libsyntax/ext/expand.rs | 28 ++++++++++++---------------- src/libsyntax/ext/tt/macro_rules.rs | 2 +- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index a5c8f2a235ed9..6ec4a944d1178 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -22,8 +22,7 @@ use std::hashmap::HashMap; // new-style macro! tt code: // -// SyntaxExpanderTT, SyntaxExpanderTTItem, MacResult, -// NormalTT, IdentTT +// MacResult, NormalTT, IdentTT // // also note that ast::mac used to have a bunch of extraneous cases and // is now probably a redundant AST node, can be merged with @@ -40,21 +39,11 @@ pub type ItemDecorator = @fn(@ExtCtxt, ~[@ast::item]) -> ~[@ast::item]; -pub struct SyntaxExpanderTT { - expander: SyntaxExpanderTTFun, - span: Option -} - pub type SyntaxExpanderTTFun = @fn(@ExtCtxt, span, &[ast::token_tree]) -> MacResult; -pub struct SyntaxExpanderTTItem { - expander: SyntaxExpanderTTItemFun, - span: Option -} - pub type SyntaxExpanderTTItemFun = @fn(@ExtCtxt, span, ast::ident, @@ -76,7 +65,7 @@ pub enum SyntaxExtension { ItemDecorator(ItemDecorator), // Token-tree expanders - NormalTT(SyntaxExpanderTT), + NormalTT(SyntaxExpanderTTFun, Option), // An IdentTT is a macro that has an // identifier in between the name of the @@ -86,7 +75,7 @@ pub enum SyntaxExtension { // perhaps macro_rules! will lose its odd special identifier argument, // and this can go away also - IdentTT(SyntaxExpanderTTItem), + IdentTT(SyntaxExpanderTTItemFun, Option), } // The SyntaxEnv is the environment that's threaded through the expansion @@ -121,11 +110,11 @@ type RenameList = ~[(ast::ident,Name)]; pub fn syntax_expander_table() -> SyntaxEnv { // utility function to simplify creating NormalTT syntax extensions fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> @Transformer { - @SE(NormalTT(SyntaxExpanderTT{expander: f, span: None})) + @SE(NormalTT(f, None)) } // utility function to simplify creating IdentTT syntax extensions fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> @Transformer { - @SE(IdentTT(SyntaxExpanderTTItem{expander: f, span: None})) + @SE(IdentTT(f, None)) } let mut syntax_expanders = HashMap::new(); // NB identifier starts with space, and can't conflict with legal idents diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 8a32c498f2211..3297fe4e2aa95 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -56,19 +56,16 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, pth.span, fmt!("macro undefined: '%s'", extnamestr)) } - Some(@SE(NormalTT(SyntaxExpanderTT{ - expander: exp, - span: exp_sp - }))) => { + Some(@SE(NormalTT(expandfun, exp_span))) => { cx.bt_push(ExpnInfo { call_site: s, callee: NameAndSpan { name: extnamestr, - span: exp_sp, + span: exp_span, }, }); - let expanded = match exp(cx, mac.span, *tts) { + let expanded = match expandfun(cx, mac.span, *tts) { MRExpr(e) => e, MRAny(expr_maker,_,_) => expr_maker(), _ => { @@ -220,7 +217,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, None => cx.span_fatal(pth.span, fmt!("macro undefined: '%s!'", extnamestr)), - Some(@SE(NormalTT(ref expand))) => { + Some(@SE(NormalTT(expander, span))) => { if it.ident != parse::token::special_idents::invalid { cx.span_fatal(pth.span, fmt!("macro %s! expects no ident argument, \ @@ -231,12 +228,12 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, call_site: it.span, callee: NameAndSpan { name: extnamestr, - span: expand.span + span: span } }); - ((*expand).expander)(cx, it.span, tts) + expander(cx, it.span, tts) } - Some(@SE(IdentTT(ref expand))) => { + Some(@SE(IdentTT(expander, span))) => { if it.ident == parse::token::special_idents::invalid { cx.span_fatal(pth.span, fmt!("macro %s! expects an ident argument", @@ -246,10 +243,10 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, call_site: it.span, callee: NameAndSpan { name: extnamestr, - span: expand.span + span: span } }); - ((*expand).expander)(cx, it.span, it.ident, tts) + expander(cx, it.span, it.ident, tts) } _ => cx.span_fatal( it.span, fmt!("%s! is not legal in item position", extnamestr)) @@ -317,13 +314,12 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, None => cx.span_fatal(pth.span, fmt!("macro undefined: '%s'", extnamestr)), - Some(@SE(NormalTT( - SyntaxExpanderTT{expander: exp, span: exp_sp}))) => { + Some(@SE(NormalTT(expandfun, exp_span))) => { cx.bt_push(ExpnInfo { call_site: sp, - callee: NameAndSpan { name: extnamestr, span: exp_sp } + callee: NameAndSpan { name: extnamestr, span: exp_span } }); - let expanded = match exp(cx, mac.span, tts) { + let expanded = match expandfun(cx, mac.span, tts) { MRExpr(e) => @codemap::spanned { node: stmt_expr(e, cx.next_id()), span: e.span}, diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 4b3c8498380cc..818783b91ffae 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -149,6 +149,6 @@ pub fn add_new_extension(cx: @ExtCtxt, return MRDef(MacroDef{ name: ident_to_str(&name), - ext: NormalTT(base::SyntaxExpanderTT{expander: exp, span: Some(sp)}) + ext: NormalTT(exp, Some(sp)) }); } From 29bb204a8138d7c2a275de44259806d95b53a113 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 28 May 2013 14:55:50 -0700 Subject: [PATCH 15/67] separate ItemDecorator from ItemDecorator --- src/libsyntax/ext/base.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 6ec4a944d1178..bf07848b2b566 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -33,7 +33,7 @@ pub struct MacroDef { ext: SyntaxExtension } -pub type ItemDecorator = @fn(@ExtCtxt, +pub type ItemDecoratorFun = @fn(@ExtCtxt, span, @ast::MetaItem, ~[@ast::item]) @@ -62,7 +62,7 @@ pub enum MacResult { pub enum SyntaxExtension { // #[auto_encode] and such - ItemDecorator(ItemDecorator), + ItemDecorator(ItemDecoratorFun), // Token-tree expanders NormalTT(SyntaxExpanderTTFun, Option), From cff793f8b83fe54cbd9585e00a99bee0db594145 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 28 May 2013 15:13:57 -0700 Subject: [PATCH 16/67] refactor so tt_fold only requires an ident->ident fn --- src/libsyntax/fold.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index f27e68641e3af..c4589d754dc7a 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -118,35 +118,37 @@ fn fold_mac_(m: &mac, fld: @ast_fold) -> mac { node: match m.node { mac_invoc_tt(ref p,ref tts) => mac_invoc_tt(fld.fold_path(p), - fold_tts(*tts,fld)) + fold_tts(*tts,|id|{fld.fold_ident(id)})) }, span: fld.new_span(m.span) } } -fn fold_tts(tts : &[token_tree], fld: @ast_fold) -> ~[token_tree] { +// build a new vector of tts by appling the given function to +// all of the identifiers in the token trees. +pub fn fold_tts(tts : &[token_tree], f: @fn(ident)->ident) -> ~[token_tree] { do tts.map |tt| { match *tt { tt_tok(span, ref tok) => - tt_tok(span,maybe_fold_ident(tok,fld)), + tt_tok(span,maybe_fold_ident(tok,f)), tt_delim(ref tts) => - tt_delim(@mut fold_tts(**tts, fld)), + tt_delim(@mut fold_tts(**tts, f)), tt_seq(span, ref pattern, ref sep, is_optional) => tt_seq(span, - @mut fold_tts(**pattern, fld), - sep.map(|tok|maybe_fold_ident(tok,fld)), + @mut fold_tts(**pattern, f), + sep.map(|tok|maybe_fold_ident(tok,f)), is_optional), tt_nonterminal(sp,ref ident) => - tt_nonterminal(sp,fld.fold_ident(*ident)) + tt_nonterminal(sp,f(*ident)) } } } // apply ident folder if it's an ident, otherwise leave it alone -fn maybe_fold_ident(t: &token::Token, fld: @ast_fold) -> token::Token { +fn maybe_fold_ident(t : &token::Token, f: @fn(ident)->ident) -> token::Token { match *t { token::IDENT(id,followed_by_colons) => - token::IDENT(fld.fold_ident(id),followed_by_colons), + token::IDENT(f(id),followed_by_colons), _ => (*t).clone() } } From 5c956454794f2539c7ec947613d2652d054fe0f9 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 25 Jun 2013 11:43:52 -0700 Subject: [PATCH 17/67] adding test case to check marking/unmarking --- src/libsyntax/ext/expand.rs | 24 ++++++++++++++++-------- src/libsyntax/parse/token.rs | 7 ++++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 3297fe4e2aa95..30fe7339192a6 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -65,7 +65,9 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, }, }); - let expanded = match expandfun(cx, mac.span, *tts) { + let fm = fresh_mark(); + let marked_tts = mark_tts(*tts,fm); + let expanded = match expandfun(cx, mac.span, marked_tts) { MRExpr(e) => e, MRAny(expr_maker,_,_) => expr_maker(), _ => { @@ -100,6 +102,12 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, } } +// apply a fresh mark to the given token trees. Used prior to expansion of a macro. +fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] { + fold_tts(tts,new_ident_marker(m)) +} + + // This is a secondary mechanism for invoking syntax extensions on items: // "decorator" attributes, such as #[auto_encode]. These are invoked by an // attribute prefixing an item, and are interpreted by feeding the item @@ -855,7 +863,7 @@ pub fn new_ident_renamer(from: ast::ident, // update the ctxts in a path to get a mark node -pub fn new_ident_marker(mark: uint) -> +pub fn new_ident_marker(mark: Mrk) -> @fn(ast::ident)->ast::ident { |id : ast::ident| ast::ident{ @@ -1031,18 +1039,18 @@ mod test { #[test] fn automatic_renaming () { - // "fn a() -> int { let b = 13; let c = b; b+c }" - // --> b & c should get new names, in the expr too. - // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" - // --> one should be renamed, one should not. - let teststrs = ~[// b & c should get new names throughout, in the expr too: @"fn a() -> int { let b = 13; let c = b; b+c }", // the use of b before the + should be renamed, the other one not: @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}", // the b before the plus should not be renamed (requires marks) - @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}"]; + @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}", + // the z flows into and out of two macros (g & f) along one path, and one (just g) along the + // other, so the result of the whole thing should be "let z_123 = 3; z_123" + @"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) + fn a(){g!(z)}" + ]; for teststrs.iter().advance |s| { // we need regexps to test these! std::io::println(expand_and_resolve_and_pretty_print(*s)); diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 8da67c3e1c9f1..012b22fc235fc 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -9,7 +9,7 @@ // except according to those terms. use ast; -use ast::Name; +use ast::{Name, Mrk}; use ast_util; use parse::token; use util::interner::StrInterner; @@ -548,6 +548,11 @@ pub fn fresh_name(src_name : &ast::ident) -> Name { gensym(fmt!("%s_%u",ident_to_str(src_name),num)) } +// create a fresh mark. +pub fn fresh_mark() -> Mrk { + gensym("mark") +} + /** * All the valid words that have meaning in the Rust language. * From 03be3c885f0804cd41ca138cda578dad785e438e Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 29 May 2013 16:21:04 -0700 Subject: [PATCH 18/67] comments only --- src/libsyntax/ast.rs | 1 + src/libsyntax/ast_util.rs | 2 ++ src/libsyntax/ext/expand.rs | 2 ++ src/libsyntax/ext/tt/macro_rules.rs | 6 ++++++ 4 files changed, 11 insertions(+) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 8e1c901540fe0..d675b05bcccab 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -617,6 +617,7 @@ pub enum matcher_ { // lo, hi position-in-match-array used: match_seq(~[matcher], Option<::parse::token::Token>, bool, uint, uint), // parse a Rust NT: name to bind, name of NT, position in match array: + // NOTE: 'name of NT' shouldnt really be represented as an ident, should it? match_nonterminal(ident, ident, uint) } diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index 676424508588d..2855946fc09fc 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -27,6 +27,8 @@ pub fn path_name_i(idents: &[ident]) -> ~str { idents.map(|i| token::interner_get(i.name)).connect("::") } +// totally scary function: ignores all but the last element, should have +// a different name pub fn path_to_ident(p: &Path) -> ident { *p.idents.last() } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 30fe7339192a6..97ac4e1de9eb1 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1050,6 +1050,8 @@ mod test { // other, so the result of the whole thing should be "let z_123 = 3; z_123" @"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) fn a(){g!(z)}" + // create a really evil test case where a $x appears inside a binding of $x but *shouldnt* + // bind because it was inserted by a different macro.... ]; for teststrs.iter().advance |s| { // we need regexps to test these! diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 818783b91ffae..5631f0377975b 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -23,11 +23,15 @@ use parse::token::{get_ident_interner, special_idents, gensym_ident, ident_to_st use parse::token::{FAT_ARROW, SEMI, nt_matchers, nt_tt}; use print; +// this procedure performs the expansion of the +// macro_rules! macro. It parses the RHS and adds +// an extension to the current context. pub fn add_new_extension(cx: @ExtCtxt, sp: span, name: ident, arg: ~[ast::token_tree]) -> base::MacResult { + // Wrap a matcher_ in a spanned to produce a matcher. // these spans won't matter, anyways fn ms(m: matcher_) -> matcher { spanned { @@ -39,11 +43,13 @@ pub fn add_new_extension(cx: @ExtCtxt, let lhs_nm = gensym_ident("lhs"); let rhs_nm = gensym_ident("rhs"); + // The pattern that macro_rules matches. // The grammar for macro_rules! is: // $( $lhs:mtcs => $rhs:tt );+ // ...quasiquoting this would be nice. let argument_gram = ~[ ms(match_seq(~[ + // NOTE : probably just use an enum for the NT_name ? ms(match_nonterminal(lhs_nm, special_idents::matchers, 0u)), ms(match_tok(FAT_ARROW)), ms(match_nonterminal(rhs_nm, special_idents::tt, 1u)), From e00fd377532719d4028bdd5755b1fb4af4a3f266 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 6 Jun 2013 18:09:31 -0700 Subject: [PATCH 19/67] remove FIXME #2888, now bug is fixed --- src/libsyntax/ext/expand.rs | 24 ++++++++++++------------ src/libsyntax/fold.rs | 5 ++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 97ac4e1de9eb1..7c122a22ddc4e 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -21,7 +21,7 @@ use ext::base::*; use fold::*; use parse; use parse::{parse_item_from_source_str}; -use parse::token::{fresh_name, ident_to_str, intern}; +use parse::token::{fresh_mark, fresh_name, ident_to_str, intern}; use visit; use visit::Visitor; @@ -1026,16 +1026,14 @@ mod test { pprust::print_mod(s, &crate.module, crate.attrs); } - fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str { - let resolver = new_ident_resolver(); - let resolver_fold = fun_to_ident_folder(resolver); - let (crate_ast,ps) = string_to_crate_and_sess(crate_str); + //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str { + //let (crate_ast,ps) = string_to_crate_and_sess(crate_str); // the cfg argument actually does matter, here... - let expanded_ast = expand_crate(ps,~[],crate_ast); + //let expanded_ast = expand_crate(ps,~[],crate_ast); // std::io::println(fmt!("expanded: %?\n",expanded_ast)); - let resolved_ast = resolver_fold.fold_crate(expanded_ast); - pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner()) - } + //let resolved_ast = mtwt_resolve_crate(expanded_ast); + //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner()) + //} #[test] fn automatic_renaming () { @@ -1046,16 +1044,18 @@ mod test { @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}", // the b before the plus should not be renamed (requires marks) @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}", + // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm + // commenting it out. // the z flows into and out of two macros (g & f) along one path, and one (just g) along the // other, so the result of the whole thing should be "let z_123 = 3; z_123" - @"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) - fn a(){g!(z)}" + //@"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) + // fn a(){g!(z)}" // create a really evil test case where a $x appears inside a binding of $x but *shouldnt* // bind because it was inserted by a different macro.... ]; for teststrs.iter().advance |s| { // we need regexps to test these! - std::io::println(expand_and_resolve_and_pretty_print(*s)); + //std::io::println(expand_and_resolve_and_pretty_print(*s)); } } diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index c4589d754dc7a..9a7fd03812a2d 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -326,9 +326,8 @@ pub fn noop_fold_item_underscore(i: &item_, fld: @ast_fold) -> item_ { ) } item_mac(ref m) => { - // FIXME #2888: we might actually want to do something here. - // ... okay, we're doing something. It would probably be nicer - // to add something to the ast_fold trait, but I'll defer + // It would probably be nicer + // to expose this in the ast_fold trait, but I'll defer // that work. item_mac(fold_mac_(m,fld)) } From 6ebd2efc30f95869d2fa1f11e7b244adbdfb1a63 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 10:39:59 -0700 Subject: [PATCH 20/67] added FIXME comment --- src/libsyntax/ast.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index d675b05bcccab..671fa7a92d8bf 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -20,6 +20,10 @@ use std::option::Option; use std::to_str::ToStr; use extra::serialize::{Encodable, Decodable, Encoder, Decoder}; + +// FIXME #6993: in librustc, uses of "ident" should be replaced +// by just "Name". + // an identifier contains a Name (index into the interner // table) and a SyntaxContext to track renaming and // macro expansion per Flatt et al., "Macros From 754a85844b1ecaedbea18006ae0bd8c166118e3b Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 10:41:18 -0700 Subject: [PATCH 21/67] drop back to a simple gensym approach for fresh-name. this is necessary so that the new idents are connected to the original strings. this is important both for error messages, and so that top-level refs get connected to the right things. --- src/libsyntax/parse/token.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 012b22fc235fc..426a14413c240 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -534,18 +534,9 @@ pub fn gensym_ident(str : &str) -> ast::ident { ast::new_ident(gensym(str)) } - -// create a fresh name. In principle, this is just a -// gensym, but for debugging purposes, you'd like the -// resulting name to have a suggestive stringify, without -// paying the cost of guaranteeing that the name is -// truly unique. I'm going to try to strike a balance -// by using a gensym with a name that has a random number -// at the end. So, the gensym guarantees the uniqueness, -// and the int helps to avoid confusion. -pub fn fresh_name(src_name : &ast::ident) -> Name { - let num = rand::rng().gen_uint_range(0,0xffff); - gensym(fmt!("%s_%u",ident_to_str(src_name),num)) +// create a fresh name that maps to the same string as the old one. +pub fn fresh_name(src : &ast::ident) -> Name { + gensym(ident_to_str(src)) } // create a fresh mark. From 0f4d24f8701307cbd23517d2a3dd1cb1c800f487 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 10:41:38 -0700 Subject: [PATCH 22/67] comments --- src/libsyntax/ast.rs | 1 + src/libsyntax/ext/expand.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 671fa7a92d8bf..c3816b03fae51 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -84,6 +84,7 @@ pub enum SyntaxContext_ { // in the "from" slot. In essence, they're all // pointers to a single "rename" event node. Rename (ident,Name,SyntaxContext), + // actually, IllegalCtxt may not be necessary. IllegalCtxt() } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 7c122a22ddc4e..bdb945a652633 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -398,6 +398,8 @@ fn expand_non_macro_stmt (exts: SyntaxEnv, let expanded_pat = fld.fold_pat(pat); // find the pat_idents in the pattern: // oh dear heaven... this is going to include the enum names, as well.... + // ... but that should be okay, as long as the new names are gensyms + // for the old ones. let idents = @mut ~[]; ((*name_finder).visit_pat) (expanded_pat, (idents, @@ -1037,6 +1039,7 @@ mod test { #[test] fn automatic_renaming () { + // need some other way to test these... let teststrs = ~[// b & c should get new names throughout, in the expr too: @"fn a() -> int { let b = 13; let c = b; b+c }", From 4432f89e454bcf355eebb87048eafc7167e59104 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 3 Jul 2013 11:34:01 -0700 Subject: [PATCH 23/67] make comparison of special_idents non-hygienic --- src/libsyntax/ext/expand.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index bdb945a652633..738cfbdd700cc 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -226,7 +226,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, fmt!("macro undefined: '%s!'", extnamestr)), Some(@SE(NormalTT(expander, span))) => { - if it.ident != parse::token::special_idents::invalid { + if it.ident.name != parse::token::special_idents::invalid.name { cx.span_fatal(pth.span, fmt!("macro %s! expects no ident argument, \ given '%s'", extnamestr, @@ -242,7 +242,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, expander(cx, it.span, tts) } Some(@SE(IdentTT(expander, span))) => { - if it.ident == parse::token::special_idents::invalid { + if it.ident.name == parse::token::special_idents::invalid.name { cx.span_fatal(pth.span, fmt!("macro %s! expects an ident argument", extnamestr)); From 832f6ee082138961446ca501cf757f5419b6da19 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 12:26:34 -0700 Subject: [PATCH 24/67] use empty_ctxt to simplify downstream --- src/libsyntax/ext/expand.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 738cfbdd700cc..72dafc3862305 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Block, Crate, decl_local, expr_, expr_mac, Local, mac_invoc_tt}; +use ast::{Block, Crate, decl_local, empty_ctxt, expr_, expr_mac, Local, mac_invoc_tt}; use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi}; use ast::{SCTable, token_tree, illegal_ctxt}; use ast; @@ -876,12 +876,14 @@ pub fn new_ident_marker(mark: Mrk) -> // perform resolution (in the MTWT sense) on all of the // idents in the tree. This is the final step in expansion. +// FIXME #6993: this function could go away, along with +// the separate mtwt_resolution pass pub fn new_ident_resolver() -> @fn(ast::ident)->ast::ident { |id : ast::ident| ast::ident { name : mtwt_resolve(id), - ctxt : illegal_ctxt + ctxt : empty_ctxt } } From 2b6de57b2f7634d0f27782978afc03108d9da75a Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 12:28:03 -0700 Subject: [PATCH 25/67] test case work --- src/libsyntax/ext/expand.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 72dafc3862305..bd9a9fe807581 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1030,12 +1030,15 @@ mod test { pprust::print_mod(s, &crate.module, crate.attrs); } - //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str { + //fn expand_and_resolve(crate_str: @str) -> ast::crate { //let (crate_ast,ps) = string_to_crate_and_sess(crate_str); // the cfg argument actually does matter, here... //let expanded_ast = expand_crate(ps,~[],crate_ast); // std::io::println(fmt!("expanded: %?\n",expanded_ast)); - //let resolved_ast = mtwt_resolve_crate(expanded_ast); + //mtwt_resolve_crate(expanded_ast) + //} + //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str { + //let resolved_ast = expand_and_resolve(crate_str); //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner()) //} @@ -1072,4 +1075,9 @@ mod test { ((*pat_idents).visit_pat)(pat, (idents, mk_vt(pat_idents))); assert_eq!(idents,@mut strs_to_idents(~["a","c","b","d"])); } + +/* #[test] + fn debugging(){ + io::println(fmt!("%?",expand_and_resolve(@~"fn main () { let x : int = 13;}"))) + }*/ } From c616b664c6a8ceba5aa0b07791a0cf6d2efab323 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 14:53:53 -0700 Subject: [PATCH 26/67] re-add debug version --- src/libsyntax/parse/token.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 426a14413c240..3de7e5346643f 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -537,6 +537,11 @@ pub fn gensym_ident(str : &str) -> ast::ident { // create a fresh name that maps to the same string as the old one. pub fn fresh_name(src : &ast::ident) -> Name { gensym(ident_to_str(src)) + // following: debug version. Could work in final except that it's incompatible with + // good error messages and uses of struct names in ambiguous could-be-binding + // locations. + /*let num = rand::rng().gen_uint_range(0,0xffff); + gensym(fmt!("%s_%u",ident_to_str(src),num))*/ } // create a fresh mark. From c0dda0ff5c6c263ac5cd81882b667d6659bf66dc Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 14:54:48 -0700 Subject: [PATCH 27/67] re-add lost call to expand_block_elts --- src/libsyntax/ext/expand.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index bd9a9fe807581..d8aab61e26e88 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -494,7 +494,8 @@ pub fn expand_block(extsbox: @mut SyntaxEnv, orig: @fn(&Block, @ast_fold) -> Block) -> Block { // see note below about treatment of exts table - with_exts_frame!(extsbox,false,orig(blk,fld)) + with_exts_frame!(extsbox,false, + expand_block_elts(*extsbox, blk, fld)) } From 62112dd9d1e89a8959931053c5084ef82e1cd056 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 15:01:28 -0700 Subject: [PATCH 28/67] test case work --- src/libsyntax/ext/expand.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index d8aab61e26e88..68efc511aa064 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1049,6 +1049,8 @@ mod test { let teststrs = ~[// b & c should get new names throughout, in the expr too: @"fn a() -> int { let b = 13; let c = b; b+c }", + // both x's should be renamed (how is this causing a bug?) + @"fn main () {let x : int = 13;x;}", // the use of b before the + should be renamed, the other one not: @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}", // the b before the plus should not be renamed (requires marks) @@ -1077,8 +1079,4 @@ mod test { assert_eq!(idents,@mut strs_to_idents(~["a","c","b","d"])); } -/* #[test] - fn debugging(){ - io::println(fmt!("%?",expand_and_resolve(@~"fn main () { let x : int = 13;}"))) - }*/ } From e373801d274fe548fef7e6ac9c8b92be9572198c Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 16:00:41 -0700 Subject: [PATCH 29/67] test case support fns, remove debugging test case --- src/libsyntax/ext/expand.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 68efc511aa064..fb8ce8e003846 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1031,10 +1031,14 @@ mod test { pprust::print_mod(s, &crate.module, crate.attrs); } - //fn expand_and_resolve(crate_str: @str) -> ast::crate { - //let (crate_ast,ps) = string_to_crate_and_sess(crate_str); + fn expand_crate_str(crate_str: @str) -> @ast::Crate { + let (crate_ast,ps) = string_to_crate_and_sess(crate_str); // the cfg argument actually does matter, here... - //let expanded_ast = expand_crate(ps,~[],crate_ast); + expand_crate(ps,~[],crate_ast) + } + + //fn expand_and_resolve(crate_str: @str) -> ast::crate { + //let expanded_ast = expand_crate_str(crate_str); // std::io::println(fmt!("expanded: %?\n",expanded_ast)); //mtwt_resolve_crate(expanded_ast) //} From e11a4f6b365e6d072c9ce89e9dbae91d56003666 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 17:06:57 -0700 Subject: [PATCH 30/67] new test that uncovers bug in fold --- src/libsyntax/fold.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 9a7fd03812a2d..c1961e653f093 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -1015,4 +1015,15 @@ mod test { token::get_ident_interner()), ~"zz!zz((zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+)))"); } + + // and in cast expressions... this appears to be an existing bug. + #[test] fn ident_transformation_in_types () { + let zz_fold = fun_to_ident_folder(to_zz()); + let ast = string_to_crate(@"fn a() {let z = 13 as int;}"); + assert_pred!(matches_codepattern, + "matches_codepattern", + pprust::to_str(&zz_fold.fold_crate(ast),fake_print_crate, + token::get_ident_interner()), + ~"fn zz(){let zz=13 as zz;}"); + } } From da0815002e8496d1f239e088eb49d9d49f644196 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 7 Jun 2013 17:14:06 -0700 Subject: [PATCH 31/67] fixed bug in fold's traversal of cast exprs --- src/libsyntax/fold.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index c1961e653f093..adf251ec48313 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -547,7 +547,7 @@ pub fn noop_fold_expr(e: &expr_, fld: @ast_fold) -> expr_ { expr_do_body(f) => expr_do_body(fld.fold_expr(f)), expr_lit(_) => (*e).clone(), expr_cast(expr, ref ty) => { - expr_cast(fld.fold_expr(expr), (*ty).clone()) + expr_cast(fld.fold_expr(expr), fld.fold_ty(ty)) } expr_addr_of(m, ohs) => expr_addr_of(m, fld.fold_expr(ohs)), expr_if(cond, ref tr, fl) => { From 4c9f1b3f14b8df03859983bc929a96ae164ed501 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 25 Jun 2013 16:48:03 -0700 Subject: [PATCH 32/67] added test for ptr_eq on fresh_name-generated idents --- src/libsyntax/parse/token.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 3de7e5346643f..ce2533e8f99ab 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -15,10 +15,12 @@ use parse::token; use util::interner::StrInterner; use util::interner; +use std::char; use std::cmp::Equiv; use std::local_data; use std::rand; use std::rand::RngUtil; +use std::ptr::to_unsafe_ptr; #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)] pub enum binop { @@ -535,15 +537,31 @@ pub fn gensym_ident(str : &str) -> ast::ident { } // create a fresh name that maps to the same string as the old one. +// note that this guarantees that ptr_eq(ident_to_str(src),interner_get(fresh_name(src))); +// that is, that the new name and the old one are connected to ptr_eq strings. pub fn fresh_name(src : &ast::ident) -> Name { gensym(ident_to_str(src)) // following: debug version. Could work in final except that it's incompatible with // good error messages and uses of struct names in ambiguous could-be-binding - // locations. + // locations. Also definitely destroys the guarantee given above about ptr_eq. /*let num = rand::rng().gen_uint_range(0,0xffff); gensym(fmt!("%s_%u",ident_to_str(src),num))*/ } +// it looks like there oughta be a str_ptr_eq fn, but no one bothered to implement it? +pub fn str_ptr_eq(a: @str, b: @str) -> bool { + // doesn't compile! ...because of rebase mangling. this should be fixed + // in the commit that follows this. + let (a_ptr, b_ptr): (*uint, *uint) = (to_unsafe_ptr(a), to_unsafe_ptr(b)); + a_ptr == b_ptr +} + + + +// return true when two identifiers refer (through the intern table) to the same ptr_eq +// string. This is used to compare identifiers in places where hygienic comparison is +// not wanted (i.e. not lexical vars). + // create a fresh mark. pub fn fresh_mark() -> Mrk { gensym("mark") @@ -698,6 +716,8 @@ mod test { use ast; use ast_util; use super::*; + use std::io; + use std::managed; fn mark_ident(id : ast::ident, m : ast::Mrk) -> ast::ident { ast::ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)} @@ -710,4 +730,13 @@ mod test { assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false))); } + #[test] fn t1() { + let ghi = str_to_ident("ghi"); + assert_eq!(ident_to_str(&ghi),@"ghi"); + let fresh = ast::new_ident(fresh_name(&ghi)); + assert_eq!(ident_to_str(&fresh),@"ghi"); + assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&fresh))); + assert_eq!(3,4); + } + } From 5c5de2bc97c475ca9d8a71dc53a0ca0e4fb3cf61 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 26 Jun 2013 10:11:19 -0700 Subject: [PATCH 33/67] added gensym_copy mechanism to ensure sharing of pointers in the interner this makes comparisons constant-time, and enables spelling-comparison of identifiers, crucial in many parts of resolve. --- src/libsyntax/parse/token.rs | 73 +++++++++++++++------------------- src/libsyntax/util/interner.rs | 63 +++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 48 deletions(-) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index ce2533e8f99ab..2d520c299ffc6 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -15,12 +15,12 @@ use parse::token; use util::interner::StrInterner; use util::interner; +use std::cast; use std::char; use std::cmp::Equiv; use std::local_data; use std::rand; use std::rand::RngUtil; -use std::ptr::to_unsafe_ptr; #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)] pub enum binop { @@ -376,30 +376,8 @@ pub fn token_to_binop(tok: &Token) -> Option { } } -pub struct ident_interner { - priv interner: StrInterner, -} - -impl ident_interner { - pub fn intern(&self, val: &str) -> Name { - self.interner.intern(val) - } - pub fn gensym(&self, val: &str) -> Name { - self.interner.gensym(val) - } - pub fn get(&self, idx: Name) -> @str { - self.interner.get(idx) - } - // is this really something that should be exposed? - pub fn len(&self) -> uint { - self.interner.len() - } - pub fn find_equiv>(&self, val: &Q) - -> Option { - self.interner.find_equiv(val) - } -} - +// looks like we can get rid of this completely... +pub type ident_interner = StrInterner; // return a fresh interner, preloaded with special identifiers. fn mk_fresh_ident_interner() -> @ident_interner { @@ -478,9 +456,7 @@ fn mk_fresh_ident_interner() -> @ident_interner { "be", // 65 ]; - @ident_interner { - interner: interner::StrInterner::prefill(init_vec) - } + @interner::StrInterner::prefill(init_vec) } // if an interner exists in TLS, return it. Otherwise, prepare a @@ -501,7 +477,7 @@ pub fn get_ident_interner() -> @ident_interner { /* for when we don't care about the contents; doesn't interact with TLD or serialization */ pub fn mk_fake_ident_interner() -> @ident_interner { - @ident_interner { interner: interner::StrInterner::new() } + @interner::StrInterner::new() } // maps a string to its interned representation @@ -537,10 +513,11 @@ pub fn gensym_ident(str : &str) -> ast::ident { } // create a fresh name that maps to the same string as the old one. -// note that this guarantees that ptr_eq(ident_to_str(src),interner_get(fresh_name(src))); +// note that this guarantees that str_ptr_eq(ident_to_str(src),interner_get(fresh_name(src))); // that is, that the new name and the old one are connected to ptr_eq strings. pub fn fresh_name(src : &ast::ident) -> Name { - gensym(ident_to_str(src)) + let interner = get_ident_interner(); + interner.gensym_copy(src.name) // following: debug version. Could work in final except that it's incompatible with // good error messages and uses of struct names in ambiguous could-be-binding // locations. Also definitely destroys the guarantee given above about ptr_eq. @@ -549,18 +526,26 @@ pub fn fresh_name(src : &ast::ident) -> Name { } // it looks like there oughta be a str_ptr_eq fn, but no one bothered to implement it? -pub fn str_ptr_eq(a: @str, b: @str) -> bool { - // doesn't compile! ...because of rebase mangling. this should be fixed - // in the commit that follows this. - let (a_ptr, b_ptr): (*uint, *uint) = (to_unsafe_ptr(a), to_unsafe_ptr(b)); - a_ptr == b_ptr -} - +// determine whether two @str values are pointer-equal +pub fn str_ptr_eq(a : @str, b : @str) -> bool { + unsafe { + let p : uint = cast::transmute(a); + let q : uint = cast::transmute(b); + let result = p == q; + // got to transmute them back, to make sure the ref count is correct: + let junk1 : @str = cast::transmute(p); + let junk2 : @str = cast::transmute(q); + result + } +} // return true when two identifiers refer (through the intern table) to the same ptr_eq // string. This is used to compare identifiers in places where hygienic comparison is // not wanted (i.e. not lexical vars). +pub fn ident_spelling_eq(a : &ast::ident, b : &ast::ident) -> bool { + str_ptr_eq(interner_get(a.name),interner_get(b.name)) +} // create a fresh mark. pub fn fresh_mark() -> Mrk { @@ -730,13 +715,21 @@ mod test { assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false))); } - #[test] fn t1() { + #[test] fn str_ptr_eq_tests(){ + let a = @"abc"; + let b = @"abc"; + let c = a; + assert!(str_ptr_eq(a,c)); + assert!(!str_ptr_eq(a,b)); + } + + #[test] fn fresh_name_pointer_sharing() { let ghi = str_to_ident("ghi"); assert_eq!(ident_to_str(&ghi),@"ghi"); + assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&ghi))) let fresh = ast::new_ident(fresh_name(&ghi)); assert_eq!(ident_to_str(&fresh),@"ghi"); assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&fresh))); - assert_eq!(3,4); } } diff --git a/src/libsyntax/util/interner.rs b/src/libsyntax/util/interner.rs index 014186c9ff415..7bc5e04e8e132 100644 --- a/src/libsyntax/util/interner.rs +++ b/src/libsyntax/util/interner.rs @@ -117,6 +117,23 @@ impl StrInterner { new_idx } + // I want these gensyms to share name pointers + // with existing entries. This would be automatic, + // except that the existing gensym creates its + // own managed ptr using to_managed. I think that + // adding this utility function is the most + // lightweight way to get what I want, though not + // necessarily the cleanest. + + // create a gensym with the same name as an existing + // entry. + pub fn gensym_copy(&self, idx : uint) -> uint { + let new_idx = self.len(); + // leave out of map to avoid colliding + self.vect.push(self.vect[idx]); + new_idx + } + // this isn't "pure" in the traditional sense, because it can go from // failing to returning a value as items are interned. But for typestate, // where we first check a pred and then rely on it, ceasing to fail is ok. @@ -144,23 +161,23 @@ mod tests { } #[test] - fn i2 () { + fn interner_tests () { let i : Interner<@str> = Interner::new(); // first one is zero: - assert_eq!(i.intern (@"dog"), 0); + assert_eq!(i.intern(@"dog"), 0); // re-use gets the same entry: - assert_eq!(i.intern (@"dog"), 0); + assert_eq!(i.intern(@"dog"), 0); // different string gets a different #: - assert_eq!(i.intern (@"cat"), 1); - assert_eq!(i.intern (@"cat"), 1); + assert_eq!(i.intern(@"cat"), 1); + assert_eq!(i.intern(@"cat"), 1); // dog is still at zero - assert_eq!(i.intern (@"dog"), 0); + assert_eq!(i.intern(@"dog"), 0); // gensym gets 3 - assert_eq!(i.gensym (@"zebra" ), 2); + assert_eq!(i.gensym(@"zebra" ), 2); // gensym of same string gets new number : assert_eq!(i.gensym (@"zebra" ), 3); // gensym of *existing* string gets new number: - assert_eq!(i.gensym (@"dog"), 4); + assert_eq!(i.gensym(@"dog"), 4); assert_eq!(i.get(0), @"dog"); assert_eq!(i.get(1), @"cat"); assert_eq!(i.get(2), @"zebra"); @@ -176,4 +193,34 @@ mod tests { assert_eq!(i.get(2), @"Carol"); assert_eq!(i.intern(@"Bob"), 1); } + + #[test] + fn string_interner_tests() { + let i : StrInterner = StrInterner::new(); + // first one is zero: + assert_eq!(i.intern("dog"), 0); + // re-use gets the same entry: + assert_eq!(i.intern ("dog"), 0); + // different string gets a different #: + assert_eq!(i.intern("cat"), 1); + assert_eq!(i.intern("cat"), 1); + // dog is still at zero + assert_eq!(i.intern("dog"), 0); + // gensym gets 3 + assert_eq!(i.gensym("zebra"), 2); + // gensym of same string gets new number : + assert_eq!(i.gensym("zebra"), 3); + // gensym of *existing* string gets new number: + assert_eq!(i.gensym("dog"), 4); + // gensym tests again with gensym_copy: + assert_eq!(i.gensym_copy(2), 5); + assert_eq!(i.get(5), @"zebra"); + assert_eq!(i.gensym_copy(2), 6); + assert_eq!(i.get(6), @"zebra"); + assert_eq!(i.get(0), @"dog"); + assert_eq!(i.get(1), @"cat"); + assert_eq!(i.get(2), @"zebra"); + assert_eq!(i.get(3), @"zebra"); + assert_eq!(i.get(4), @"dog"); + } } From 0994c3c91edc2cec7de9f44e0f7fefe05008d138 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 27 Jun 2013 16:00:18 -0700 Subject: [PATCH 34/67] added IterBytes for 4-tuples --- src/libstd/to_bytes.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libstd/to_bytes.rs b/src/libstd/to_bytes.rs index 60df31fd4ca8c..011129c010809 100644 --- a/src/libstd/to_bytes.rs +++ b/src/libstd/to_bytes.rs @@ -251,6 +251,20 @@ impl IterBytes for (A,B,C) { } } +impl IterBytes for (A,B,C,D) { + #[inline] + fn iter_bytes(&self, lsb0: bool, f: Cb) -> bool { + match *self { + (ref a, ref b, ref c, ref d) => { + a.iter_bytes(lsb0, |b| f(b)) && + b.iter_bytes(lsb0, |b| f(b)) && + c.iter_bytes(lsb0, |b| f(b)) && + d.iter_bytes(lsb0, |b| f(b)) + } + } + } +} + // Move this to vec, probably. fn borrow<'x,A>(a: &'x [A]) -> &'x [A] { a @@ -359,3 +373,13 @@ impl ToBytes for A { } } } + +#[cfg(test)] +mod test { + use super::*; + // just test to see if it compiles: + #[test] fn iterbytes_compiles () { + takes_iterbytes((3,4,5,false)); + } + fn takes_iterbytes(x : T) {} +} From 19971b4d38f404fbb38dbc2da298fce62e774b95 Mon Sep 17 00:00:00 2001 From: John Clements Date: Sat, 29 Jun 2013 05:59:08 -0700 Subject: [PATCH 35/67] marking on both input and output from macros. nice shiny new test case framework --- src/libsyntax/ext/expand.rs | 190 ++++++++++++++++++++++++++++++------ 1 file changed, 161 insertions(+), 29 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index fb8ce8e003846..5cc8d6a3327ae 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -64,10 +64,10 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, span: exp_span, }, }); - let fm = fresh_mark(); - let marked_tts = mark_tts(*tts,fm); - let expanded = match expandfun(cx, mac.span, marked_tts) { + // mark before: + let marked_before = mark_tts(*tts,fm); + let expanded = match expandfun(cx, mac.span, marked_before) { MRExpr(e) => e, MRAny(expr_maker,_,_) => expr_maker(), _ => { @@ -80,10 +80,12 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, ) } }; + // mark after: + let marked_after = mark_expr(expanded,fm); //keep going, outside-in let fully_expanded = - fld.fold_expr(expanded).node.clone(); + fld.fold_expr(marked_after).node.clone(); cx.bt_pop(); (fully_expanded, s) @@ -102,12 +104,6 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, } } -// apply a fresh mark to the given token trees. Used prior to expansion of a macro. -fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] { - fold_tts(tts,new_ident_marker(m)) -} - - // This is a secondary mechanism for invoking syntax extensions on items: // "decorator" attributes, such as #[auto_encode]. These are invoked by an // attribute prefixing an item, and are interpreted by feeding the item @@ -155,7 +151,6 @@ pub fn expand_mod_items(extsbox: @mut SyntaxEnv, ast::_mod { items: new_items, ..module_ } } - // eval $e with a new exts frame: macro_rules! with_exts_frame ( ($extsboxexpr:expr,$macros_escape:expr,$e:expr) => @@ -221,6 +216,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, let extname = &pth.idents[0]; let extnamestr = ident_to_str(extname); + let fm = fresh_mark(); let expanded = match (*extsbox).find(&extname.name) { None => cx.span_fatal(pth.span, fmt!("macro undefined: '%s!'", extnamestr)), @@ -239,7 +235,11 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, span: span } }); - expander(cx, it.span, tts) + // mark before expansion: + let marked_tts = mark_tts(tts,fm); + // mark after expansion: + // RIGHT HERE: can't apply mark_item to MacResult ... :( + expander(cx, it.span, marked_tts) } Some(@SE(IdentTT(expander, span))) => { if it.ident.name == parse::token::special_idents::invalid.name { @@ -254,18 +254,24 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, span: span } }); - expander(cx, it.span, it.ident, tts) + let fm = fresh_mark(); + // mark before expansion: + let marked_tts = mark_tts(tts,fm); + expander(cx, it.span, it.ident, marked_tts) } _ => cx.span_fatal( it.span, fmt!("%s! is not legal in item position", extnamestr)) }; let maybe_it = match expanded { - MRItem(it) => fld.fold_item(it), + MRItem(it) => mark_item(it,fm).chain(|i| {fld.fold_item(i)}), MRExpr(_) => cx.span_fatal(pth.span, fmt!("expr macro in item position: %s", extnamestr)), - MRAny(_, item_maker, _) => item_maker().chain(|i| {fld.fold_item(i)}), + MRAny(_, item_maker, _) => item_maker().chain(|i| {mark_item(i,fm)}) + .chain(|i| {fld.fold_item(i)}), MRDef(ref mdef) => { + // yikes... no idea how to apply the mark to this. I'm afraid + // we're going to have to wait-and-see on this one. insert_macro(*extsbox,intern(mdef.name), @SE((*mdef).ext)); None } @@ -300,6 +306,8 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, orig: @fn(&stmt_, span, @ast_fold) -> (Option, span)) -> (Option, span) { + // why the copying here and not in expand_expr? + // looks like classic changed-in-only-one-place let (mac, pth, tts, semi) = match *s { stmt_mac(ref mac, semi) => { match mac.node { @@ -327,7 +335,10 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, call_site: sp, callee: NameAndSpan { name: extnamestr, span: exp_span } }); - let expanded = match expandfun(cx, mac.span, tts) { + let fm = fresh_mark(); + // mark before expansion: + let marked_tts = mark_tts(tts,fm); + let expanded = match expandfun(cx, mac.span, marked_tts) { MRExpr(e) => @codemap::spanned { node: stmt_expr(e, cx.next_id()), span: e.span}, @@ -336,9 +347,10 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, pth.span, fmt!("non-stmt macro in stmt pos: %s", extnamestr)) }; + let marked_after = mark_stmt(expanded,fm); //keep going, outside-in - let fully_expanded = match fld.fold_stmt(expanded) { + let fully_expanded = match fld.fold_stmt(marked_after) { Some(stmt) => { let fully_expanded = &stmt.node; cx.bt_pop(); @@ -401,6 +413,7 @@ fn expand_non_macro_stmt (exts: SyntaxEnv, // ... but that should be okay, as long as the new names are gensyms // for the old ones. let idents = @mut ~[]; + // is this really the right way to call a visitor? it's so clunky! ((*name_finder).visit_pat) (expanded_pat, (idents, visit::mk_vt(name_finder))); @@ -467,6 +480,27 @@ pub fn new_name_finder() -> @Visitor<@mut ~[ast::ident]> { } } +// return a visitor that extracts the paths +// from a given pattern and puts them in a mutable +// array (passed in to the traversal) +pub fn new_path_finder() -> @Visitor<@mut ~[@ast::Path]> { + let default_visitor = visit::default_visitor(); + @Visitor{ + visit_expr : + |e:@ast::expr, + (path_accum, v):(@mut ~[@ast::Path], visit::vt<@mut ~[@ast::Path]>)| { + match *e { + ast::expr{id:_,span:_,node:ast::expr_path(ref p)} => { + path_accum.push(@p.clone()); + // not calling visit_path, should be fine. + } + _ => visit::visit_expr(e,(path_accum,v)) + } + }, + .. *default_visitor + } +} + // given a mutable list of renames, return a tree-folder that applies those // renames. // FIXME #4536: currently pub to allow testing @@ -864,7 +898,6 @@ pub fn new_ident_renamer(from: ast::ident, } } - // update the ctxts in a path to get a mark node pub fn new_ident_marker(mark: Mrk) -> @fn(ast::ident)->ast::ident { @@ -888,20 +921,42 @@ pub fn new_ident_resolver() -> } } +// apply a given mark to the given token trees. Used prior to expansion of a macro. +fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] { + fold_tts(tts,new_ident_marker(m)) +} + +// apply a given mark to the given expr. Used following the expansion of a macro. +fn mark_expr(expr : @ast::expr, m : Mrk) -> @ast::expr { + fun_to_ident_folder(new_ident_marker(m)).fold_expr(expr) +} + +// apply a given mark to the given stmt. Used following the expansion of a macro. +fn mark_stmt(expr : &ast::stmt, m : Mrk) -> @ast::stmt { + fun_to_ident_folder(new_ident_marker(m)).fold_stmt(expr).get() +} + +// apply a given mark to the given item. Used following the expansion of a macro. +fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> { + fun_to_ident_folder(new_ident_marker(m)).fold_item(expr) +} + #[cfg(test)] mod test { use super::*; use ast; use ast::{Attribute_, AttrOuter, MetaWord, empty_ctxt}; - use ast_util::{get_sctable, new_ident, new_rename}; + use ast_util::{get_sctable, mtwt_resolve, new_ident, new_rename}; use codemap; use codemap::spanned; use parse; use parse::token::{gensym, intern, get_ident_interner}; use print::pprust; use std; + use std::vec; use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat, strs_to_idents}; use visit::{mk_vt}; + use visit; // make sure that fail! is present #[test] fn fail_exists_test () { @@ -1047,30 +1102,107 @@ mod test { //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner()) //} + // renaming tests expand a crate and then check that the bindings match + // the right varrefs. The specification of the test case includes the + // text of the crate, and also an array of arrays. Each element in the + // outer array corresponds to a binding in the traversal of the AST + // induced by visit. Each of these arrays contains a list of indexes, + // interpreted as the varrefs in the varref traversal that this binding + // should match. So, for instance, in a program with two bindings and + // three varrefs, the array ~[~[1,2],~[0]] would indicate that the first + // binding should match the second two varrefs, and the second binding + // should match the first varref. + // + // The comparisons are done post-mtwt-resolve, so we're comparing renamed + // names; differences in marks don't matter any more. + type renaming_test = (&'static str, ~[~[uint]]); + #[test] fn automatic_renaming () { // need some other way to test these... - let teststrs = + let tests : ~[renaming_test] = ~[// b & c should get new names throughout, in the expr too: - @"fn a() -> int { let b = 13; let c = b; b+c }", + ("fn a() -> int { let b = 13; let c = b; b+c }", + ~[~[0,1],~[2]]), // both x's should be renamed (how is this causing a bug?) - @"fn main () {let x : int = 13;x;}", - // the use of b before the + should be renamed, the other one not: - @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}", + ("fn main () {let x : int = 13;x;}", + ~[~[0]]), + // the use of b after the + should be renamed, the other one not: + ("macro_rules! f (($x:ident) => (b + $x)) fn a() -> int { let b = 13; f!(b)}", + ~[~[1]]), // the b before the plus should not be renamed (requires marks) - @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}", + ("macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}", + ~[~[1]]), + // the marks going in and out of letty should cancel, allowing that $x to + // capture the one following the semicolon. + // this was an awesome test case, and caught a *lot* of bugs. + ("macro_rules! letty(($x:ident) => (let $x = 15;)) + macro_rules! user(($x:ident) => ({letty!($x); $x})) + fn main() -> int {user!(z)}", + ~[~[0]]) // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm // commenting it out. // the z flows into and out of two macros (g & f) along one path, and one (just g) along the // other, so the result of the whole thing should be "let z_123 = 3; z_123" - //@"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) + //"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) // fn a(){g!(z)}" // create a really evil test case where a $x appears inside a binding of $x but *shouldnt* // bind because it was inserted by a different macro.... ]; - for teststrs.iter().advance |s| { - // we need regexps to test these! - //std::io::println(expand_and_resolve_and_pretty_print(*s)); + for tests.iter().advance |s| { + run_renaming_test(s); + } + } + + + fn run_renaming_test(t : &renaming_test) { + let nv = new_name_finder(); + let pv = new_path_finder(); + let (teststr, bound_connections) = match *t { + (ref str,ref conns) => (str.to_managed(), conns.clone()) + }; + let cr = expand_crate_str(teststr.to_managed()); + let bindings = @mut ~[]; + visit::visit_crate(cr, (bindings, mk_vt(nv))); + let varrefs = @mut ~[]; + visit::visit_crate(cr, (varrefs, mk_vt(pv))); + // must be one check clause for each binding: + assert_eq!(bindings.len(),bound_connections.len()); + for bound_connections.iter().enumerate().advance |(binding_idx,shouldmatch)| { + let binding_name = mtwt_resolve(bindings[binding_idx]); + // shouldmatch can't name varrefs that don't exist: + assert!((shouldmatch.len() == 0) || + (varrefs.len() > *shouldmatch.iter().max().get())); + for varrefs.iter().enumerate().advance |(idx,varref)| { + if shouldmatch.contains(&idx) { + // it should be a path of length 1, and it should + // be free-identifier=? to the given binding + assert_eq!(varref.idents.len(),1); + let varref_name = mtwt_resolve(varref.idents[0]); + if (!(varref_name==binding_name)){ + std::io::println("uh oh, should match but doesn't:"); + std::io::println(fmt!("varref: %?",varref)); + std::io::println(fmt!("binding: %?", bindings[binding_idx])); + let table = get_sctable(); + std::io::println("SC table:"); + for table.table.iter().enumerate().advance |(idx,val)| { + std::io::println(fmt!("%4u : %?",idx,val)); + } + } + assert_eq!(varref_name,binding_name); + } else { + let fail = (varref.idents.len() == 1) + && (mtwt_resolve(varref.idents[0]) == binding_name); + // temp debugging: + if (fail) { + std::io::println("uh oh, matches but shouldn't:"); + std::io::println(fmt!("varref: %?",varref)); + std::io::println(fmt!("binding: %?", bindings[binding_idx])); + std::io::println(fmt!("sc_table: %?",get_sctable())); + } + assert!(!fail); + } + } } } From ef6383e2a573a43d3f0dfa9a583b6b73cb022339 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 15:10:16 -0700 Subject: [PATCH 36/67] add test case for macro token comparison --- src/libsyntax/ext/expand.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 5cc8d6a3327ae..c433e3f886f2e 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1102,6 +1102,10 @@ mod test { //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner()) //} + #[test] fn macro_tokens_should_match(){ + expand_crate_str(@"macro_rules! m((a)=>(13)) fn main(){m!(a);}"); + } + // renaming tests expand a crate and then check that the bindings match // the right varrefs. The specification of the test case includes the // text of the crate, and also an array of arrays. Each element in the From 2acb0f4faf3712e997946229d59e147741b796a9 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 3 Jul 2013 15:15:45 -0700 Subject: [PATCH 37/67] comments --- src/libsyntax/parse/token.rs | 39 ++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 2d520c299ffc6..779be26991b25 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -304,22 +304,23 @@ pub fn is_bar(t: &Token) -> bool { match *t { BINOP(OR) | OROR => true, _ => false } } - pub mod special_idents { use ast::ident; - pub static underscore : ident = ident { name: 0, ctxt: 0}; + pub static underscore : ident = ident { name: 0, ctxt: 0}; // apparently unused? pub static anon : ident = ident { name: 1, ctxt: 0}; pub static invalid : ident = ident { name: 2, ctxt: 0}; // '' - pub static unary : ident = ident { name: 3, ctxt: 0}; - pub static not_fn : ident = ident { name: 4, ctxt: 0}; - pub static idx_fn : ident = ident { name: 5, ctxt: 0}; - pub static unary_minus_fn : ident = ident { name: 6, ctxt: 0}; + pub static unary : ident = ident { name: 3, ctxt: 0}; // apparently unused? + pub static not_fn : ident = ident { name: 4, ctxt: 0}; // apparently unused? + pub static idx_fn : ident = ident { name: 5, ctxt: 0}; // apparently unused? + pub static unary_minus_fn : ident = ident { name: 6, ctxt: 0}; // apparently unused? pub static clownshoes_extensions : ident = ident { name: 7, ctxt: 0}; pub static self_ : ident = ident { name: 8, ctxt: 0}; // 'self' /* for matcher NTs */ + // none of these appear to be used, but perhaps references to + // these are artificially fabricated by the macro system.... pub static item : ident = ident { name: 9, ctxt: 0}; pub static block : ident = ident { name: 10, ctxt: 0}; pub static stmt : ident = ident { name: 11, ctxt: 0}; @@ -331,7 +332,7 @@ pub mod special_idents { pub static tt : ident = ident { name: 17, ctxt: 0}; pub static matchers : ident = ident { name: 18, ctxt: 0}; - pub static str : ident = ident { name: 19, ctxt: 0}; // for the type + pub static str : ident = ident { name: 19, ctxt: 0}; // for the type // apparently unused? /* outside of libsyntax */ pub static arg : ident = ident { name: 20, ctxt: 0}; @@ -344,10 +345,32 @@ pub mod special_idents { pub static statik : ident = ident { name: 27, ctxt: 0}; pub static clownshoes_foreign_mod: ident = ident { name: 28, ctxt: 0}; pub static unnamed_field: ident = ident { name: 29, ctxt: 0}; - pub static c_abi: ident = ident { name: 30, ctxt: 0}; + pub static c_abi: ident = ident { name: 30, ctxt: 0}; // apparently unused? pub static type_self: ident = ident { name: 31, ctxt: 0}; // `Self` } +// here are the ones that actually occur in the source. Maybe the rest +// should be removed? +/* +special_idents::anon +special_idents::arg +special_idents::blk +special_idents::clownshoe_abi +special_idents::clownshoe_stack_shim +special_idents::clownshoes_extensions +special_idents::clownshoes_foreign_mod +special_idents::descrim +special_idents::invalid +special_idents::main +special_idents::matchers +special_idents::opaque +special_idents::self_ +special_idents::statik +special_idents::tt +special_idents::type_self +special_idents::unnamed_field +*/ + /** * Maps a token to a record specifying the corresponding binary * operator From 6cfdae0b953d0076e983ca35640e6d99dcc3570a Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 3 Jul 2013 15:16:04 -0700 Subject: [PATCH 38/67] add temporarily unused ctxt field to mac_invoc_tt --- src/libsyntax/ast.rs | 2 +- src/libsyntax/ext/expand.rs | 6 +++--- src/libsyntax/fold.rs | 5 +++-- src/libsyntax/parse/parser.rs | 10 +++++----- src/libsyntax/print/pprust.rs | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index c3816b03fae51..156b4d66aabe4 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -634,7 +634,7 @@ pub type mac = spanned; // There's only one flavor, now, so this could presumably be simplified. #[deriving(Clone, Eq, Encodable, Decodable, IterBytes)] pub enum mac_ { - mac_invoc_tt(Path,~[token_tree]), // new macro-invocation + mac_invoc_tt(Path,~[token_tree],SyntaxContext), // new macro-invocation } pub type lit = spanned; diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index c433e3f886f2e..9713f02c70ff2 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -40,7 +40,7 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, expr_mac(ref mac) => { match (*mac).node { // Token-tree macros: - mac_invoc_tt(ref pth, ref tts) => { + mac_invoc_tt(ref pth, ref tts, ctxt) => { if (pth.idents.len() > 1u) { cx.span_fatal( pth.span, @@ -208,7 +208,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, fld: @ast_fold) -> Option<@ast::item> { let (pth, tts) = match it.node { - item_mac(codemap::spanned { node: mac_invoc_tt(ref pth, ref tts), _}) => { + item_mac(codemap::spanned { node: mac_invoc_tt(ref pth, ref tts, ctxt), _}) => { (pth, (*tts).clone()) } _ => cx.span_bug(it.span, "invalid item macro invocation") @@ -311,7 +311,7 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, let (mac, pth, tts, semi) = match *s { stmt_mac(ref mac, semi) => { match mac.node { - mac_invoc_tt(ref pth, ref tts) => { + mac_invoc_tt(ref pth, ref tts, ctxt) => { ((*mac).clone(), pth, (*tts).clone(), semi) } } diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index adf251ec48313..5db33c8c75e97 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -116,9 +116,10 @@ fn fold_arg_(a: arg, fld: @ast_fold) -> arg { fn fold_mac_(m: &mac, fld: @ast_fold) -> mac { spanned { node: match m.node { - mac_invoc_tt(ref p,ref tts) => + mac_invoc_tt(ref p,ref tts,ctxt) => mac_invoc_tt(fld.fold_path(p), - fold_tts(*tts,|id|{fld.fold_ident(id)})) + fold_tts(*tts,|id|{fld.fold_ident(id)}), + ctxt) }, span: fld.new_span(m.span) } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 61617b67e7767..f5a0f5373f9bd 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -19,7 +19,7 @@ use ast::{_mod, add, arg, arm, Attribute, bind_by_ref, bind_infer}; use ast::{bitand, bitor, bitxor, Block}; use ast::{blk_check_mode, box}; use ast::{Crate, CrateConfig, decl, decl_item}; -use ast::{decl_local, default_blk, deref, div, enum_def, explicit_self}; +use ast::{decl_local, default_blk, deref, div, empty_ctxt, enum_def, explicit_self}; use ast::{expr, expr_, expr_addr_of, expr_match, expr_again}; use ast::{expr_assign, expr_assign_op, expr_binary, expr_block}; use ast::{expr_break, expr_call, expr_cast, expr_do_body}; @@ -1730,7 +1730,7 @@ impl Parser { |p| p.parse_token_tree()); let hi = self.span.hi; - return self.mk_mac_expr(lo, hi, mac_invoc_tt(pth, tts)); + return self.mk_mac_expr(lo, hi, mac_invoc_tt(pth, tts, empty_ctxt)); } else if *self.token == token::LBRACE { // This might be a struct literal. if self.looking_at_record_literal() { @@ -3039,14 +3039,14 @@ impl Parser { if id == token::special_idents::invalid { return @spanned(lo, hi, stmt_mac( - spanned(lo, hi, mac_invoc_tt(pth, tts)), false)); + spanned(lo, hi, mac_invoc_tt(pth, tts, empty_ctxt)), false)); } else { // if it has a special ident, it's definitely an item return @spanned(lo, hi, stmt_decl( @spanned(lo, hi, decl_item( self.mk_item( lo, hi, id /*id is good here*/, - item_mac(spanned(lo, hi, mac_invoc_tt(pth, tts))), + item_mac(spanned(lo, hi, mac_invoc_tt(pth, tts, empty_ctxt))), inherited, ~[/*no attrs*/]))), self.get_id())); } @@ -4639,7 +4639,7 @@ impl Parser { _ => self.fatal("expected open delimiter") }; // single-variant-enum... : - let m = ast::mac_invoc_tt(pth, tts); + let m = ast::mac_invoc_tt(pth, tts, empty_ctxt); let m: ast::mac = codemap::spanned { node: m, span: mk_sp(self.span.lo, self.span.hi) }; diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 8e2c24cacfed7..dbc729d4f1c2e 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -617,7 +617,7 @@ pub fn print_item(s: @ps, item: &ast::item) { } bclose(s, item.span); } - ast::item_mac(codemap::spanned { node: ast::mac_invoc_tt(ref pth, ref tts), + ast::item_mac(codemap::spanned { node: ast::mac_invoc_tt(ref pth, ref tts, ctxt), _}) => { print_visibility(s, item.vis); print_path(s, pth, false); @@ -1019,7 +1019,7 @@ pub fn print_if(s: @ps, test: &ast::expr, blk: &ast::Block, pub fn print_mac(s: @ps, m: &ast::mac) { match m.node { - ast::mac_invoc_tt(ref pth, ref tts) => { + ast::mac_invoc_tt(ref pth, ref tts, ctxt) => { print_path(s, pth, false); word(s.s, "!"); popen(s); From fe1184559ac8689f7ceddf9d4d767fbe8c1fdd69 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 15:56:21 -0700 Subject: [PATCH 39/67] rework fold so that fold_tts takes an ast_fold rather than a thunk, stop using closures in ident traversal --- src/libsyntax/ext/expand.rs | 105 +++++++++++++++++++++--------------- src/libsyntax/fold.rs | 14 ++--- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 9713f02c70ff2..7025adb77b05e 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -876,38 +876,52 @@ pub fn expand_crate(parse_sess: @mut parse::ParseSess, @f.fold_crate(c) } -// given a function from idents to idents, produce -// an ast_fold that applies that function: -pub fn fun_to_ident_folder(f: @fn(ast::ident)->ast::ident) -> @ast_fold{ - let afp = default_ast_fold(); - let f_pre = @AstFoldFns{ - fold_ident : |id, _| f(id), - .. *afp - }; - make_fold(f_pre) +// a function in SyntaxContext -> SyntaxContext +pub trait CtxtFn{ + pub fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext; } -// update the ctxts in a path to get a rename node -pub fn new_ident_renamer(from: ast::ident, - to: ast::Name) -> - @fn(ast::ident)->ast::ident { - |id : ast::ident| - ast::ident{ - name: id.name, - ctxt: new_rename(from,to,id.ctxt) +pub struct Renamer { + from : ast::ident, + to : ast::Name +} + +impl CtxtFn for Renamer { + pub fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext { + new_rename(self.from,self.to,ctxt) } } -// update the ctxts in a path to get a mark node -pub fn new_ident_marker(mark: Mrk) -> - @fn(ast::ident)->ast::ident { - |id : ast::ident| - ast::ident{ - name: id.name, - ctxt: new_mark(mark,id.ctxt) +pub struct Marker { mark : Mrk } + +impl CtxtFn for Marker { + pub fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext { + new_mark(self.mark,ctxt) } } +// given a function from ctxts to ctxts, produce +// an ast_fold that applies that function to all ctxts: +pub fn fun_to_ctxt_folder(cf: @T) -> @AstFoldFns { + let afp = default_ast_fold(); + let fi : @fn(ast::ident, @ast_fold) -> ast::ident = + |ast::ident{name, ctxt}, _| { + ast::ident{name:name,ctxt:cf.f(ctxt)} + }; + @AstFoldFns{ + fold_ident : fi, + // check that it works, then add the fold_expr clause.... + .. *afp + } +} + +// just a convenience: +pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) } +pub fn new_rename_folder(from : ast::ident, to : ast::Name) -> @AstFoldFns { + fun_to_ctxt_folder(@Renamer{from:from,to:to}) +} + +/* // perform resolution (in the MTWT sense) on all of the // idents in the tree. This is the final step in expansion. // FIXME #6993: this function could go away, along with @@ -920,25 +934,26 @@ pub fn new_ident_resolver() -> ctxt : empty_ctxt } } +*/ // apply a given mark to the given token trees. Used prior to expansion of a macro. fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] { - fold_tts(tts,new_ident_marker(m)) + fold_tts(tts,new_mark_folder(m) as @ast_fold) } // apply a given mark to the given expr. Used following the expansion of a macro. fn mark_expr(expr : @ast::expr, m : Mrk) -> @ast::expr { - fun_to_ident_folder(new_ident_marker(m)).fold_expr(expr) + new_mark_folder(m).fold_expr(expr) } // apply a given mark to the given stmt. Used following the expansion of a macro. fn mark_stmt(expr : &ast::stmt, m : Mrk) -> @ast::stmt { - fun_to_ident_folder(new_ident_marker(m)).fold_stmt(expr).get() + new_mark_folder(m).fold_stmt(expr).get() } // apply a given mark to the given item. Used following the expansion of a macro. fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> { - fun_to_ident_folder(new_ident_marker(m)).fold_item(expr) + new_mark_folder(m).fold_item(expr) } #[cfg(test)] @@ -954,7 +969,7 @@ mod test { use print::pprust; use std; use std::vec; - use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat, strs_to_idents}; + use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item, string_to_pat, strs_to_idents}; use visit::{mk_vt}; use visit; @@ -1056,30 +1071,32 @@ mod test { #[test] fn renaming () { - let item_ast = string_to_item(@"fn a() -> int { let b = 13; b }").get(); + let item_ast = string_to_crate(@"fn f() -> int { a }"); let a_name = intern("a"); let a2_name = gensym("a2"); - let renamer = new_ident_renamer(ast::ident{name:a_name,ctxt:empty_ctxt}, + let renamer = new_rename_folder(ast::ident{name:a_name,ctxt:empty_ctxt}, a2_name); - let renamed_ast = fun_to_ident_folder(renamer).fold_item(item_ast).get(); - let resolver = new_ident_resolver(); - let resolver_fold = fun_to_ident_folder(resolver); - let resolved_ast = resolver_fold.fold_item(renamed_ast).get(); - let resolved_as_str = pprust::item_to_str(resolved_ast, - get_ident_interner()); - assert_eq!(resolved_as_str,~"fn a2() -> int { let b = 13; b }"); + let renamed_ast = renamer.fold_crate(item_ast); + let pv = new_path_finder(); + let varrefs = @mut ~[]; + visit::visit_crate(&renamed_ast, (varrefs, mk_vt(pv))); + match varrefs { + @[@Path{idents:[id],_}] => assert_eq!(mtwt_resolve(id),a2_name), + _ => assert_eq!(0,1) + } // try a double-rename, with pending_renames. let a3_name = gensym("a3"); let ctxt2 = new_rename(new_ident(a_name),a2_name,empty_ctxt); let pending_renames = @mut ~[(new_ident(a_name),a2_name), (ast::ident{name:a_name,ctxt:ctxt2},a3_name)]; - let double_renamed = renames_to_fold(pending_renames).fold_item(item_ast).get(); - let resolved_again = resolver_fold.fold_item(double_renamed).get(); - let double_renamed_as_str = pprust::item_to_str(resolved_again, - get_ident_interner()); - assert_eq!(double_renamed_as_str,~"fn a3() -> int { let b = 13; b }"); - + let double_renamed = renames_to_fold(pending_renames).fold_crate(item_ast); + let varrefs = @mut ~[]; + visit::visit_crate(&double_renamed, (varrefs, mk_vt(pv))); + match varrefs { + @[@Path{idents:[id],_}] => assert_eq!(mtwt_resolve(id),a2_name), + _ => assert_eq!(0,1) + } } fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) { diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 5db33c8c75e97..3bc260e6f41f9 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -118,16 +118,16 @@ fn fold_mac_(m: &mac, fld: @ast_fold) -> mac { node: match m.node { mac_invoc_tt(ref p,ref tts,ctxt) => mac_invoc_tt(fld.fold_path(p), - fold_tts(*tts,|id|{fld.fold_ident(id)}), + fold_tts(*tts,fld), ctxt) }, span: fld.new_span(m.span) } } -// build a new vector of tts by appling the given function to +// build a new vector of tts by appling the ast_fold's fold_ident to // all of the identifiers in the token trees. -pub fn fold_tts(tts : &[token_tree], f: @fn(ident)->ident) -> ~[token_tree] { +pub fn fold_tts(tts : &[token_tree], f : @ast_fold) -> ~[token_tree] { do tts.map |tt| { match *tt { tt_tok(span, ref tok) => @@ -140,16 +140,16 @@ pub fn fold_tts(tts : &[token_tree], f: @fn(ident)->ident) -> ~[token_tree] { sep.map(|tok|maybe_fold_ident(tok,f)), is_optional), tt_nonterminal(sp,ref ident) => - tt_nonterminal(sp,f(*ident)) + tt_nonterminal(sp,f.fold_ident(*ident)) } } } // apply ident folder if it's an ident, otherwise leave it alone -fn maybe_fold_ident(t : &token::Token, f: @fn(ident)->ident) -> token::Token { +fn maybe_fold_ident(t : &token::Token, f: @ast_fold) -> token::Token { match *t { token::IDENT(id,followed_by_colons) => - token::IDENT(f(id),followed_by_colons), + token::IDENT(f.fold_ident(id),followed_by_colons), _ => (*t).clone() } } @@ -944,6 +944,8 @@ impl AstFoldExtensions for @ast_fold { } } +// brson agrees with me that this function's existence is probably +// not a good or useful thing. pub fn make_fold(afp: ast_fold_fns) -> @ast_fold { afp as @ast_fold } From 6dc49cbc8ee15af0df981e52ae9227a09dab7718 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 5 Jul 2013 13:57:53 -0700 Subject: [PATCH 40/67] WIP: adding context to macros --- src/libsyntax/ext/expand.rs | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 7025adb77b05e..67c083e4db8b9 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -450,7 +450,7 @@ fn expand_non_macro_stmt (exts: SyntaxEnv, // return a visitor that extracts the pat_ident paths -// from a given pattern and puts them in a mutable +// from a given thingy and puts them in a mutable // array (passed in to the traversal) pub fn new_name_finder() -> @Visitor<@mut ~[ast::ident]> { let default_visitor = visit::default_visitor(); @@ -481,7 +481,7 @@ pub fn new_name_finder() -> @Visitor<@mut ~[ast::ident]> { } // return a visitor that extracts the paths -// from a given pattern and puts them in a mutable +// from a given thingy and puts them in a mutable // array (passed in to the traversal) pub fn new_path_finder() -> @Visitor<@mut ~[@ast::Path]> { let default_visitor = visit::default_visitor(); @@ -501,6 +501,24 @@ pub fn new_path_finder() -> @Visitor<@mut ~[@ast::Path]> { } } +// return a visitor that extracts the exprs +// from a given thingy and puts them in a mutable +// array. Maybe it's time for some abstraction, here.... +pub fn new_expr_finder() -> @Visitor<@mut ~[@ast::expr]> { + let default_visitor = visit::default_visitor(); + @Visitor{ + visit_expr : + |e:@ast::expr, + (path_accum, v):(@mut ~[@ast::expr], visit::vt<@mut ~[@ast::expr]>)| { + path_accum.push(e); + // call the built-in one as well to recur: + visit::visit_expr(e,(path_accum,v)) + }, + .. *default_visitor + } +} + + // given a mutable list of renames, return a tree-folder that applies those // renames. // FIXME #4536: currently pub to allow testing @@ -521,6 +539,7 @@ pub fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @ast_fold { } +// expand a block. pushes a new exts_frame, then calls expand_block_elts pub fn expand_block(extsbox: @mut SyntaxEnv, _cx: @ExtCtxt, blk: &Block, @@ -532,7 +551,7 @@ pub fn expand_block(extsbox: @mut SyntaxEnv, expand_block_elts(*extsbox, blk, fld)) } - +// expand the elements of a block. pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block { let block_info = get_block_info(exts); let pending_renames = block_info.pending_renames; @@ -557,6 +576,7 @@ pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block { } // rename_fold should never return "None". +// (basically, just .get() with a better message...) fn mustbesome(val : Option) -> T { match val { Some(v) => v, @@ -579,8 +599,8 @@ pub fn new_span(cx: @ExtCtxt, sp: span) -> span { } // FIXME (#2247): this is a moderately bad kludge to inject some macros into -// the default compilation environment. It would be much nicer to use -// a mechanism like syntax_quote to ensure hygiene. +// the default compilation environment in that it injects strings, rather than +// syntax elements. pub fn std_macros() -> @str { return @@ -908,6 +928,10 @@ pub fn fun_to_ctxt_folder(cf: @T) -> @AstFoldFns { |ast::ident{name, ctxt}, _| { ast::ident{name:name,ctxt:cf.f(ctxt)} }; + // we've also got to pick up macro invocations; they can + // appear as exprs, stmts, items, and types. urg, it's going + // to be easier just to add a fold_mac, I think. + //let fold_ex : @ @AstFoldFns{ fold_ident : fi, // check that it works, then add the fold_expr clause.... @@ -1087,6 +1111,7 @@ mod test { // try a double-rename, with pending_renames. let a3_name = gensym("a3"); + // a context that renames from ("a",empty) to "a2" : let ctxt2 = new_rename(new_ident(a_name),a2_name,empty_ctxt); let pending_renames = @mut ~[(new_ident(a_name),a2_name), (ast::ident{name:a_name,ctxt:ctxt2},a3_name)]; @@ -1094,7 +1119,7 @@ mod test { let varrefs = @mut ~[]; visit::visit_crate(&double_renamed, (varrefs, mk_vt(pv))); match varrefs { - @[@Path{idents:[id],_}] => assert_eq!(mtwt_resolve(id),a2_name), + @[@Path{idents:[id],_}] => assert_eq!(mtwt_resolve(id),a3_name), _ => assert_eq!(0,1) } } From 649ed8b1c7b0f44ca2ebe709de872e019ff34f16 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 5 Jul 2013 14:16:39 -0700 Subject: [PATCH 41/67] add fold_mac field to fold.rs --- src/libsyntax/fold.rs | 59 ++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 3bc260e6f41f9..9db28ce00dddd 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -14,6 +14,14 @@ use codemap::{span, spanned}; use parse::token; use opt_vec::OptVec; +// this file defines an ast_fold trait for objects that can perform +// a "fold" on Rust ASTs. It also contains a structure that implements +// that trait, and a "default_fold" whose fields contain closures +// that perform "default traversals", visiting all of the sub-elements +// and re-assembling the result. The "fun_to_ident_folder" in the +// test module provides a simple example of creating a very simple +// fold that only looks at identifiers. + pub trait ast_fold { fn fold_crate(@self, &Crate) -> Crate; fn fold_view_item(@self, &view_item) -> view_item; @@ -35,6 +43,7 @@ pub trait ast_fold { fn fold_ident(@self, ident) -> ident; fn fold_path(@self, &Path) -> Path; fn fold_local(@self, @Local) -> @Local; + fn fold_mac(@self, &mac) -> mac; fn map_exprs(@self, @fn(@expr) -> @expr, &[@expr]) -> ~[@expr]; fn new_id(@self, node_id) -> node_id; fn new_span(@self, span) -> span; @@ -64,6 +73,7 @@ pub struct AstFoldFns { fold_ident: @fn(ident, @ast_fold) -> ident, fold_path: @fn(&Path, @ast_fold) -> Path, fold_local: @fn(@Local, @ast_fold) -> @Local, + fold_mac: @fn(&mac_, span, @ast_fold) -> (mac_, span), map_exprs: @fn(@fn(@expr) -> @expr, &[@expr]) -> ~[@expr], new_id: @fn(node_id) -> node_id, new_span: @fn(span) -> span @@ -112,19 +122,6 @@ fn fold_arg_(a: arg, fld: @ast_fold) -> arg { } } -//used in noop_fold_expr, and possibly elsewhere in the future -fn fold_mac_(m: &mac, fld: @ast_fold) -> mac { - spanned { - node: match m.node { - mac_invoc_tt(ref p,ref tts,ctxt) => - mac_invoc_tt(fld.fold_path(p), - fold_tts(*tts,fld), - ctxt) - }, - span: fld.new_span(m.span) - } -} - // build a new vector of tts by appling the ast_fold's fold_ident to // all of the identifiers in the token trees. pub fn fold_tts(tts : &[token_tree], f : @ast_fold) -> ~[token_tree] { @@ -330,7 +327,7 @@ pub fn noop_fold_item_underscore(i: &item_, fld: @ast_fold) -> item_ { // It would probably be nicer // to expose this in the ast_fold trait, but I'll defer // that work. - item_mac(fold_mac_(m,fld)) + item_mac(fld.fold_mac(m)) } } } @@ -399,7 +396,6 @@ pub fn noop_fold_block(b: &Block, fld: @ast_fold) -> Block { } fn noop_fold_stmt(s: &stmt_, fld: @ast_fold) -> Option { - let fold_mac = |x| fold_mac_(x, fld); match *s { stmt_decl(d, nid) => { match fld.fold_decl(d) { @@ -413,7 +409,7 @@ fn noop_fold_stmt(s: &stmt_, fld: @ast_fold) -> Option { stmt_semi(e, nid) => { Some(stmt_semi(fld.fold_expr(e), fld.new_id(nid))) } - stmt_mac(ref mac, semi) => Some(stmt_mac(fold_mac(mac), semi)) + stmt_mac(ref mac, semi) => Some(stmt_mac(fld.fold_mac(mac), semi)) } } @@ -481,6 +477,12 @@ fn noop_fold_decl(d: &decl_, fld: @ast_fold) -> Option { } } +// lift a function in ast-thingy X fold -> ast-thingy to a function +// in (ast-thingy X span X fold) -> (ast-thingy X fold). Basically, +// carries the span around. +// It seems strange to me that the call to new_fold doesn't happen +// here but instead in the impl down below.... probably just an +// accident? pub fn wrap(f: @fn(&T, @ast_fold) -> T) -> @fn(&T, span, @ast_fold) -> (T, span) { let result: @fn(&T, span, @ast_fold) -> (T, span) = |x, s, fld| { @@ -499,8 +501,6 @@ pub fn noop_fold_expr(e: &expr_, fld: @ast_fold) -> expr_ { } let fold_field = |x| fold_field_(x, fld); - let fold_mac = |x| fold_mac_(x, fld); - match *e { expr_vstore(e, v) => { expr_vstore(fld.fold_expr(e), v) @@ -628,7 +628,7 @@ pub fn noop_fold_expr(e: &expr_, fld: @ast_fold) -> expr_ { .. (*a).clone() }) } - expr_mac(ref mac) => expr_mac(fold_mac(mac)), + expr_mac(ref mac) => expr_mac(fld.fold_mac(mac)), expr_struct(ref path, ref fields, maybe_expr) => { expr_struct( fld.fold_path(path), @@ -641,7 +641,6 @@ pub fn noop_fold_expr(e: &expr_, fld: @ast_fold) -> expr_ { } pub fn noop_fold_ty(t: &ty_, fld: @ast_fold) -> ty_ { - let fold_mac = |x| fold_mac_(x, fld); fn fold_mt(mt: &mt, fld: @ast_fold) -> mt { mt { ty: ~fld.fold_ty(mt.ty), @@ -698,7 +697,7 @@ pub fn noop_fold_ty(t: &ty_, fld: @ast_fold) -> ty_ { fld.fold_expr(e) ) } - ty_mac(ref mac) => ty_mac(fold_mac(mac)) + ty_mac(ref mac) => ty_mac(fld.fold_mac(mac)) } } @@ -783,6 +782,19 @@ fn noop_fold_local(l: @Local, fld: @ast_fold) -> @Local { } } +// the default macro traversal. visit the path +// using fold_path, and the tts using fold_tts, +// and the span using new_span +fn noop_fold_mac(m: &mac_, fld: @ast_fold) -> mac_ { + match *m { + mac_invoc_tt(ref p,ref tts,ctxt) => + mac_invoc_tt(fld.fold_path(p), + fold_tts(*tts,fld), + ctxt) + } +} + + /* temporarily eta-expand because of a compiler bug with using `fn` as a value */ fn noop_map_exprs(f: @fn(@expr) -> @expr, es: &[@expr]) -> ~[@expr] { @@ -815,6 +827,7 @@ pub fn default_ast_fold() -> ast_fold_fns { fold_ident: noop_fold_ident, fold_path: noop_fold_path, fold_local: noop_fold_local, + fold_mac: wrap(noop_fold_mac), map_exprs: noop_map_exprs, new_id: noop_id, new_span: noop_span, @@ -920,6 +933,10 @@ impl ast_fold for AstFoldFns { fn fold_local(@self, x: @Local) -> @Local { (self.fold_local)(x, self as @ast_fold) } + fn fold_mac(@self, x: &mac) -> mac { + let (n, s) = (self.fold_mac)(&x.node, x.span, self as @ast_fold); + spanned { node: n, span: (self.new_span)(s) } + } fn map_exprs(@self, f: @fn(@expr) -> @expr, e: &[@expr]) From b15ba1c5467b157efe9a50b69b8a3d5a2086c351 Mon Sep 17 00:00:00 2001 From: John Clements Date: Mon, 8 Jul 2013 10:37:07 -0700 Subject: [PATCH 42/67] add fold_mac clause to fun_to_ctxt_folder --- src/libsyntax/ext/expand.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 67c083e4db8b9..6ede2f7ab35c6 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -928,13 +928,20 @@ pub fn fun_to_ctxt_folder(cf: @T) -> @AstFoldFns { |ast::ident{name, ctxt}, _| { ast::ident{name:name,ctxt:cf.f(ctxt)} }; - // we've also got to pick up macro invocations; they can - // appear as exprs, stmts, items, and types. urg, it's going - // to be easier just to add a fold_mac, I think. - //let fold_ex : @ + let fm : @fn(&ast::mac_, span, @ast_fold) -> (ast::mac_,span) = + |m, sp, fld| { + match *m { + mac_invoc_tt(ref path, ref tts, ctxt) => + (mac_invoc_tt(fld.fold_path(path), + fold_tts(*tts,fld), + cf.f(ctxt)), + sp) + } + + }; @AstFoldFns{ fold_ident : fi, - // check that it works, then add the fold_expr clause.... + fold_mac : fm, .. *afp } } From ed71739c0cd25270b2590565e73fecea81a97c7c Mon Sep 17 00:00:00 2001 From: John Clements Date: Mon, 8 Jul 2013 15:55:14 -0700 Subject: [PATCH 43/67] capturing macros now implemented --- src/libsyntax/ext/base.rs | 84 ++++++++++----- src/libsyntax/ext/expand.rs | 153 +++++++++++++++++++++------- src/libsyntax/ext/quote.rs | 59 ++++++++--- src/libsyntax/ext/tt/macro_rules.rs | 4 +- src/libsyntax/fold.rs | 1 + 5 files changed, 219 insertions(+), 82 deletions(-) diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index bf07848b2b566..ceedfb77057d8 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -33,6 +33,10 @@ pub struct MacroDef { ext: SyntaxExtension } +// No context arg for an Item Decorator macro, simply because +// adding it would require adding a ctxt field to all items. +// we could do this if it turns out to be useful. + pub type ItemDecoratorFun = @fn(@ExtCtxt, span, @ast::MetaItem, @@ -41,15 +45,34 @@ pub type ItemDecoratorFun = @fn(@ExtCtxt, pub type SyntaxExpanderTTFun = @fn(@ExtCtxt, span, - &[ast::token_tree]) + &[ast::token_tree], + ast::SyntaxContext) -> MacResult; pub type SyntaxExpanderTTItemFun = @fn(@ExtCtxt, + span, + ast::ident, + ~[ast::token_tree], + ast::SyntaxContext) + -> MacResult; + +// oog... in order to make the presentation of builtin_normal_tt_no_ctxt +// and builtin_ident_tt_no_ctxt palatable, we need one-off types for +// functions that don't consume a ctxt: + +pub type SyntaxExpanderTTFunNoCtxt = @fn(@ExtCtxt, + span, + &[ast::token_tree]) + -> MacResult; + +pub type SyntaxExpanderTTItemFunNoCtxt = @fn(@ExtCtxt, span, ast::ident, ~[ast::token_tree]) -> MacResult; + + pub enum MacResult { MRExpr(@ast::expr), MRItem(@ast::item), @@ -78,6 +101,7 @@ pub enum SyntaxExtension { IdentTT(SyntaxExpanderTTItemFun, Option), } + // The SyntaxEnv is the environment that's threaded through the expansion // of macros. It contains bindings for macros, and also a special binding // for " block" (not a legal identifier) that maps to a BlockInfo @@ -109,12 +133,16 @@ type RenameList = ~[(ast::ident,Name)]; // AST nodes into full ASTs pub fn syntax_expander_table() -> SyntaxEnv { // utility function to simplify creating NormalTT syntax extensions - fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> @Transformer { - @SE(NormalTT(f, None)) + // that ignore their contexts + fn builtin_normal_tt_no_ctxt(f: SyntaxExpanderTTFunNoCtxt) -> @Transformer { + let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,d|{f(a,b,c)}; + @SE(NormalTT(wrapped_expander, None)) } // utility function to simplify creating IdentTT syntax extensions - fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> @Transformer { - @SE(IdentTT(f, None)) + // that ignore their contexts + fn builtin_item_tt_no_ctxt(f: SyntaxExpanderTTItemFunNoCtxt) -> @Transformer { + let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,e|{f(a,b,c,d)}; + @SE(IdentTT(wrapped_expander, None)) } let mut syntax_expanders = HashMap::new(); // NB identifier starts with space, and can't conflict with legal idents @@ -124,10 +152,10 @@ pub fn syntax_expander_table() -> SyntaxEnv { pending_renames : @mut ~[] })); syntax_expanders.insert(intern(&"macro_rules"), - builtin_item_tt( + builtin_item_tt_no_ctxt( ext::tt::macro_rules::add_new_extension)); syntax_expanders.insert(intern(&"fmt"), - builtin_normal_tt(ext::fmt::expand_syntax_ext)); + builtin_normal_tt_no_ctxt(ext::fmt::expand_syntax_ext)); syntax_expanders.insert( intern(&"auto_encode"), @SE(ItemDecorator(ext::auto_encode::expand_auto_encode))); @@ -135,14 +163,14 @@ pub fn syntax_expander_table() -> SyntaxEnv { intern(&"auto_decode"), @SE(ItemDecorator(ext::auto_encode::expand_auto_decode))); syntax_expanders.insert(intern(&"env"), - builtin_normal_tt(ext::env::expand_syntax_ext)); + builtin_normal_tt_no_ctxt(ext::env::expand_syntax_ext)); syntax_expanders.insert(intern("bytes"), - builtin_normal_tt(ext::bytes::expand_syntax_ext)); + builtin_normal_tt_no_ctxt(ext::bytes::expand_syntax_ext)); syntax_expanders.insert(intern("concat_idents"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::concat_idents::expand_syntax_ext)); syntax_expanders.insert(intern(&"log_syntax"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::log_syntax::expand_syntax_ext)); syntax_expanders.insert(intern(&"deriving"), @SE(ItemDecorator( @@ -150,49 +178,49 @@ pub fn syntax_expander_table() -> SyntaxEnv { // Quasi-quoting expanders syntax_expanders.insert(intern(&"quote_tokens"), - builtin_normal_tt(ext::quote::expand_quote_tokens)); + @SE(NormalTT(ext::quote::expand_quote_tokens, None))); syntax_expanders.insert(intern(&"quote_expr"), - builtin_normal_tt(ext::quote::expand_quote_expr)); + @SE(NormalTT(ext::quote::expand_quote_expr, None))); syntax_expanders.insert(intern(&"quote_ty"), - builtin_normal_tt(ext::quote::expand_quote_ty)); + @SE(NormalTT(ext::quote::expand_quote_ty, None))); syntax_expanders.insert(intern(&"quote_item"), - builtin_normal_tt(ext::quote::expand_quote_item)); + @SE(NormalTT(ext::quote::expand_quote_item, None))); syntax_expanders.insert(intern(&"quote_pat"), - builtin_normal_tt(ext::quote::expand_quote_pat)); + @SE(NormalTT(ext::quote::expand_quote_pat, None))); syntax_expanders.insert(intern(&"quote_stmt"), - builtin_normal_tt(ext::quote::expand_quote_stmt)); + @SE(NormalTT(ext::quote::expand_quote_stmt, None))); syntax_expanders.insert(intern(&"line"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_line)); syntax_expanders.insert(intern(&"col"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_col)); syntax_expanders.insert(intern(&"file"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_file)); syntax_expanders.insert(intern(&"stringify"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_stringify)); syntax_expanders.insert(intern(&"include"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_include)); syntax_expanders.insert(intern(&"include_str"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_include_str)); syntax_expanders.insert(intern(&"include_bin"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_include_bin)); syntax_expanders.insert(intern(&"module_path"), - builtin_normal_tt( + builtin_normal_tt_no_ctxt( ext::source_util::expand_mod)); syntax_expanders.insert(intern(&"proto"), - builtin_item_tt(ext::pipes::expand_proto)); + builtin_item_tt_no_ctxt(ext::pipes::expand_proto)); syntax_expanders.insert(intern(&"asm"), - builtin_normal_tt(ext::asm::expand_asm)); + builtin_normal_tt_no_ctxt(ext::asm::expand_asm)); syntax_expanders.insert( intern(&"trace_macros"), - builtin_normal_tt(ext::trace_macros::expand_trace_macros)); + builtin_normal_tt_no_ctxt(ext::trace_macros::expand_trace_macros)); MapChain::new(~syntax_expanders) } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 6ede2f7ab35c6..3ee5499439955 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -9,7 +9,7 @@ // except according to those terms. use ast::{Block, Crate, decl_local, empty_ctxt, expr_, expr_mac, Local, mac_invoc_tt}; -use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi}; +use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi, SyntaxContext}; use ast::{SCTable, token_tree, illegal_ctxt}; use ast; use ast_util::{new_rename, new_mark, mtwt_resolve}; @@ -39,6 +39,11 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, // entry-point for all syntax extensions. expr_mac(ref mac) => { match (*mac).node { + // it would almost certainly be cleaner to pass the whole + // macro invocation in, rather than pulling it apart and + // marking the tts and the ctxt separately. This also goes + // for the other three macro invocation chunks of code + // in this file. // Token-tree macros: mac_invoc_tt(ref pth, ref tts, ctxt) => { if (pth.idents.len() > 1u) { @@ -67,7 +72,8 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, let fm = fresh_mark(); // mark before: let marked_before = mark_tts(*tts,fm); - let expanded = match expandfun(cx, mac.span, marked_before) { + let marked_ctxt = new_mark(fm, ctxt); + let expanded = match expandfun(cx, mac.span, marked_before, marked_ctxt) { MRExpr(e) => e, MRAny(expr_maker,_,_) => expr_maker(), _ => { @@ -207,9 +213,9 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, cx: @ExtCtxt, it: @ast::item, fld: @ast_fold) -> Option<@ast::item> { - let (pth, tts) = match it.node { + let (pth, tts, ctxt) = match it.node { item_mac(codemap::spanned { node: mac_invoc_tt(ref pth, ref tts, ctxt), _}) => { - (pth, (*tts).clone()) + (pth, (*tts).clone(), ctxt) } _ => cx.span_bug(it.span, "invalid item macro invocation") }; @@ -236,10 +242,9 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, } }); // mark before expansion: - let marked_tts = mark_tts(tts,fm); - // mark after expansion: - // RIGHT HERE: can't apply mark_item to MacResult ... :( - expander(cx, it.span, marked_tts) + let marked_before = mark_tts(tts,fm); + let marked_ctxt = new_mark(fm,ctxt); + expander(cx, it.span, marked_before, marked_ctxt) } Some(@SE(IdentTT(expander, span))) => { if it.ident.name == parse::token::special_idents::invalid.name { @@ -257,7 +262,8 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, let fm = fresh_mark(); // mark before expansion: let marked_tts = mark_tts(tts,fm); - expander(cx, it.span, it.ident, marked_tts) + let marked_ctxt = new_mark(fm,ctxt); + expander(cx, it.span, it.ident, marked_tts, marked_ctxt) } _ => cx.span_fatal( it.span, fmt!("%s! is not legal in item position", extnamestr)) @@ -308,11 +314,11 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, -> (Option, span) { // why the copying here and not in expand_expr? // looks like classic changed-in-only-one-place - let (mac, pth, tts, semi) = match *s { + let (mac, pth, tts, semi, ctxt) = match *s { stmt_mac(ref mac, semi) => { match mac.node { mac_invoc_tt(ref pth, ref tts, ctxt) => { - ((*mac).clone(), pth, (*tts).clone(), semi) + ((*mac).clone(), pth, (*tts).clone(), semi, ctxt) } } } @@ -338,7 +344,8 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv, let fm = fresh_mark(); // mark before expansion: let marked_tts = mark_tts(tts,fm); - let expanded = match expandfun(cx, mac.span, marked_tts) { + let marked_ctxt = new_mark(fm,ctxt); + let expanded = match expandfun(cx, mac.span, marked_tts, marked_ctxt) { MRExpr(e) => @codemap::spanned { node: stmt_expr(e, cx.next_id()), span: e.span}, @@ -518,27 +525,6 @@ pub fn new_expr_finder() -> @Visitor<@mut ~[@ast::expr]> { } } - -// given a mutable list of renames, return a tree-folder that applies those -// renames. -// FIXME #4536: currently pub to allow testing -pub fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @ast_fold { - let afp = default_ast_fold(); - let f_pre = @AstFoldFns { - fold_ident: |id,_| { - // the individual elements are memoized... it would - // also be possible to memoize on the whole list at once. - let new_ctxt = renames.iter().fold(id.ctxt,|ctxt,&(from,to)| { - new_rename(from,to,ctxt) - }); - ast::ident{name:id.name,ctxt:new_ctxt} - }, - .. *afp - }; - make_fold(f_pre) -} - - // expand a block. pushes a new exts_frame, then calls expand_block_elts pub fn expand_block(extsbox: @mut SyntaxEnv, _cx: @ExtCtxt, @@ -901,6 +887,7 @@ pub trait CtxtFn{ pub fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext; } +// a renamer adds a rename to the syntax context pub struct Renamer { from : ast::ident, to : ast::Name @@ -912,6 +899,22 @@ impl CtxtFn for Renamer { } } +// a renamer that performs a whole bunch of renames +pub struct MultiRenamer { + renames : @mut ~[(ast::ident,ast::Name)] +} + +impl CtxtFn for MultiRenamer { + pub fn f(&self, starting_ctxt : ast::SyntaxContext) -> ast::SyntaxContext { + // the individual elements are memoized... it would + // also be possible to memoize on the whole list at once. + self.renames.iter().fold(starting_ctxt,|ctxt,&(from,to)| { + new_rename(from,to,ctxt) + }) + } +} + +// a marker adds the given mark to the syntax context pub struct Marker { mark : Mrk } impl CtxtFn for Marker { @@ -920,6 +923,15 @@ impl CtxtFn for Marker { } } +// a repainter just replaces the given context with the one it's closed over +pub struct Repainter { ctxt : SyntaxContext } + +impl CtxtFn for Repainter { + pub fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext { + self.ctxt + } +} + // given a function from ctxts to ctxts, produce // an ast_fold that applies that function to all ctxts: pub fn fun_to_ctxt_folder(cf: @T) -> @AstFoldFns { @@ -946,6 +958,15 @@ pub fn fun_to_ctxt_folder(cf: @T) -> @AstFoldFns { } } + + +// given a mutable list of renames, return a tree-folder that applies those +// renames. +// FIXME #4536: currently pub to allow testing +pub fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @AstFoldFns { + fun_to_ctxt_folder(@MultiRenamer{renames : renames}) +} + // just a convenience: pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) } pub fn new_rename_folder(from : ast::ident, to : ast::Name) -> @AstFoldFns { @@ -987,6 +1008,12 @@ fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> { new_mark_folder(m).fold_item(expr) } +// replace all contexts in a given expr with the given mark. Used +// for capturing macros +pub fn replace_ctxts(expr : @ast::expr, ctxt : SyntaxContext) -> @ast::expr { + fun_to_ctxt_folder(@Repainter{ctxt:ctxt}).fold_expr(expr) +} + #[cfg(test)] mod test { use super::*; @@ -996,7 +1023,7 @@ mod test { use codemap; use codemap::spanned; use parse; - use parse::token::{gensym, intern, get_ident_interner}; + use parse::token::{gensym, intern, get_ident_interner, ident_to_str}; use print::pprust; use std; use std::vec; @@ -1131,8 +1158,9 @@ mod test { } } - fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) { - pprust::print_mod(s, &crate.module, crate.attrs); + fn fake_print_crate(crate: @ast::Crate) { + let s = pprust::rust_printer(std::io::stderr(),get_ident_interner()); + pprust::print_crate_(s, crate); } fn expand_crate_str(crate_str: @str) -> @ast::Crate { @@ -1259,6 +1287,59 @@ mod test { } } + #[test] fn quote_expr_test() { + quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_expr!(dontcare);}"); + } + #[test] fn quote_item_test() { + quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_item!(dontcare);}"); + } + #[test] fn quote_pat_test() { + quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_pat!(dontcare);}"); + } + #[test] fn quote_ty_test() { + quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_ty!(dontcare);}"); + } + #[test] fn quote_tokens_test() { + quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_tokens!(dontcare);}"); + } + + fn quote_ext_cx_test(crate_str : @str) { + let crate = expand_crate_str(crate_str); + let nv = new_name_finder(); + let pv = new_path_finder(); + // find the ext_cx binding + let bindings = @mut ~[]; + visit::visit_crate(crate, (bindings, mk_vt(nv))); + let cxbinds : ~[&ast::ident] = bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect(); + let cxbind = match cxbinds { + [b] => b, + _ => fail!("expected just one binding for ext_cx") + }; + let resolved_binding = mtwt_resolve(*cxbind); + // find all the ext_cx varrefs: + let varrefs = @mut ~[]; + visit::visit_crate(crate, (varrefs, mk_vt(pv))); + // the ext_cx binding should bind all of the ext_cx varrefs: + for varrefs.iter().filter(|p|{ p.idents.len() == 1 + && (@"ext_cx" == (ident_to_str(&p.idents[0]))) + }).enumerate().advance |(idx,v)| { + if (mtwt_resolve(v.idents[0]) != resolved_binding) { + std::io::println("uh oh, ext_cx binding didn't match ext_cx varref:"); + std::io::println(fmt!("this is varref # %?",idx)); + std::io::println(fmt!("binding: %?",cxbind)); + std::io::println(fmt!("resolves to: %?",resolved_binding)); + std::io::println(fmt!("varref: %?",v.idents[0])); + std::io::println(fmt!("resolves to: %?",mtwt_resolve(v.idents[0]))); + let table = get_sctable(); + std::io::println("SC table:"); + for table.table.iter().enumerate().advance |(idx,val)| { + std::io::println(fmt!("%4u : %?",idx,val)); + } + } + assert_eq!(mtwt_resolve(v.idents[0]),resolved_binding); + }; + } + #[test] fn pat_idents(){ let pat = string_to_pat(@"(a,Foo{x:c @ (b,9),y:Bar(4,d)})"); diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 1439f4cabab14..78d7b78af320b 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -12,6 +12,7 @@ use ast; use codemap::{BytePos, Pos, span}; use ext::base::ExtCtxt; use ext::base; +use ext::expand; use ext::build::AstBuilder; use parse::token::*; use parse::token; @@ -365,46 +366,72 @@ pub mod rt { pub fn expand_quote_tokens(cx: @ExtCtxt, sp: span, - tts: &[ast::token_tree]) -> base::MacResult { - base::MRExpr(expand_tts(cx, sp, tts)) + tts: &[ast::token_tree], + ctxt: ast::SyntaxContext) -> base::MacResult { + let expanded = expand_tts(cx, sp, tts); + // repaint the expanded code so it's as though it was the original text. + let repainted = expand::replace_ctxts(expanded,ctxt); + base::MRExpr(repainted) } pub fn expand_quote_expr(cx: @ExtCtxt, sp: span, - tts: &[ast::token_tree]) -> base::MacResult { - base::MRExpr(expand_parse_call(cx, sp, "parse_expr", ~[], tts)) + tts: &[ast::token_tree], + ctxt: ast::SyntaxContext) -> base::MacResult { + let expanded = expand_parse_call(cx, sp, "parse_expr", ~[], tts); + // repaint the expanded code so it's as though it was the original text. + let repainted = expand::replace_ctxts(expanded,ctxt); + base::MRExpr(repainted) } +// these probably need to be capturing, too... + pub fn expand_quote_item(cx: @ExtCtxt, sp: span, - tts: &[ast::token_tree]) -> base::MacResult { + tts: &[ast::token_tree], + ctxt: ast::SyntaxContext) -> base::MacResult { let e_attrs = cx.expr_vec_uniq(sp, ~[]); - base::MRExpr(expand_parse_call(cx, sp, "parse_item", - ~[e_attrs], tts)) + let expanded = expand_parse_call(cx, sp, "parse_item", + ~[e_attrs], tts); + // repaint the expanded code so it's as though it was the original text. + let repainted = expand::replace_ctxts(expanded,ctxt); + base::MRExpr(repainted) } pub fn expand_quote_pat(cx: @ExtCtxt, sp: span, - tts: &[ast::token_tree]) -> base::MacResult { + tts: &[ast::token_tree], + ctxt: ast::SyntaxContext) -> base::MacResult { let e_refutable = cx.expr_lit(sp, ast::lit_bool(true)); - base::MRExpr(expand_parse_call(cx, sp, "parse_pat", - ~[e_refutable], tts)) + let expanded = expand_parse_call(cx, sp, "parse_pat", + ~[e_refutable], tts); + // repaint the expanded code so it's as though it was the original text. + let repainted = expand::replace_ctxts(expanded,ctxt); + base::MRExpr(repainted) } pub fn expand_quote_ty(cx: @ExtCtxt, sp: span, - tts: &[ast::token_tree]) -> base::MacResult { + tts: &[ast::token_tree], + ctxt: ast::SyntaxContext) -> base::MacResult { let e_param_colons = cx.expr_lit(sp, ast::lit_bool(false)); - base::MRExpr(expand_parse_call(cx, sp, "parse_ty", - ~[e_param_colons], tts)) + let expanded = expand_parse_call(cx, sp, "parse_ty", + ~[e_param_colons], tts); + // repaint the expanded code so it's as though it was the original text. + let repainted = expand::replace_ctxts(expanded,ctxt); + base::MRExpr(repainted) } pub fn expand_quote_stmt(cx: @ExtCtxt, sp: span, - tts: &[ast::token_tree]) -> base::MacResult { + tts: &[ast::token_tree], + ctxt: ast::SyntaxContext) -> base::MacResult { let e_attrs = cx.expr_vec_uniq(sp, ~[]); - base::MRExpr(expand_parse_call(cx, sp, "parse_stmt", - ~[e_attrs], tts)) + let expanded = expand_parse_call(cx, sp, "parse_stmt", + ~[e_attrs], tts); + // repaint the expanded code so it's as though it was the original text. + let repainted = expand::replace_ctxts(expanded,ctxt); + base::MRExpr(repainted) } fn ids_ext(strs: ~[~str]) -> ~[ast::ident] { diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 5631f0377975b..a3f3446606ba2 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -150,8 +150,8 @@ pub fn add_new_extension(cx: @ExtCtxt, cx.span_fatal(best_fail_spot, best_fail_msg); } - let exp: @fn(@ExtCtxt, span, &[ast::token_tree]) -> MacResult = - |cx, sp, arg| generic_extension(cx, sp, name, arg, *lhses, *rhses); + let exp: @fn(@ExtCtxt, span, &[ast::token_tree], ctxt: ast::SyntaxContext) -> MacResult = + |cx, sp, arg, ctxt| generic_extension(cx, sp, name, arg, *lhses, *rhses); return MRDef(MacroDef{ name: ident_to_str(&name), diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 9db28ce00dddd..6223166808c2b 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -209,6 +209,7 @@ pub fn noop_fold_crate(c: &Crate, fld: @ast_fold) -> Crate { } fn noop_fold_view_item(vi: &view_item_, _fld: @ast_fold) -> view_item_ { + // FIXME #7654: doesn't iterate over idents in a view_item_use return /* FIXME (#2543) */ (*vi).clone(); } From 025ae38cdf5e7e7e7a5f05dd02d210958c45a1de Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 11:43:54 -0700 Subject: [PATCH 44/67] comments --- src/libsyntax/parse/token.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 779be26991b25..eb62b66cbd2ce 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -706,6 +706,7 @@ pub fn is_reserved_keyword(tok: &Token) -> bool { } } +// not currently used anywhere... pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool { if (*t1 == *t2) { true From d96481b4b81860eb5934b7ea822c82de5c8c2f07 Mon Sep 17 00:00:00 2001 From: John Clements Date: Tue, 9 Jul 2013 14:24:03 -0700 Subject: [PATCH 45/67] remove dead code --- src/libsyntax/ext/expand.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 3ee5499439955..6d1beda017640 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -973,21 +973,6 @@ pub fn new_rename_folder(from : ast::ident, to : ast::Name) -> @AstFoldFns { fun_to_ctxt_folder(@Renamer{from:from,to:to}) } -/* -// perform resolution (in the MTWT sense) on all of the -// idents in the tree. This is the final step in expansion. -// FIXME #6993: this function could go away, along with -// the separate mtwt_resolution pass -pub fn new_ident_resolver() -> - @fn(ast::ident)->ast::ident { - |id : ast::ident| - ast::ident { - name : mtwt_resolve(id), - ctxt : empty_ctxt - } -} -*/ - // apply a given mark to the given token trees. Used prior to expansion of a macro. fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] { fold_tts(tts,new_mark_folder(m) as @ast_fold) From 1435c515ce0a66fc95f7412cb7dd2fbc1cb11a2f Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 10 Jul 2013 11:52:39 -0700 Subject: [PATCH 46/67] fix one remaining token comparison, refactor token comparison to avoid == check --- src/libsyntax/ext/tt/macro_parser.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index a60feb066ec75..78d8a43c40453 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -234,16 +234,12 @@ pub fn parse_or_else( } } -// temporary for testing +// perform a token equality check, ignoring syntax context (that is, an unhygienic comparison) pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { - if (*t1 == *t2) { - true - } else { - match (t1,t2) { - (&token::IDENT(id1,_),&token::IDENT(id2,_)) => - id1.name == id2.name, - _ => false - } + match (t1,t2) { + (&token::IDENT(id1,_),&token::IDENT(id2,_)) => + id1.name == id2.name, + _ => *t1 == *t2 } } @@ -310,7 +306,10 @@ pub fn parse( // the *_t vars are workarounds for the lack of unary move match ei.sep { Some(ref t) if idx == len => { // we need a separator - if tok == (*t) { //pass the separator + // i'm conflicted about whether this should be hygienic.... + // though in this case, if the separators are never legal + // idents, it shouldn't matter. + if token_name_eq(&tok, t) { //pass the separator let mut ei_t = ei.clone(); ei_t.idx += 1; next_eis.push(ei_t); @@ -367,7 +366,7 @@ pub fn parse( } /* error messages here could be improved with links to orig. rules */ - if tok == EOF { + if token_name_eq(&tok, &EOF) { if eof_eis.len() == 1u { let mut v = ~[]; for eof_eis[0u].matches.mut_iter().advance |dv| { From b17d7293bd9ecc2fc992eddbf591407fd6140094 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 10 Jul 2013 11:52:59 -0700 Subject: [PATCH 47/67] added utility function --- src/libsyntax/ast_util.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index 2855946fc09fc..da8b54ae7d270 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -795,6 +795,32 @@ pub fn getLast(arr: &~[Mrk]) -> uint { *arr.last() } +// are two paths equal when compared unhygienically? +// since I'm using this to replace ==, it seems appropriate +// to compare the span, global, etc. fields as well. +pub fn path_name_eq(a : &ast::Path, b : &ast::Path) -> bool { + (a.span == b.span) + && (a.global == b.global) + // NOTE: ident->name in lifetimes! + && (a.rp == b.rp) + // NOTE: can a type contain an ident? + && (a.types == b.types) + && (idents_name_eq(a.idents, b.idents)) +} + +// are two arrays of idents equal when compared unhygienically? +pub fn idents_name_eq(a : &[ast::ident], b : &[ast::ident]) -> bool { + if (a.len() != b.len()) { + false + } else { + for a.iter().enumerate().advance |(idx,id)|{ + if (id.name != b[idx].name) { + return false; + } + } + true + } +} #[cfg(test)] mod test { @@ -802,6 +828,17 @@ mod test { use super::*; use std::io; + #[test] fn idents_name_eq_test() { + assert!(idents_name_eq(~[ident{name:3,ctxt:4}, + ident{name:78,ctxt:82}], + ~[ident{name:3,ctxt:104}, + ident{name:78,ctxt:182}])); + assert!(!idents_name_eq(~[ident{name:3,ctxt:4}, + ident{name:78,ctxt:82}], + ~[ident{name:3,ctxt:104}, + ident{name:77,ctxt:182}])); + } + #[test] fn xorpush_test () { let mut s = ~[]; xorPush(&mut s,14); From 7d2c6271bc724705e5f1d6d3ecfdc081cbf5eda9 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 10 Jul 2013 13:44:58 -0700 Subject: [PATCH 48/67] ident->name in NamedField, elsewhere --- src/librustc/middle/borrowck/mod.rs | 2 +- src/librustc/middle/entry.rs | 2 +- src/librustc/middle/mem_categorization.rs | 7 ++++--- src/librustc/middle/moves.rs | 2 +- src/librustc/middle/privacy.rs | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/librustc/middle/borrowck/mod.rs b/src/librustc/middle/borrowck/mod.rs index de06f267e1e90..bb8b0e24a93df 100644 --- a/src/librustc/middle/borrowck/mod.rs +++ b/src/librustc/middle/borrowck/mod.rs @@ -750,7 +750,7 @@ impl BorrowckCtxt { match fname { mc::NamedField(ref fname) => { out.push_char('.'); - out.push_str(token::ident_to_str(fname)); + out.push_str(token::interner_get(*fname)); } mc::PositionalField(idx) => { out.push_char('#'); // invent a notation here diff --git a/src/librustc/middle/entry.rs b/src/librustc/middle/entry.rs index 28927764a9f15..d2a077651b7dc 100644 --- a/src/librustc/middle/entry.rs +++ b/src/librustc/middle/entry.rs @@ -69,7 +69,7 @@ pub fn find_entry_point(session: Session, crate: &Crate, ast_map: ast_map::map) fn find_item(item: @item, ctxt: @mut EntryContext, visitor: EntryVisitor) { match item.node { item_fn(*) => { - if item.ident == special_idents::main { + if item.ident.name == special_idents::main.name { match ctxt.ast_map.find(&item.id) { Some(&ast_map::node_item(_, path)) => { if path.len() == 0 { diff --git a/src/librustc/middle/mem_categorization.rs b/src/librustc/middle/mem_categorization.rs index 8416e0212d845..fa5e6bd307a80 100644 --- a/src/librustc/middle/mem_categorization.rs +++ b/src/librustc/middle/mem_categorization.rs @@ -57,6 +57,7 @@ use syntax::ast::{m_imm, m_const, m_mutbl}; use syntax::ast; use syntax::codemap::span; use syntax::print::pprust; +use syntax::parse::token; #[deriving(Eq)] pub enum categorization { @@ -101,7 +102,7 @@ pub enum InteriorKind { #[deriving(Eq, IterBytes)] pub enum FieldName { - NamedField(ast::ident), + NamedField(ast::Name), PositionalField(uint) } @@ -624,7 +625,7 @@ impl mem_categorization_ctxt { @cmt_ { id: node.id(), span: node.span(), - cat: cat_interior(base_cmt, InteriorField(NamedField(f_name))), + cat: cat_interior(base_cmt, InteriorField(NamedField(f_name.name))), mutbl: base_cmt.mutbl.inherit(), ty: f_ty } @@ -1239,7 +1240,7 @@ pub fn ptr_sigil(ptr: ptr_kind) -> ~str { impl Repr for InteriorKind { fn repr(&self, tcx: ty::ctxt) -> ~str { match *self { - InteriorField(NamedField(fld)) => tcx.sess.str_of(fld).to_owned(), + InteriorField(NamedField(fld)) => token::interner_get(fld).to_owned(), InteriorField(PositionalField(i)) => fmt!("#%?", i), InteriorElement(_) => ~"[]", } diff --git a/src/librustc/middle/moves.rs b/src/librustc/middle/moves.rs index d8ce0a9663631..57c841a8b07e1 100644 --- a/src/librustc/middle/moves.rs +++ b/src/librustc/middle/moves.rs @@ -417,7 +417,7 @@ impl VisitContext { // specified and (2) have a type that // moves-by-default: let consume_with = with_fields.iter().any(|tf| { - !fields.iter().any(|f| f.ident == tf.ident) && + !fields.iter().any(|f| f.ident.name == tf.ident.name) && ty::type_moves_by_default(self.tcx, tf.mt.ty) }); diff --git a/src/librustc/middle/privacy.rs b/src/librustc/middle/privacy.rs index e768a6d687c2e..ea813a193939f 100644 --- a/src/librustc/middle/privacy.rs +++ b/src/librustc/middle/privacy.rs @@ -207,7 +207,7 @@ pub fn check_crate<'mm>(tcx: ty::ctxt, |span, id, ident| { let fields = ty::lookup_struct_fields(tcx, id); for fields.iter().advance |field| { - if field.ident != ident { loop; } + if field.ident.name != ident.name { loop; } if field.vis == private { tcx.sess.span_err(span, fmt!("field `%s` is private", token::ident_to_str(&ident))); From f5f4d79bb7f7a84b4443c713c79ba9d21e646690 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 11 Jul 2013 14:41:57 -0700 Subject: [PATCH 49/67] uncomment mtwt_resolve calls --- src/librustc/middle/resolve.rs | 14 ++++++++++---- src/libsyntax/ext/tt/macro_parser.rs | 4 ++-- src/libsyntax/parse/token.rs | 12 ++++-------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/librustc/middle/resolve.rs b/src/librustc/middle/resolve.rs index 6ec5badcdf7a4..d44df64dfd75f 100644 --- a/src/librustc/middle/resolve.rs +++ b/src/librustc/middle/resolve.rs @@ -22,7 +22,7 @@ use middle::pat_util::pat_bindings; use syntax::ast::*; use syntax::ast; -use syntax::ast_util::{def_id_of_def, local_def}; // mtwt_resolve +use syntax::ast_util::{def_id_of_def, local_def, mtwt_resolve}; use syntax::ast_util::{path_to_ident, walk_pat, trait_method_to_ty_method}; use syntax::ast_util::{Privacy, Public, Private}; use syntax::ast_util::{variant_visibility_to_privacy, visibility_to_privacy}; @@ -4006,10 +4006,14 @@ impl Resolver { None, visitor); } + // build a map from pattern identifiers to binding-info's. + // this is done hygienically. This could arise for a macro + // that expands into an or-pattern where one 'x' was from the + // user and one 'x' came from the macro. pub fn binding_mode_map(@mut self, pat: @pat) -> BindingMap { let mut result = HashMap::new(); do pat_bindings(self.def_map, pat) |binding_mode, _id, sp, path| { - let name = path_to_ident(path).name; // mtwt_resolve(path_to_ident(path)); + let name = mtwt_resolve(path_to_ident(path)); result.insert(name, binding_info {span: sp, binding_mode: binding_mode}); @@ -4017,6 +4021,8 @@ impl Resolver { return result; } + // check that all of the arms in an or-pattern have exactly the + // same set of bindings, with the same binding modes for each. pub fn check_consistent_bindings(@mut self, arm: &arm) { if arm.pats.len() == 0 { return; } let map_0 = self.binding_mode_map(arm.pats[0]); @@ -4212,7 +4218,7 @@ impl Resolver { // what you want). let ident = path.idents[0]; - let renamed = ident.name; // mtwt_resolve(ident); + let renamed = mtwt_resolve(ident); match self.resolve_bare_identifier_pattern(ident) { FoundStructOrEnumVariant(def) @@ -4731,7 +4737,7 @@ impl Resolver { let search_result; match namespace { ValueNS => { - let renamed = ident.name; // mtwt_resolve(ident); + let renamed = mtwt_resolve(ident); search_result = self.search_ribs(self.value_ribs, renamed, span, DontAllowCapturingSelf); diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 78d8a43c40453..df82fd4a0e256 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -355,8 +355,8 @@ pub fn parse( match_nonterminal(_,_,_) => { bb_eis.push(ei) } match_tok(ref t) => { let mut ei_t = ei.clone(); - if (token_name_eq(t,&tok)) { - //if (token::mtwt_token_eq(t,&tok)) { + //if (token_name_eq(t,&tok)) { + if (token::mtwt_token_eq(t,&tok)) { ei_t.idx += 1; next_eis.push(ei_t); } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index eb62b66cbd2ce..468d671b01d91 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -708,14 +708,10 @@ pub fn is_reserved_keyword(tok: &Token) -> bool { // not currently used anywhere... pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool { - if (*t1 == *t2) { - true - } else { - match (t1,t2) { - (&IDENT(id1,_),&IDENT(id2,_)) => - ast_util::mtwt_resolve(id1) == ast_util::mtwt_resolve(id2), - _ => false - } + match (t1,t2) { + (&IDENT(id1,_),&IDENT(id2,_)) => + ast_util::mtwt_resolve(id1) == ast_util::mtwt_resolve(id2), + _ => *t1 == *t2 } } From c9a9cf725204f346d4a7cf0301f417b49a9d409b Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 10 Jul 2013 16:40:09 -0700 Subject: [PATCH 50/67] memoization for resolve --- src/libsyntax/ast_util.rs | 111 ++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 29 deletions(-) diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index da8b54ae7d270..ce747fa16be3f 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -725,30 +725,63 @@ fn idx_push(vec: &mut ~[T], val: T) -> uint { /// Resolve a syntax object to a name, per MTWT. pub fn mtwt_resolve(id : ident) -> Name { - resolve_internal(id, get_sctable()) + resolve_internal(id, get_sctable(), get_resolve_table()) +} + +// FIXME #4536: must be pub for testing +pub type ResolveTable = HashMap<(Name,SyntaxContext),Name>; + +// okay, I admit, putting this in TLS is not so nice: +// fetch the SCTable from TLS, create one if it doesn't yet exist. +pub fn get_resolve_table() -> @mut ResolveTable { + static resolve_table_key: local_data::Key<@@mut ResolveTable> = &local_data::Key; + match local_data::get(resolve_table_key, |k| k.map(|&k| *k)) { + None => { + let new_table = @@mut HashMap::new(); + local_data::set(resolve_table_key,new_table); + *new_table + }, + Some(intr) => *intr + } } // Resolve a syntax object to a name, per MTWT. +// adding memoization to possibly resolve 500+ seconds in resolve for librustc (!) // FIXME #4536 : currently pub to allow testing -pub fn resolve_internal(id : ident, table : &mut SCTable) -> Name { - match table.table[id.ctxt] { - EmptyCtxt => id.name, - // ignore marks here: - Mark(_,subctxt) => resolve_internal(ident{name:id.name, ctxt: subctxt},table), - // do the rename if necessary: - Rename(ident{name,ctxt},toname,subctxt) => { - // this could be cached or computed eagerly: - let resolvedfrom = resolve_internal(ident{name:name,ctxt:ctxt},table); - let resolvedthis = resolve_internal(ident{name:id.name,ctxt:subctxt},table); - if ((resolvedthis == resolvedfrom) - && (marksof(ctxt,resolvedthis,table) - == marksof(subctxt,resolvedthis,table))) { - toname - } else { - resolvedthis - } +pub fn resolve_internal(id : ident, + table : &mut SCTable, + resolve_table : &mut ResolveTable) -> Name { + let key = (id.name,id.ctxt); + match resolve_table.contains_key(&key) { + false => { + let resolved = { + match table.table[id.ctxt] { + EmptyCtxt => id.name, + // ignore marks here: + Mark(_,subctxt) => resolve_internal(ident{name:id.name, ctxt: subctxt},table,resolve_table), + // do the rename if necessary: + Rename(ident{name,ctxt},toname,subctxt) => { + let resolvedfrom = resolve_internal(ident{name:name,ctxt:ctxt},table,resolve_table); + let resolvedthis = resolve_internal(ident{name:id.name,ctxt:subctxt},table,resolve_table); + if ((resolvedthis == resolvedfrom) + && (marksof(ctxt,resolvedthis,table) + == marksof(subctxt,resolvedthis,table))) { + toname + } else { + resolvedthis + } + } + IllegalCtxt() => fail!(~"expected resolvable context, got IllegalCtxt") + } + }; + resolve_table.insert(key,resolved); + resolved + } + true => { + // it's guaranteed to be there, because we just checked that it was + // there and we never remove anything from the table: + *(resolve_table.find(&key).get()) } - IllegalCtxt() => fail!(~"expected resolvable context, got IllegalCtxt") } } @@ -827,6 +860,7 @@ mod test { use ast::*; use super::*; use std::io; + use std::hash::HashMap; #[test] fn idents_name_eq_test() { assert!(idents_name_eq(~[ident{name:3,ctxt:4}, @@ -968,29 +1002,30 @@ mod test { #[test] fn resolve_tests () { let a = 40; let mut t = new_sctable_internal(); + let mut rt = HashMap::new(); // - ctxt is MT - assert_eq!(resolve_internal(id(a,empty_ctxt),&mut t),a); + assert_eq!(resolve_internal(id(a,empty_ctxt),&mut t, &mut rt),a); // - simple ignored marks { let sc = unfold_marks(~[1,2,3],empty_ctxt,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t),a);} + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt),a);} // - orthogonal rename where names don't match { let sc = unfold_test_sc(~[R(id(50,empty_ctxt),51),M(12)],empty_ctxt,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t),a);} + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt),a);} // - rename where names do match, but marks don't { let sc1 = new_mark_internal(1,empty_ctxt,&mut t); let sc = unfold_test_sc(~[R(id(a,sc1),50), M(1), M(2)], empty_ctxt,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t), a);} + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), a);} // - rename where names and marks match { let sc1 = unfold_test_sc(~[M(1),M(2)],empty_ctxt,&mut t); let sc = unfold_test_sc(~[R(id(a,sc1),50),M(1),M(2)],empty_ctxt,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t), 50); } + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), 50); } // - rename where names and marks match by literal sharing { let sc1 = unfold_test_sc(~[M(1),M(2)],empty_ctxt,&mut t); let sc = unfold_test_sc(~[R(id(a,sc1),50)],sc1,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t), 50); } + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), 50); } // - two renames of the same var.. can only happen if you use // local-expand to prevent the inner binding from being renamed // during the rename-pass caused by the first: @@ -998,22 +1033,28 @@ mod test { { let sc = unfold_test_sc(~[R(id(a,empty_ctxt),50), R(id(a,empty_ctxt),51)], empty_ctxt,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t), 51); } + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), 51); } // the simplest double-rename: { let a_to_a50 = new_rename_internal(id(a,empty_ctxt),50,empty_ctxt,&mut t); let a50_to_a51 = new_rename_internal(id(a,a_to_a50),51,a_to_a50,&mut t); - assert_eq!(resolve_internal(id(a,a50_to_a51),&mut t),51); + assert_eq!(resolve_internal(id(a,a50_to_a51),&mut t, &mut rt),51); // mark on the outside doesn't stop rename: let sc = new_mark_internal(9,a50_to_a51,&mut t); - assert_eq!(resolve_internal(id(a,sc),&mut t),51); + assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt),51); // but mark on the inside does: let a50_to_a51_b = unfold_test_sc(~[R(id(a,a_to_a50),51), M(9)], a_to_a50, &mut t); - assert_eq!(resolve_internal(id(a,a50_to_a51_b),&mut t),50);} + assert_eq!(resolve_internal(id(a,a50_to_a51_b),&mut t, &mut rt),50);} + } + + #[test] fn mtwt_resolve_test(){ + let a = 40; + assert_eq!(mtwt_resolve(id(a,empty_ctxt)),a); } + #[test] fn hashing_tests () { let mut t = new_sctable_internal(); assert_eq!(new_mark_internal(12,empty_ctxt,&mut t),2); @@ -1023,4 +1064,16 @@ mod test { // I'm assuming that the rename table will behave the same.... } + #[test] fn resolve_table_hashing_tests() { + let mut t = new_sctable_internal(); + let mut rt = HashMap::new(); + assert_eq!(rt.len(),0); + resolve_internal(id(30,empty_ctxt),&mut t, &mut rt); + assert_eq!(rt.len(),1); + resolve_internal(id(39,empty_ctxt),&mut t, &mut rt); + assert_eq!(rt.len(),2); + resolve_internal(id(30,empty_ctxt),&mut t, &mut rt); + assert_eq!(rt.len(),2); + } + } From 203bbe031dca1eb407f303bd5b0f6872eebf3c01 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 11 Jul 2013 22:58:14 -0700 Subject: [PATCH 51/67] whitespace, reindentation, and comments only --- src/librustc/middle/trans/base.rs | 3 +- src/librustc/middle/trans/expr.rs | 5 ++-- src/libsyntax/ast.rs | 8 +++++- src/libsyntax/ast_util.rs | 13 +++++---- src/libsyntax/ext/expand.rs | 43 ++++++++++++++++------------- src/libsyntax/ext/tt/macro_rules.rs | 1 - 6 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/librustc/middle/trans/base.rs b/src/librustc/middle/trans/base.rs index e77b884211f19..363d1ca865b73 100644 --- a/src/librustc/middle/trans/base.rs +++ b/src/librustc/middle/trans/base.rs @@ -2760,7 +2760,8 @@ pub fn declare_intrinsics(llmod: ModuleRef) -> HashMap<&'static str, ValueRef> { pub fn declare_dbg_intrinsics(llmod: ModuleRef, intrinsics: &mut HashMap<&'static str, ValueRef>) { ifn!(intrinsics, "llvm.dbg.declare", [Type::metadata(), Type::metadata()], Type::void()); - ifn!(intrinsics, "llvm.dbg.value", [Type::metadata(), Type::i64(), Type::metadata()], Type::void()); + ifn!(intrinsics, + "llvm.dbg.value", [Type::metadata(), Type::i64(), Type::metadata()], Type::void()); } pub fn trap(bcx: @mut Block) { diff --git a/src/librustc/middle/trans/expr.rs b/src/librustc/middle/trans/expr.rs index ba58c94eefdeb..64426ef1b2f42 100644 --- a/src/librustc/middle/trans/expr.rs +++ b/src/librustc/middle/trans/expr.rs @@ -1139,8 +1139,9 @@ fn trans_rec_or_struct(bcx: @mut Block, let mut need_base = vec::from_elem(field_tys.len(), true); let numbered_fields = do fields.map |field| { - let opt_pos = field_tys.iter().position(|field_ty| - field_ty.ident.name == field.ident.name); + let opt_pos = + field_tys.iter().position(|field_ty| + field_ty.ident.name == field.ident.name); match opt_pos { Some(i) => { need_base[i] = false; diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 156b4d66aabe4..f8f00fc34e51d 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -36,6 +36,12 @@ impl Eq for ident { if (self.ctxt == other.ctxt) { self.name == other.name } else { + // IF YOU SEE ONE OF THESE FAILS: it means that you're comparing + // idents that have different contexts. You can't fix this without + // knowing whether the comparison should be hygienic or non-hygienic. + // if it should be non-hygienic (most things are), just compare the + // 'name' fields of the idents. Or, even better, replace the idents + // with Name's. fail!(fmt!("not allowed to compare these idents: %?, %?", self, other)); } } @@ -114,6 +120,7 @@ pub type fn_ident = Option; pub struct Lifetime { id: node_id, span: span, + // FIXME #7743 : change this to Name! ident: ident } @@ -622,7 +629,6 @@ pub enum matcher_ { // lo, hi position-in-match-array used: match_seq(~[matcher], Option<::parse::token::Token>, bool, uint, uint), // parse a Rust NT: name to bind, name of NT, position in match array: - // NOTE: 'name of NT' shouldnt really be represented as an ident, should it? match_nonterminal(ident, ident, uint) } diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index ce747fa16be3f..43f5a5d14079b 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -758,11 +758,14 @@ pub fn resolve_internal(id : ident, match table.table[id.ctxt] { EmptyCtxt => id.name, // ignore marks here: - Mark(_,subctxt) => resolve_internal(ident{name:id.name, ctxt: subctxt},table,resolve_table), + Mark(_,subctxt) => + resolve_internal(ident{name:id.name, ctxt: subctxt},table,resolve_table), // do the rename if necessary: Rename(ident{name,ctxt},toname,subctxt) => { - let resolvedfrom = resolve_internal(ident{name:name,ctxt:ctxt},table,resolve_table); - let resolvedthis = resolve_internal(ident{name:id.name,ctxt:subctxt},table,resolve_table); + let resolvedfrom = + resolve_internal(ident{name:name,ctxt:ctxt},table,resolve_table); + let resolvedthis = + resolve_internal(ident{name:id.name,ctxt:subctxt},table,resolve_table); if ((resolvedthis == resolvedfrom) && (marksof(ctxt,resolvedthis,table) == marksof(subctxt,resolvedthis,table))) { @@ -834,9 +837,9 @@ pub fn getLast(arr: &~[Mrk]) -> uint { pub fn path_name_eq(a : &ast::Path, b : &ast::Path) -> bool { (a.span == b.span) && (a.global == b.global) - // NOTE: ident->name in lifetimes! + // FIXME #7743: ident->name in lifetimes! && (a.rp == b.rp) - // NOTE: can a type contain an ident? + // can types contain idents? && (a.types == b.types) && (idents_name_eq(a.idents, b.idents)) } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 6d1beda017640..7fbf03d4c3dc0 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -73,19 +73,20 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv, // mark before: let marked_before = mark_tts(*tts,fm); let marked_ctxt = new_mark(fm, ctxt); - let expanded = match expandfun(cx, mac.span, marked_before, marked_ctxt) { - MRExpr(e) => e, - MRAny(expr_maker,_,_) => expr_maker(), - _ => { - cx.span_fatal( - pth.span, - fmt!( - "non-expr macro in expr pos: %s", - extnamestr + let expanded = + match expandfun(cx, mac.span, marked_before, marked_ctxt) { + MRExpr(e) => e, + MRAny(expr_maker,_,_) => expr_maker(), + _ => { + cx.span_fatal( + pth.span, + fmt!( + "non-expr macro in expr pos: %s", + extnamestr + ) ) - ) - } - }; + } + }; // mark after: let marked_after = mark_expr(expanded,fm); @@ -1012,7 +1013,8 @@ mod test { use print::pprust; use std; use std::vec; - use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item, string_to_pat, strs_to_idents}; + use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item} + use util::parser_testing::{string_to_pat, strs_to_idents}; use visit::{mk_vt}; use visit; @@ -1208,12 +1210,14 @@ mod test { ~[~[0]]) // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm // commenting it out. - // the z flows into and out of two macros (g & f) along one path, and one (just g) along the - // other, so the result of the whole thing should be "let z_123 = 3; z_123" - //"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) + // the z flows into and out of two macros (g & f) along one path, and one + // (just g) along the other, so the result of the whole thing should + // be "let z_123 = 3; z_123" + //"macro_rules! g (($x:ident) => + // ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)})) // fn a(){g!(z)}" - // create a really evil test case where a $x appears inside a binding of $x but *shouldnt* - // bind because it was inserted by a different macro.... + // create a really evil test case where a $x appears inside a binding of $x + // but *shouldnt* bind because it was inserted by a different macro.... ]; for tests.iter().advance |s| { run_renaming_test(s); @@ -1295,7 +1299,8 @@ mod test { // find the ext_cx binding let bindings = @mut ~[]; visit::visit_crate(crate, (bindings, mk_vt(nv))); - let cxbinds : ~[&ast::ident] = bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect(); + let cxbinds : ~[&ast::ident] = + bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect(); let cxbind = match cxbinds { [b] => b, _ => fail!("expected just one binding for ext_cx") diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index a3f3446606ba2..928f252984018 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -49,7 +49,6 @@ pub fn add_new_extension(cx: @ExtCtxt, // ...quasiquoting this would be nice. let argument_gram = ~[ ms(match_seq(~[ - // NOTE : probably just use an enum for the NT_name ? ms(match_nonterminal(lhs_nm, special_idents::matchers, 0u)), ms(match_tok(FAT_ARROW)), ms(match_nonterminal(rhs_nm, special_idents::tt, 1u)), From c0b5ab59a38f1684bef2334e51d464f19c9b1b32 Mon Sep 17 00:00:00 2001 From: John Clements Date: Thu, 11 Jul 2013 23:07:34 -0700 Subject: [PATCH 52/67] remove unneeded imports, clean up unused var warnings --- src/libsyntax/ext/base.rs | 4 ++-- src/libsyntax/ext/expand.rs | 16 ++++++++-------- src/libsyntax/parse/token.rs | 8 ++------ src/libsyntax/print/pprust.rs | 6 ++++-- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index ceedfb77057d8..fc244463a0f36 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -135,13 +135,13 @@ pub fn syntax_expander_table() -> SyntaxEnv { // utility function to simplify creating NormalTT syntax extensions // that ignore their contexts fn builtin_normal_tt_no_ctxt(f: SyntaxExpanderTTFunNoCtxt) -> @Transformer { - let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,d|{f(a,b,c)}; + let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,_d|{f(a,b,c)}; @SE(NormalTT(wrapped_expander, None)) } // utility function to simplify creating IdentTT syntax extensions // that ignore their contexts fn builtin_item_tt_no_ctxt(f: SyntaxExpanderTTItemFunNoCtxt) -> @Transformer { - let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,e|{f(a,b,c,d)}; + let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,_e|{f(a,b,c,d)}; @SE(IdentTT(wrapped_expander, None)) } let mut syntax_expanders = HashMap::new(); diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 7fbf03d4c3dc0..47205bbc53fc5 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -8,11 +8,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Block, Crate, decl_local, empty_ctxt, expr_, expr_mac, Local, mac_invoc_tt}; +use ast::{Block, Crate, decl_local, expr_, expr_mac, Local, mac_invoc_tt}; use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi, SyntaxContext}; -use ast::{SCTable, token_tree, illegal_ctxt}; +use ast::{token_tree}; use ast; -use ast_util::{new_rename, new_mark, mtwt_resolve}; +use ast_util::{new_rename, new_mark}; use attr; use attr::AttrMetaMethods; use codemap; @@ -431,7 +431,7 @@ fn expand_non_macro_stmt (exts: SyntaxEnv, let new_name = fresh_name(ident); new_pending_renames.push((*ident,new_name)); } - let mut rename_fld = renames_to_fold(new_pending_renames); + let rename_fld = renames_to_fold(new_pending_renames); // rewrite the pattern using the new names (the old ones // have already been applied): let rewritten_pat = rename_fld.fold_pat(expanded_pat); @@ -531,7 +531,7 @@ pub fn expand_block(extsbox: @mut SyntaxEnv, _cx: @ExtCtxt, blk: &Block, fld: @ast_fold, - orig: @fn(&Block, @ast_fold) -> Block) + _orig: @fn(&Block, @ast_fold) -> Block) -> Block { // see note below about treatment of exts table with_exts_frame!(extsbox,false, @@ -542,7 +542,7 @@ pub fn expand_block(extsbox: @mut SyntaxEnv, pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block { let block_info = get_block_info(exts); let pending_renames = block_info.pending_renames; - let mut rename_fld = renames_to_fold(pending_renames); + let rename_fld = renames_to_fold(pending_renames); let new_view_items = b.view_items.map(|x| fld.fold_view_item(x)); let mut new_stmts = ~[]; for b.stmts.iter().advance |x| { @@ -928,7 +928,7 @@ impl CtxtFn for Marker { pub struct Repainter { ctxt : SyntaxContext } impl CtxtFn for Repainter { - pub fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext { + pub fn f(&self, _ctxt : ast::SyntaxContext) -> ast::SyntaxContext { self.ctxt } } @@ -1013,7 +1013,7 @@ mod test { use print::pprust; use std; use std::vec; - use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item} + use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item}; use util::parser_testing::{string_to_pat, strs_to_idents}; use visit::{mk_vt}; use visit; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 468d671b01d91..06c8da956c860 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -16,11 +16,7 @@ use util::interner::StrInterner; use util::interner; use std::cast; -use std::char; -use std::cmp::Equiv; use std::local_data; -use std::rand; -use std::rand::RngUtil; #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)] pub enum binop { @@ -557,8 +553,8 @@ pub fn str_ptr_eq(a : @str, b : @str) -> bool { let q : uint = cast::transmute(b); let result = p == q; // got to transmute them back, to make sure the ref count is correct: - let junk1 : @str = cast::transmute(p); - let junk2 : @str = cast::transmute(q); + let _junk1 : @str = cast::transmute(p); + let _junk2 : @str = cast::transmute(q); result } } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index dbc729d4f1c2e..abd2af6dba632 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -617,7 +617,8 @@ pub fn print_item(s: @ps, item: &ast::item) { } bclose(s, item.span); } - ast::item_mac(codemap::spanned { node: ast::mac_invoc_tt(ref pth, ref tts, ctxt), + // I think it's reasonable to hide the context here: + ast::item_mac(codemap::spanned { node: ast::mac_invoc_tt(ref pth, ref tts, _), _}) => { print_visibility(s, item.vis); print_path(s, pth, false); @@ -1019,7 +1020,8 @@ pub fn print_if(s: @ps, test: &ast::expr, blk: &ast::Block, pub fn print_mac(s: @ps, m: &ast::mac) { match m.node { - ast::mac_invoc_tt(ref pth, ref tts, ctxt) => { + // I think it's reasonable to hide the ctxt here: + ast::mac_invoc_tt(ref pth, ref tts, _) => { print_path(s, pth, false); word(s.s, "!"); popen(s); From 13811b392a369e2bc846f87f2b35983d116e772e Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 12 Jul 2013 00:28:48 -0700 Subject: [PATCH 53/67] ident->name --- src/librustc/middle/typeck/check/_match.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/librustc/middle/typeck/check/_match.rs b/src/librustc/middle/typeck/check/_match.rs index c16e9a0a92856..8daf27ffb88e6 100644 --- a/src/librustc/middle/typeck/check/_match.rs +++ b/src/librustc/middle/typeck/check/_match.rs @@ -284,13 +284,13 @@ pub fn check_struct_pat_fields(pcx: &pat_ctxt, // Index the class fields. let mut field_map = HashMap::new(); for class_fields.iter().enumerate().advance |(i, class_field)| { - field_map.insert(class_field.ident, i); + field_map.insert(class_field.ident.name, i); } // Typecheck each field. let mut found_fields = HashSet::new(); for fields.iter().advance |field| { - match field_map.find(&field.ident) { + match field_map.find(&field.ident.name) { Some(&index) => { let class_field = class_fields[index]; let field_type = ty::lookup_field_type(tcx, From 77fe4e2b8908e0543edae9f46896bc704b7b520e Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 12 Jul 2013 01:18:59 -0700 Subject: [PATCH 54/67] xfailed unhygienic test --- src/test/run-pass/syntax-extension-minor.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/run-pass/syntax-extension-minor.rs b/src/test/run-pass/syntax-extension-minor.rs index 497a55b7c78e2..593784e282670 100644 --- a/src/test/run-pass/syntax-extension-minor.rs +++ b/src/test/run-pass/syntax-extension-minor.rs @@ -8,6 +8,9 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +// xfail-test +// this now fails (correctly, I claim) because hygiene prevents +// the assembled identifier from being a reference to the binding. pub fn main() { let asdf_fdsa = ~"<.<"; From 5da521c9d96a3ca54f657cf70c049363a5c526a3 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 12 Jul 2013 08:55:12 -0700 Subject: [PATCH 55/67] ident->name --- src/libsyntax/print/pprust.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index abd2af6dba632..204ea5d5b4d0e 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1897,7 +1897,7 @@ pub fn print_arg(s: @ps, input: &ast::arg) { match input.pat.node { ast::pat_ident(_, ref path, _) if path.idents.len() == 1 && - path.idents[0] == parse::token::special_idents::invalid => { + path.idents[0].name == parse::token::special_idents::invalid.name => { // Do nothing. } _ => { From 46a15eb44f1e1592af0d9e25de1e97df06836fd2 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 12 Jul 2013 18:35:05 -0700 Subject: [PATCH 56/67] expose mtwt_marksof for testing --- src/libsyntax/ast_util.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index 43f5a5d14079b..f4ca6c55b4a1f 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -789,6 +789,11 @@ pub fn resolve_internal(id : ident, } /// Compute the marks associated with a syntax context. +pub fn mtwt_marksof(ctxt: SyntaxContext, stopname: Name) -> ~[Mrk] { + marksof(ctxt, stopname, get_sctable()) +} + +// the internal function for computing marks // it's not clear to me whether it's better to use a [] mutable // vector or a cons-list for this. pub fn marksof(ctxt: SyntaxContext, stopname: Name, table: &SCTable) -> ~[Mrk] { From a295e6e4c29fcd8cf0b7be090a4dbd22bfb15729 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 12 Jul 2013 18:35:47 -0700 Subject: [PATCH 57/67] awesome new bug! added test case --- src/libsyntax/ext/expand.rs | 82 +++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 47205bbc53fc5..3cb9e1534554d 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -260,7 +260,6 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv, span: span } }); - let fm = fresh_mark(); // mark before expansion: let marked_tts = mark_tts(tts,fm); let marked_ctxt = new_mark(fm,ctxt); @@ -1005,7 +1004,7 @@ mod test { use super::*; use ast; use ast::{Attribute_, AttrOuter, MetaWord, empty_ctxt}; - use ast_util::{get_sctable, mtwt_resolve, new_ident, new_rename}; + use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_ident, new_rename}; use codemap; use codemap::spanned; use parse; @@ -1183,31 +1182,40 @@ mod test { // // The comparisons are done post-mtwt-resolve, so we're comparing renamed // names; differences in marks don't matter any more. - type renaming_test = (&'static str, ~[~[uint]]); + // + // oog... I also want tests that check "binding-identifier-=?". That is, + // not just "do these have the same name", but "do they have the same + // name *and* the same marks"? Understanding this is really pretty painful. + // in principle, you might want to control this boolean on a per-varref basis, + // but that would make things even harder to understand, and might not be + // necessary for thorough testing. + type renaming_test = (&'static str, ~[~[uint]], bool); #[test] fn automatic_renaming () { - // need some other way to test these... let tests : ~[renaming_test] = ~[// b & c should get new names throughout, in the expr too: ("fn a() -> int { let b = 13; let c = b; b+c }", - ~[~[0,1],~[2]]), + ~[~[0,1],~[2]], false), // both x's should be renamed (how is this causing a bug?) ("fn main () {let x : int = 13;x;}", - ~[~[0]]), + ~[~[0]], false), // the use of b after the + should be renamed, the other one not: ("macro_rules! f (($x:ident) => (b + $x)) fn a() -> int { let b = 13; f!(b)}", - ~[~[1]]), + ~[~[1]], false), // the b before the plus should not be renamed (requires marks) ("macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}", - ~[~[1]]), + ~[~[1]], false), // the marks going in and out of letty should cancel, allowing that $x to // capture the one following the semicolon. // this was an awesome test case, and caught a *lot* of bugs. ("macro_rules! letty(($x:ident) => (let $x = 15;)) macro_rules! user(($x:ident) => ({letty!($x); $x})) fn main() -> int {user!(z)}", - ~[~[0]]) + ~[~[0]], false), + // can't believe I missed this one : a macro def that refers to a local var: + ("fn main() {let x = 19; macro_rules! getx(()=>(x)); getx!();}", + ~[~[0]], true) // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm // commenting it out. // the z flows into and out of two macros (g & f) along one path, and one @@ -1224,12 +1232,12 @@ mod test { } } - + // run one of the renaming tests fn run_renaming_test(t : &renaming_test) { let nv = new_name_finder(); let pv = new_path_finder(); - let (teststr, bound_connections) = match *t { - (ref str,ref conns) => (str.to_managed(), conns.clone()) + let (teststr, bound_connections, bound_ident_check) = match *t { + (ref str,ref conns, bic) => (str.to_managed(), conns.clone(), bic) }; let cr = expand_crate_str(teststr.to_managed()); let bindings = @mut ~[]; @@ -1240,15 +1248,17 @@ mod test { assert_eq!(bindings.len(),bound_connections.len()); for bound_connections.iter().enumerate().advance |(binding_idx,shouldmatch)| { let binding_name = mtwt_resolve(bindings[binding_idx]); + let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,binding_name); // shouldmatch can't name varrefs that don't exist: assert!((shouldmatch.len() == 0) || (varrefs.len() > *shouldmatch.iter().max().get())); for varrefs.iter().enumerate().advance |(idx,varref)| { if shouldmatch.contains(&idx) { // it should be a path of length 1, and it should - // be free-identifier=? to the given binding + // be free-identifier=? or bound-identifier=? to the given binding assert_eq!(varref.idents.len(),1); let varref_name = mtwt_resolve(varref.idents[0]); + let varref_marks = mtwt_marksof(varref.idents[0].ctxt, binding_name); if (!(varref_name==binding_name)){ std::io::println("uh oh, should match but doesn't:"); std::io::println(fmt!("varref: %?",varref)); @@ -1260,6 +1270,10 @@ mod test { } } assert_eq!(varref_name,binding_name); + if (bound_ident_check) { + // we need to check the marks, too: + assert_eq!(varref_marks,binding_marks.clone()); + } } else { let fail = (varref.idents.len() == 1) && (mtwt_resolve(varref.idents[0]) == binding_name); @@ -1330,6 +1344,48 @@ mod test { }; } + #[test] fn fmt_in_macro_used_inside_module_macro() { + let crate_str = @"macro_rules! fmt_wrap(($b:expr)=>(fmt!(\"left: %?\", $b))) +macro_rules! foo_module (() => (mod generated { fn a() { let xx = 147; fmt_wrap!(xx);}})) +foo_module!() +"; + let cr = expand_crate_str(crate_str); + let nv = new_name_finder(); + let pv = new_path_finder(); + // find the xx binding + let bindings = @mut ~[]; + visit::visit_crate(cr, (bindings, mk_vt(nv))); + let cxbinds : ~[&ast::ident] = + bindings.iter().filter(|b|{@"xx" == (ident_to_str(*b))}).collect(); + let cxbind = match cxbinds { + [b] => b, + _ => fail!("expected just one binding for ext_cx") + }; + let resolved_binding = mtwt_resolve(*cxbind); + // find all the xx varrefs: + let varrefs = @mut ~[]; + visit::visit_crate(cr, (varrefs, mk_vt(pv))); + // the xx binding should bind all of the xx varrefs: + for varrefs.iter().filter(|p|{ p.idents.len() == 1 + && (@"xx" == (ident_to_str(&p.idents[0]))) + }).enumerate().advance |(idx,v)| { + if (mtwt_resolve(v.idents[0]) != resolved_binding) { + std::io::println("uh oh, xx binding didn't match xx varref:"); + std::io::println(fmt!("this is xx varref # %?",idx)); + std::io::println(fmt!("binding: %?",cxbind)); + std::io::println(fmt!("resolves to: %?",resolved_binding)); + std::io::println(fmt!("varref: %?",v.idents[0])); + std::io::println(fmt!("resolves to: %?",mtwt_resolve(v.idents[0]))); + let table = get_sctable(); + std::io::println("SC table:"); + for table.table.iter().enumerate().advance |(idx,val)| { + std::io::println(fmt!("%4u : %?",idx,val)); + } + } + assert_eq!(mtwt_resolve(v.idents[0]),resolved_binding); + }; + } + #[test] fn pat_idents(){ let pat = string_to_pat(@"(a,Foo{x:c @ (b,9),y:Bar(4,d)})"); From 2863a795a8889f7aa4839183c065e82afb9e0f87 Mon Sep 17 00:00:00 2001 From: John Clements Date: Sat, 13 Jul 2013 10:18:37 -0700 Subject: [PATCH 58/67] comment on hygienic context extension train fns --- src/libsyntax/ext/expand.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 3cb9e1534554d..82f4db9cc1532 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -882,6 +882,17 @@ pub fn expand_crate(parse_sess: @mut parse::ParseSess, @f.fold_crate(c) } +// HYGIENIC CONTEXT EXTENSION: +// all of these functions are for walking over +// ASTs and making some change to the context of every +// element that has one. a CtxtFn is a trait-ified +// version of a closure in (SyntaxContext -> SyntaxContext). +// the ones defined here include: +// Renamer - add a rename to a context +// MultiRenamer - add a set of renames to a context +// Marker - add a mark to a context +// Repainter - replace a context (maybe Replacer would be a better name?) + // a function in SyntaxContext -> SyntaxContext pub trait CtxtFn{ pub fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext; From 8479d054d4718efcc7b068bcd2b14eeefeb051cd Mon Sep 17 00:00:00 2001 From: John Clements Date: Sat, 13 Jul 2013 19:12:25 -0700 Subject: [PATCH 59/67] comments --- src/libsyntax/ast.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index f8f00fc34e51d..4dbc7a1283ffc 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -68,6 +68,15 @@ pub fn new_ident(name: Name) -> ident { ident {name: name, ctxt: empty_ctxt}} // storage. pub type SyntaxContext = uint; +// the SCTable contains a table of SyntaxContext_'s. It +// represents a flattened tree structure, to avoid having +// managed pointers everywhere (that caused an ICE). +// the mark_memo and rename_memo fields are side-tables +// that ensure that adding the same mark to the same context +// gives you back the same context as before. This shouldn't +// change the semantics--everything here is immutable--but +// it should cut down on memory use *a lot*; applying a mark +// to a tree containing 50 identifiers would otherwise generate pub struct SCTable { table : ~[SyntaxContext_], mark_memo : HashMap<(SyntaxContext,Mrk),SyntaxContext>, From f2cd518aeda275a4e3fe5a633582fcd30c52957e Mon Sep 17 00:00:00 2001 From: John Clements Date: Sun, 14 Jul 2013 15:23:56 -0400 Subject: [PATCH 60/67] added string_to_tts --- src/libsyntax/util/parser_testing.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs index 2d967d6ac3e94..46e1cc88e1838 100644 --- a/src/libsyntax/util/parser_testing.rs +++ b/src/libsyntax/util/parser_testing.rs @@ -22,6 +22,12 @@ pub fn string_to_tts_and_sess (source_str : @str) -> (~[ast::token_tree],@mut Pa (filemap_to_tts(ps,string_to_filemap(ps,source_str,@"bogofile")),ps) } +// map a string to tts, using a made-up filename: +pub fn string_to_tts(source_str : @str) -> ~[ast::token_tree] { + let (tts,_) = string_to_tts_and_sess(source_str); + tts +} + pub fn string_to_parser_and_sess(source_str: @str) -> (Parser,@mut ParseSess) { let ps = new_parse_sess(None); (new_parser_from_source_str(ps,~[],@"bogofile",source_str),ps) From af9898385e46be9799daee4be168df80dddee8b4 Mon Sep 17 00:00:00 2001 From: John Clements Date: Sun, 14 Jul 2013 15:25:04 -0400 Subject: [PATCH 61/67] WIP: adding mark-cancelling for macro_rules --- src/libsyntax/ast_util.rs | 10 +++++++ src/libsyntax/ext/base.rs | 3 +- src/libsyntax/ext/expand.rs | 46 ++++++++++++++++++++++++++--- src/libsyntax/ext/tt/macro_rules.rs | 5 +++- 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index f4ca6c55b4a1f..0786ed584a7ac 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -820,6 +820,16 @@ pub fn marksof(ctxt: SyntaxContext, stopname: Name, table: &SCTable) -> ~[Mrk] { } } +/// Return the outer mark for a context with a mark at the outside. +/// FAILS when outside is not a mark. +pub fn mtwt_outer_mark(ctxt: SyntaxContext) -> Mrk { + let sctable = get_sctable(); + match sctable.table[ctxt] { + ast::Mark(mrk,_) => mrk, + _ => fail!("can't retrieve outer mark when outside is not a mark") + } +} + /// Push a name... unless it matches the one on top, in which /// case pop and discard (so two of the same marks cancel) pub fn xorPush(marks: &mut ~[uint], mark: uint) { diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index fc244463a0f36..27d2564095099 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -152,8 +152,7 @@ pub fn syntax_expander_table() -> SyntaxEnv { pending_renames : @mut ~[] })); syntax_expanders.insert(intern(&"macro_rules"), - builtin_item_tt_no_ctxt( - ext::tt::macro_rules::add_new_extension)); + @SE(IdentTT(ext::tt::macro_rules::add_new_extension, None))); syntax_expanders.insert(intern(&"fmt"), builtin_normal_tt_no_ctxt(ext::fmt::expand_syntax_ext)); syntax_expanders.insert( diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 82f4db9cc1532..0adaf809d7e4d 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -12,7 +12,8 @@ use ast::{Block, Crate, decl_local, expr_, expr_mac, Local, mac_invoc_tt}; use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi, SyntaxContext}; use ast::{token_tree}; use ast; -use ast_util::{new_rename, new_mark}; +use ast_util::{mtwt_outer_mark, new_rename, new_mark}; +use ast_util; use attr; use attr::AttrMetaMethods; use codemap; @@ -979,7 +980,10 @@ pub fn renames_to_fold(renames : @mut ~[(ast::ident,ast::Name)]) -> @AstFoldFns } // just a convenience: -pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) } +pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { + fun_to_ctxt_folder(@Marker{mark:m}) +} + pub fn new_rename_folder(from : ast::ident, to : ast::Name) -> @AstFoldFns { fun_to_ctxt_folder(@Renamer{from:from,to:to}) } @@ -1010,6 +1014,16 @@ pub fn replace_ctxts(expr : @ast::expr, ctxt : SyntaxContext) -> @ast::expr { fun_to_ctxt_folder(@Repainter{ctxt:ctxt}).fold_expr(expr) } +// take the mark from the given ctxt (that has a mark at the outside), +// and apply it to everything in the token trees, thereby cancelling +// that mark. +pub fn mtwt_cancel_outer_mark(tts: &[ast::token_tree], ctxt: ast::SyntaxContext) + -> ~[ast::token_tree] { + let outer_mark = mtwt_outer_mark(ctxt); + mark_tts(tts,outer_mark) +} + + #[cfg(test)] mod test { use super::*; @@ -1018,13 +1032,15 @@ mod test { use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_ident, new_rename}; use codemap; use codemap::spanned; + use fold; use parse; - use parse::token::{gensym, intern, get_ident_interner, ident_to_str}; + use parse::token::{fresh_mark, gensym, intern, get_ident_interner, ident_to_str}; + use parse::token; use print::pprust; use std; use std::vec; use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item}; - use util::parser_testing::{string_to_pat, strs_to_idents}; + use util::parser_testing::{string_to_pat, string_to_tts, strs_to_idents}; use visit::{mk_vt}; use visit; @@ -1124,6 +1140,28 @@ mod test { } } + #[test] fn cancel_outer_mark_test(){ + let invalid_name = token::special_idents::invalid.name; + let ident_str = @"x"; + let tts = string_to_tts(ident_str); + let fm = fresh_mark(); + let marked_once = fold::fold_tts(tts,new_mark_folder(fm) as @fold::ast_fold); + assert_eq!(marked_once.len(),1); + let marked_once_ctxt = + match marked_once[0] { + ast::tt_tok(_,token::IDENT(id,_)) => id.ctxt, + _ => fail!(fmt!("unexpected shape for marked tts: %?",marked_once[0])) + }; + assert_eq!(mtwt_marksof(marked_once_ctxt,invalid_name),~[fm]); + let remarked = mtwt_cancel_outer_mark(marked_once,marked_once_ctxt); + assert_eq!(remarked.len(),1); + match remarked[0] { + ast::tt_tok(_,token::IDENT(id,_)) => + assert_eq!(mtwt_marksof(id.ctxt,invalid_name),~[]), + _ => fail!(fmt!("unexpected shape for marked tts: %?",remarked[0])) + } + } + #[test] fn renaming () { let item_ast = string_to_crate(@"fn f() -> int { a }"); diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 928f252984018..3321cc1416058 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -14,6 +14,7 @@ use ast; use codemap::{span, spanned, dummy_sp}; use ext::base::{ExtCtxt, MacResult, MRAny, MRDef, MacroDef, NormalTT}; use ext::base; +use ext::expand; use ext::tt::macro_parser::{error}; use ext::tt::macro_parser::{named_match, matched_seq, matched_nonterminal}; use ext::tt::macro_parser::{parse, parse_or_else, success, failure}; @@ -29,8 +30,10 @@ use print; pub fn add_new_extension(cx: @ExtCtxt, sp: span, name: ident, - arg: ~[ast::token_tree]) + arg: ~[ast::token_tree], + stx_ctxt: ast::SyntaxContext) -> base::MacResult { + let arg = expand::mtwt_cancel_outer_mark(arg,stx_ctxt); // Wrap a matcher_ in a spanned to produce a matcher. // these spans won't matter, anyways fn ms(m: matcher_) -> matcher { From b7c99ee5865a1024a572b88faa12836ca840e15f Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 26 Jul 2013 12:57:30 -0400 Subject: [PATCH 62/67] fixed a bug that caused double-expand-traversal of macros that expand into modules. --- src/libsyntax/ext/expand.rs | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 0adaf809d7e4d..2b7fda4eb0a4c 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -182,25 +182,16 @@ pub fn expand_item(extsbox: @mut SyntaxEnv, fld: @ast_fold, orig: @fn(@ast::item, @ast_fold) -> Option<@ast::item>) -> Option<@ast::item> { - // need to do expansion first... it might turn out to be a module. - let maybe_it = match it.node { - ast::item_mac(*) => expand_item_mac(extsbox, cx, it, fld), - _ => Some(it) - }; - match maybe_it { - Some(it) => { - match it.node { - ast::item_mod(_) | ast::item_foreign_mod(_) => { - cx.mod_push(it.ident); - let macro_escape = contains_macro_escape(it.attrs); - let result = with_exts_frame!(extsbox,macro_escape,orig(it,fld)); - cx.mod_pop(); - result - } - _ => orig(it,fld) - } - } - None => None + match it.node { + ast::item_mac(*) => expand_item_mac(extsbox, cx, it, fld), + ast::item_mod(_) | ast::item_foreign_mod(_) => { + cx.mod_push(it.ident); + let macro_escape = contains_macro_escape(it.attrs); + let result = with_exts_frame!(extsbox,macro_escape,orig(it,fld)); + cx.mod_pop(); + result + }, + _ => orig(it,fld) } } From 16d65911b55cbcab2195b413e759549bfd370af9 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 26 Jul 2013 13:27:38 -0400 Subject: [PATCH 63/67] add display_sctable fn to ast_util --- src/libsyntax/ast_util.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index 0786ed584a7ac..36cc774fc99bb 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -717,6 +717,15 @@ pub fn get_sctable() -> @mut SCTable { } } +/// print out an SCTable for debugging +pub fn display_sctable(table : &SCTable) { + debug!("SC table:"); + for table.table.iter().enumerate().advance |(idx,val)| { + debug!("%4u : %?",idx,val); + } +} + + /// Add a value to the end of a vec, return its index fn idx_push(vec: &mut ~[T], val: T) -> uint { vec.push(val); From b1c3411bac04c1ac11ce49edb6efc1149c986a4f Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 26 Jul 2013 14:40:07 -0400 Subject: [PATCH 64/67] add test case, cleanup --- src/libsyntax/ext/expand.rs | 29 ++++++++++++++--------------- src/libsyntax/ext/tt/macro_rules.rs | 2 ++ src/libsyntax/parse/token.rs | 2 -- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 2b7fda4eb0a4c..65414dc70e2e7 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -13,7 +13,6 @@ use ast::{item_mac, Mrk, stmt_, stmt_decl, stmt_mac, stmt_expr, stmt_semi, Synta use ast::{token_tree}; use ast; use ast_util::{mtwt_outer_mark, new_rename, new_mark}; -use ast_util; use attr; use attr::AttrMetaMethods; use codemap; @@ -1021,6 +1020,7 @@ mod test { use ast; use ast::{Attribute_, AttrOuter, MetaWord, empty_ctxt}; use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_ident, new_rename}; + use ast_util; use codemap; use codemap::spanned; use fold; @@ -1029,8 +1029,7 @@ mod test { use parse::token; use print::pprust; use std; - use std::vec; - use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item}; + use util::parser_testing::{string_to_crate, string_to_crate_and_sess}; use util::parser_testing::{string_to_pat, string_to_tts, strs_to_idents}; use visit::{mk_vt}; use visit; @@ -1253,9 +1252,11 @@ mod test { macro_rules! user(($x:ident) => ({letty!($x); $x})) fn main() -> int {user!(z)}", ~[~[0]], false), - // can't believe I missed this one : a macro def that refers to a local var: - ("fn main() {let x = 19; macro_rules! getx(()=>(x)); getx!();}", - ~[~[0]], true) + // FIXME #8062: this test exposes a *potential* bug; our system does + // not behave exactly like MTWT, but I haven't thought of a way that + // this could cause a bug in Rust, yet. + // ("fn main() {let hrcoo = 19; macro_rules! getx(()=>(hrcoo)); getx!();}", + // ~[~[0]], true) // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm // commenting it out. // the z flows into and out of two macros (g & f) along one path, and one @@ -1274,6 +1275,7 @@ mod test { // run one of the renaming tests fn run_renaming_test(t : &renaming_test) { + let invalid_name = token::special_idents::invalid.name; let nv = new_name_finder(); let pv = new_path_finder(); let (teststr, bound_connections, bound_ident_check) = match *t { @@ -1288,7 +1290,7 @@ mod test { assert_eq!(bindings.len(),bound_connections.len()); for bound_connections.iter().enumerate().advance |(binding_idx,shouldmatch)| { let binding_name = mtwt_resolve(bindings[binding_idx]); - let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,binding_name); + let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,invalid_name); // shouldmatch can't name varrefs that don't exist: assert!((shouldmatch.len() == 0) || (varrefs.len() > *shouldmatch.iter().max().get())); @@ -1298,20 +1300,17 @@ mod test { // be free-identifier=? or bound-identifier=? to the given binding assert_eq!(varref.idents.len(),1); let varref_name = mtwt_resolve(varref.idents[0]); - let varref_marks = mtwt_marksof(varref.idents[0].ctxt, binding_name); + let varref_marks = mtwt_marksof(varref.idents[0].ctxt, invalid_name); if (!(varref_name==binding_name)){ std::io::println("uh oh, should match but doesn't:"); std::io::println(fmt!("varref: %?",varref)); std::io::println(fmt!("binding: %?", bindings[binding_idx])); - let table = get_sctable(); - std::io::println("SC table:"); - for table.table.iter().enumerate().advance |(idx,val)| { - std::io::println(fmt!("%4u : %?",idx,val)); - } + ast_util::display_sctable(get_sctable()); } assert_eq!(varref_name,binding_name); if (bound_ident_check) { - // we need to check the marks, too: + // we're checking bound-identifier=?, and the marks + // should be the same, too: assert_eq!(varref_marks,binding_marks.clone()); } } else { @@ -1322,7 +1321,7 @@ mod test { std::io::println("uh oh, matches but shouldn't:"); std::io::println(fmt!("varref: %?",varref)); std::io::println(fmt!("binding: %?", bindings[binding_idx])); - std::io::println(fmt!("sc_table: %?",get_sctable())); + ast_util::display_sctable(get_sctable()); } assert!(!fail); } diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 3321cc1416058..6e318f2256c7f 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -11,6 +11,7 @@ use ast::{ident, matcher_, matcher, match_tok, match_nonterminal, match_seq}; use ast::{tt_delim}; use ast; +use ast_util; use codemap::{span, spanned, dummy_sp}; use ext::base::{ExtCtxt, MacResult, MRAny, MRDef, MacroDef, NormalTT}; use ext::base; @@ -22,6 +23,7 @@ use parse::lexer::{new_tt_reader, reader}; use parse::parser::Parser; use parse::token::{get_ident_interner, special_idents, gensym_ident, ident_to_str}; use parse::token::{FAT_ARROW, SEMI, nt_matchers, nt_tt}; +use parse::token; use print; // this procedure performs the expansion of the diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 06c8da956c860..41e040f324d54 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -717,8 +717,6 @@ mod test { use ast; use ast_util; use super::*; - use std::io; - use std::managed; fn mark_ident(id : ast::ident, m : ast::Mrk) -> ast::ident { ast::ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)} From efcfaeb621b20fd6ca7562b413b4c0d329d64aba Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 26 Jul 2013 16:10:56 -0400 Subject: [PATCH 65/67] make macro hygienic --- src/test/bench/core-std.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/bench/core-std.rs b/src/test/bench/core-std.rs index 754181f9cd848..638eee1677365 100644 --- a/src/test/bench/core-std.rs +++ b/src/test/bench/core-std.rs @@ -25,20 +25,20 @@ use std::util; use std::vec; macro_rules! bench ( - ($id:ident) => (maybe_run_test(argv, stringify!($id).to_owned(), $id)) + ($argv:expr, $id:ident) => (maybe_run_test($argv, stringify!($id).to_owned(), $id)) ) fn main() { let argv = os::args(); let _tests = argv.slice(1, argv.len()); - bench!(shift_push); - bench!(read_line); - bench!(vec_plus); - bench!(vec_append); - bench!(vec_push_all); - bench!(is_utf8_ascii); - bench!(is_utf8_multibyte); + bench!(argv, shift_push); + bench!(argv, read_line); + bench!(argv, vec_plus); + bench!(argv, vec_append); + bench!(argv, vec_push_all); + bench!(argv, is_utf8_ascii); + bench!(argv, is_utf8_multibyte); } fn maybe_run_test(argv: &[~str], name: ~str, test: &fn()) { From ed08749308e965a602d1c601bd9de2c05f45789d Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 26 Jul 2013 16:30:05 -0400 Subject: [PATCH 66/67] add hygiene test, add copyright to another --- src/test/run-pass/hygiene-dodging-1.rs | 10 ++++++++++ src/test/run-pass/let-var-hygiene.rs | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 src/test/run-pass/let-var-hygiene.rs diff --git a/src/test/run-pass/hygiene-dodging-1.rs b/src/test/run-pass/hygiene-dodging-1.rs index 20ffa74e68775..55e15cc02dd39 100644 --- a/src/test/run-pass/hygiene-dodging-1.rs +++ b/src/test/run-pass/hygiene-dodging-1.rs @@ -1,3 +1,13 @@ +// Copyright 2012 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + mod x { pub fn g() -> uint {14} } diff --git a/src/test/run-pass/let-var-hygiene.rs b/src/test/run-pass/let-var-hygiene.rs new file mode 100644 index 0000000000000..1e29d2e8969c5 --- /dev/null +++ b/src/test/run-pass/let-var-hygiene.rs @@ -0,0 +1,16 @@ +// Copyright 2012 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// shouldn't affect evaluation of $ex: +macro_rules! bad_macro (($ex:expr) => ({let _x = 9; $ex})) +fn main() { + let _x = 8; + assert_eq!(bad_macro!(_x),8) +} From da2ed92c86c282682684958766f3457ee625bf56 Mon Sep 17 00:00:00 2001 From: John Clements Date: Sun, 28 Jul 2013 16:34:19 -0400 Subject: [PATCH 67/67] comment fix --- src/libsyntax/fold.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 6223166808c2b..e9fd04c29d02a 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -479,7 +479,7 @@ fn noop_fold_decl(d: &decl_, fld: @ast_fold) -> Option { } // lift a function in ast-thingy X fold -> ast-thingy to a function -// in (ast-thingy X span X fold) -> (ast-thingy X fold). Basically, +// in (ast-thingy X span X fold) -> (ast-thingy X span). Basically, // carries the span around. // It seems strange to me that the call to new_fold doesn't happen // here but instead in the impl down below.... probably just an