diff --git a/src/librustc/middle/save/span_utils.rs b/src/librustc/middle/save/span_utils.rs index 93ad29cff906a..44cad45519029 100644 --- a/src/librustc/middle/save/span_utils.rs +++ b/src/librustc/middle/save/span_utils.rs @@ -92,7 +92,7 @@ impl<'a> SpanUtils<'a> { let mut toks = self.retokenise_span(span); let mut bracket_count = 0u; loop { - let ts = toks.next_token(); + let ts = toks.real_token(); if ts.tok == token::Eof { return self.make_sub_span(span, result) } @@ -115,7 +115,7 @@ impl<'a> SpanUtils<'a> { let mut toks = self.retokenise_span(span); let mut bracket_count = 0u; loop { - let ts = toks.next_token(); + let ts = toks.real_token(); if ts.tok == token::Eof { return None; } @@ -137,13 +137,13 @@ impl<'a> SpanUtils<'a> { // any brackets, or the last span. pub fn sub_span_for_meth_name(&self, span: Span) -> Option { let mut toks = self.retokenise_span(span); - let mut prev = toks.next_token(); + let mut prev = toks.real_token(); let mut result = None; let mut bracket_count = 0u; let mut last_span = None; while prev.tok != token::Eof { last_span = None; - let mut next = toks.next_token(); + let mut next = toks.real_token(); if (next.tok == token::OpenDelim(token::Paren) || next.tok == token::Lt) && @@ -156,7 +156,7 @@ impl<'a> SpanUtils<'a> { next.tok == token::ModSep { let old = prev; prev = next; - next = toks.next_token(); + next = toks.real_token(); if next.tok == token::Lt && old.tok.is_ident() { result = Some(old.sp); @@ -185,11 +185,11 @@ impl<'a> SpanUtils<'a> { // brackets, or the last span. pub fn sub_span_for_type_name(&self, span: Span) -> Option { let mut toks = self.retokenise_span(span); - let mut prev = toks.next_token(); + let mut prev = toks.real_token(); let mut result = None; let mut bracket_count = 0u; loop { - let next = toks.next_token(); + let next = toks.real_token(); if (next.tok == token::Lt || next.tok == token::Colon) && @@ -234,7 +234,7 @@ impl<'a> SpanUtils<'a> { // We keep track of how many brackets we're nested in let mut bracket_count = 0i; loop { - let ts = toks.next_token(); + let ts = toks.real_token(); if ts.tok == token::Eof { if bracket_count != 0 { let loc = self.sess.codemap().lookup_char_pos(span.lo); @@ -263,12 +263,12 @@ impl<'a> SpanUtils<'a> { pub fn sub_span_before_token(&self, span: Span, tok: Token) -> Option { let mut toks = self.retokenise_span(span); - let mut prev = toks.next_token(); + let mut prev = toks.real_token(); loop { if prev.tok == token::Eof { return None; } - let next = toks.next_token(); + let next = toks.real_token(); if next.tok == tok { return self.make_sub_span(span, Some(prev.sp)); } @@ -281,15 +281,16 @@ impl<'a> SpanUtils<'a> { keyword: keywords::Keyword) -> Option { let mut toks = self.retokenise_span(span); loop { - let ts = toks.next_token(); + let ts = toks.real_token(); if ts.tok == token::Eof { return None; } if ts.tok.is_keyword(keyword) { - let ts = toks.next_token(); + let ts = toks.real_token(); if ts.tok == token::Eof { return None } else { + println!("found keyword: {} at {}", ts, ts.sp); return self.make_sub_span(span, Some(ts.sp)); } } diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 293b91111b5b2..1bc1d42d888dd 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -35,6 +35,19 @@ pub trait Reader { /// Report a non-fatal error with the current span. fn err(&self, &str); fn peek(&self) -> TokenAndSpan; + /// Get a token the parser cares about. + fn real_token(&mut self) -> TokenAndSpan { + let mut t = self.next_token(); + loop { + match t.tok { + token::Whitespace | token::Comment | token::Shebang(_) => { + t = self.next_token(); + }, + _ => break + } + } + t + } } #[deriving(Clone, PartialEq, Eq, Show)] diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index aa3b9668d4632..e3157d7ac755a 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -338,27 +338,13 @@ fn is_plain_ident_or_underscore(t: &token::Token) -> bool { t.is_plain_ident() || *t == token::Underscore } -/// Get a token the parser cares about -fn real_token(rdr: &mut Reader) -> TokenAndSpan { - let mut t = rdr.next_token(); - loop { - match t.tok { - token::Whitespace | token::Comment | token::Shebang(_) => { - t = rdr.next_token(); - }, - _ => break - } - } - t -} - impl<'a> Parser<'a> { pub fn new(sess: &'a ParseSess, cfg: ast::CrateConfig, mut rdr: Box) -> Parser<'a> { - let tok0 = real_token(&mut *rdr); + let tok0 = rdr.real_token(); let span = tok0.sp; let placeholder = TokenAndSpan { tok: token::Underscore, @@ -898,7 +884,7 @@ impl<'a> Parser<'a> { None }; let next = if self.buffer_start == self.buffer_end { - real_token(&mut *self.reader) + self.reader.real_token() } else { // Avoid token copies with `replace`. let buffer_start = self.buffer_start as uint; @@ -942,7 +928,7 @@ impl<'a> Parser<'a> { -> R { let dist = distance as int; while self.buffer_length() < dist { - self.buffer[self.buffer_end as uint] = real_token(&mut *self.reader); + self.buffer[self.buffer_end as uint] = self.reader.real_token(); self.buffer_end = (self.buffer_end + 1) & 3; } f(&self.buffer[((self.buffer_start + dist - 1) & 3) as uint].tok)