Skip to content

Move retokenize hack to save_analysis #76057

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 30, 2020
Merged

Move retokenize hack to save_analysis #76057

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3918,8 +3918,8 @@ dependencies = [
"rustc_data_structures",
"rustc_hir",
"rustc_hir_pretty",
"rustc_lexer",
"rustc_middle",
"rustc_parse",
"rustc_session",
"rustc_span",
"serde_json",
Expand Down
30 changes: 2 additions & 28 deletions src/librustc_parse/lexer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,19 +46,10 @@ impl<'a> StringReader<'a> {
source_file: Lrc<rustc_span::SourceFile>,
override_span: Option<Span>,
) -> Self {
// Make sure external source is loaded first, before accessing it.
// While this can't show up during normal parsing, `retokenize` may
// be called with a source file from an external crate.
sess.source_map().ensure_source_file_source_present(Lrc::clone(&source_file));

let src = if let Some(src) = &source_file.src {
Lrc::clone(&src)
} else if let Some(src) = source_file.external_src.borrow().get_source() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why was the external_src logic removed?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assumed from the comment that it was only needed for retokenize. But let me check blame to see if the two were actually added simultaneously.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yup, both ensure_source_file_present and this branch were added in #70172.

I think it should be fine to remove thouse now, as we no longer have retokenize. It might be the case that some other code-path now relies on this path, but this seems unlikely (StringReader is not really used outside of librustc_parse), and, in any case, lexing external files which might or might not be there feels just weird.

Lrc::clone(&src)
} else {
let src = source_file.src.clone().unwrap_or_else(|| {
sess.span_diagnostic
.bug(&format!("cannot lex `source_file` without source: {}", source_file.name));
};
});

StringReader {
sess,
Expand All @@ -70,23 +61,6 @@ impl<'a> StringReader<'a> {
}
}

pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
let begin = sess.source_map().lookup_byte_offset(span.lo());
let end = sess.source_map().lookup_byte_offset(span.hi());

// Make the range zero-length if the span is invalid.
if begin.sf.start_pos != end.sf.start_pos {
span = span.shrink_to_lo();
}

let mut sr = StringReader::new(sess, begin.sf, None);

// Seek the lexer to the right byte range.
sr.end_src_index = sr.src_index(span.hi());

sr
}

fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span {
self.override_span.unwrap_or_else(|| Span::with_root_ctxt(lo, hi))
}
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_save_analysis/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ rustc_ast_pretty = { path = "../librustc_ast_pretty" }
rustc_data_structures = { path = "../librustc_data_structures" }
rustc_hir = { path = "../librustc_hir" }
rustc_hir_pretty = { path = "../librustc_hir_pretty" }
rustc_parse = { path = "../librustc_parse" }
rustc_lexer = { path = "../librustc_lexer" }
serde_json = "1"
rustc_session = { path = "../librustc_session" }
rustc_span = { path = "../librustc_span" }
Expand Down
6 changes: 2 additions & 4 deletions src/librustc_save_analysis/dump_visitor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
//! recording the output.

use rustc_ast as ast;
use rustc_ast::{token, walk_list};
use rustc_ast::walk_list;
use rustc_data_structures::fx::FxHashSet;
use rustc_hir as hir;
use rustc_hir::def::{DefKind as HirDefKind, Res};
Expand Down Expand Up @@ -1207,9 +1207,7 @@ impl<'tcx> Visitor<'tcx> for DumpVisitor<'tcx> {

// Otherwise it's a span with wrong macro expansion info, which
// we don't want to track anyway, since it's probably macro-internal `use`
if let Some(sub_span) =
self.span.sub_span_of_token(item.span, token::BinOp(token::Star))
{
if let Some(sub_span) = self.span.sub_span_of_star(item.span) {
if !self.span.filter_generated(item.span) {
let access = access_from!(self.save_ctxt, item, item.hir_id);
let span = self.span_from_span(sub_span);
Expand Down
84 changes: 30 additions & 54 deletions src/librustc_save_analysis/span_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::generated_code;
use rustc_ast::token::{self, TokenKind};
use rustc_parse::lexer::{self, StringReader};
use rustc_data_structures::sync::Lrc;
use rustc_lexer::{tokenize, TokenKind};
use rustc_session::Session;
use rustc_span::*;

Expand Down Expand Up @@ -43,61 +43,37 @@ impl<'a> SpanUtils<'a> {
}
}

pub fn retokenise_span(&self, span: Span) -> StringReader<'a> {
lexer::StringReader::retokenize(&self.sess.parse_sess, span)
}

pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
let mut toks = self.retokenise_span(span);
loop {
let next = toks.next_token();
if next == token::Eof {
return None;
}
if next == tok {
return Some(next.span);
}
/// Finds the span of `*` token withing the larger `span`.
pub fn sub_span_of_star(&self, mut span: Span) -> Option<Span> {
let begin = self.sess.source_map().lookup_byte_offset(span.lo());
let end = self.sess.source_map().lookup_byte_offset(span.hi());
// Make the range zero-length if the span is invalid.
if begin.sf.start_pos != end.sf.start_pos {
span = span.shrink_to_lo();
}
}

// // Return the name for a macro definition (identifier after first `!`)
// pub fn span_for_macro_def_name(&self, span: Span) -> Option<Span> {
// let mut toks = self.retokenise_span(span);
// loop {
// let ts = toks.real_token();
// if ts == token::Eof {
// return None;
// }
// if ts == token::Not {
// let ts = toks.real_token();
// if ts.kind.is_ident() {
// return Some(ts.sp);
// } else {
// return None;
// }
// }
// }
// }
let sf = Lrc::clone(&begin.sf);

// // Return the name for a macro use (identifier before first `!`).
// pub fn span_for_macro_use_name(&self, span:Span) -> Option<Span> {
// let mut toks = self.retokenise_span(span);
// let mut prev = toks.real_token();
// loop {
// if prev == token::Eof {
// return None;
// }
// let ts = toks.real_token();
// if ts == token::Not {
// if prev.kind.is_ident() {
// return Some(prev.sp);
// } else {
// return None;
// }
// }
// prev = ts;
// }
// }
self.sess.source_map().ensure_source_file_source_present(Lrc::clone(&sf));
let src =
sf.src.clone().or_else(|| sf.external_src.borrow().get_source().map(Lrc::clone))?;
let to_index = |pos: BytePos| -> usize { (pos - sf.start_pos).0 as usize };
let text = &src[to_index(span.lo())..to_index(span.hi())];
let start_pos = {
let mut pos = 0;
tokenize(text)
.map(|token| {
let start = pos;
pos += token.len;
(start, token)
})
.find(|(_pos, token)| token.kind == TokenKind::Star)?
.0
};
let lo = span.lo() + BytePos(start_pos as u32);
let hi = lo + BytePos(1);
Some(span.with_lo(lo).with_hi(hi))
}

/// Return true if the span is generated code, and
/// it is not a subspan of the root callsite.
Expand Down