Skip to content

Commit 7a4c186

Browse files
Aaron1011Aaron1011
committed
Move functions to librustc_parse
1 parent a9ca1ec commit 7a4c186

File tree

3 files changed

+123
-179
lines changed

3 files changed

+123
-179
lines changed

‎src/librustc_ast/token.rs

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -673,62 +673,6 @@ impl Token {
673673

674674
Some(Token::new(kind, self.span.to(joint.span)))
675675
}
676-
677-
// See comments in `Nonterminal::to_tokenstream` for why we care about
678-
// *probably* equal here rather than actual equality
679-
crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
680-
if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) {
681-
return false;
682-
}
683-
match (&self.kind, &other.kind) {
684-
(&Eq, &Eq)
685-
| (&Lt, &Lt)
686-
| (&Le, &Le)
687-
| (&EqEq, &EqEq)
688-
| (&Ne, &Ne)
689-
| (&Ge, &Ge)
690-
| (&Gt, &Gt)
691-
| (&AndAnd, &AndAnd)
692-
| (&OrOr, &OrOr)
693-
| (&Not, &Not)
694-
| (&Tilde, &Tilde)
695-
| (&At, &At)
696-
| (&Dot, &Dot)
697-
| (&DotDot, &DotDot)
698-
| (&DotDotDot, &DotDotDot)
699-
| (&DotDotEq, &DotDotEq)
700-
| (&Comma, &Comma)
701-
| (&Semi, &Semi)
702-
| (&Colon, &Colon)
703-
| (&ModSep, &ModSep)
704-
| (&RArrow, &RArrow)
705-
| (&LArrow, &LArrow)
706-
| (&FatArrow, &FatArrow)
707-
| (&Pound, &Pound)
708-
| (&Dollar, &Dollar)
709-
| (&Question, &Question)
710-
| (&Whitespace, &Whitespace)
711-
| (&Comment, &Comment)
712-
| (&Eof, &Eof) => true,
713-
714-
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
715-
716-
(&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
717-
718-
(&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
719-
720-
(&Literal(a), &Literal(b)) => a == b,
721-
722-
(&Lifetime(a), &Lifetime(b)) => a == b,
723-
(&Ident(a, b), &Ident(c, d)) => {
724-
b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
725-
}
726-
727-
(&Interpolated(_), &Interpolated(_)) => false,
728-
729-
_ => panic!("forgot to add a token?"),
730-
}
731-
}
732676
}
733677

734678
impl PartialEq<TokenKind> for Token {

‎src/librustc_ast/tokenstream.rs

Lines changed: 0 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -68,23 +68,6 @@ impl TokenTree {
6868
}
6969
}
7070

71-
// See comments in `Nonterminal::to_tokenstream` for why we care about
72-
// *probably* equal here rather than actual equality
73-
//
74-
// This is otherwise the same as `eq_unspanned`, only recursing with a
75-
// different method.
76-
pub fn probably_equal_for_proc_macro(&self, other: &TokenTree) -> bool {
77-
match (self, other) {
78-
(TokenTree::Token(token), TokenTree::Token(token2)) => {
79-
token.probably_equal_for_proc_macro(token2)
80-
}
81-
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
82-
delim == delim2 && tts.probably_equal_for_proc_macro(&tts2)
83-
}
84-
_ => false,
85-
}
86-
}
87-
8871
/// Retrieves the TokenTree's span.
8972
pub fn span(&self) -> Span {
9073
match self {
@@ -307,111 +290,7 @@ impl TokenStream {
307290
t1.next().is_none() && t2.next().is_none()
308291
}
309292

310-
// See comments in `Nonterminal::to_tokenstream` for why we care about
311-
// *probably* equal here rather than actual equality
312-
//
313-
// This is otherwise the same as `eq_unspanned`, only recursing with a
314-
// different method.
315-
pub fn probably_equal_for_proc_macro(&self, other: &TokenStream) -> bool {
316-
// When checking for `probably_eq`, we ignore certain tokens that aren't
317-
// preserved in the AST. Because they are not preserved, the pretty
318-
// printer arbitrarily adds or removes them when printing as token
319-
// streams, making a comparison between a token stream generated from an
320-
// AST and a token stream which was parsed into an AST more reliable.
321-
fn semantic_tree(tree: &TokenTree) -> bool {
322-
if let TokenTree::Token(token) = tree {
323-
if let
324-
// The pretty printer tends to add trailing commas to
325-
// everything, and in particular, after struct fields.
326-
| token::Comma
327-
// The pretty printer emits `NoDelim` as whitespace.
328-
| token::OpenDelim(DelimToken::NoDelim)
329-
| token::CloseDelim(DelimToken::NoDelim)
330-
// The pretty printer collapses many semicolons into one.
331-
| token::Semi
332-
// The pretty printer collapses whitespace arbitrarily and can
333-
// introduce whitespace from `NoDelim`.
334-
| token::Whitespace
335-
// The pretty printer can turn `$crate` into `::crate_name`
336-
| token::ModSep = token.kind {
337-
return false;
338-
}
339-
}
340-
true
341-
}
342293

343-
// When comparing two `TokenStream`s, we ignore the `IsJoint` information.
344-
//
345-
// However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will
346-
// use `Token.glue` on adjacent tokens with the proper `IsJoint`.
347-
// Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`)
348-
// and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent
349-
// when determining if two `TokenStream`s are 'probably equal'.
350-
//
351-
// Therefore, we use `break_two_token_op` to convert all tokens
352-
// to the 'unglued' form (if it exists). This ensures that two
353-
// `TokenStream`s which differ only in how their tokens are glued
354-
// will be considered 'probably equal', which allows us to keep spans.
355-
//
356-
// This is important when the original `TokenStream` contained
357-
// extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces
358-
// will be omitted when we pretty-print, which can cause the original
359-
// and reparsed `TokenStream`s to differ in the assignment of `IsJoint`,
360-
// leading to some tokens being 'glued' together in one stream but not
361-
// the other. See #68489 for more details.
362-
fn break_tokens(tree: TokenTree) -> impl Iterator<Item = TokenTree> {
363-
// In almost all cases, we should have either zero or one levels
364-
// of 'unglueing'. However, in some unusual cases, we may need
365-
// to iterate breaking tokens mutliple times. For example:
366-
// '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]'
367-
let mut token_trees: SmallVec<[_; 2]>;
368-
if let TokenTree::Token(token) = &tree {
369-
let mut out = SmallVec::<[_; 2]>::new();
370-
out.push(token.clone());
371-
// Iterate to fixpoint:
372-
// * We start off with 'out' containing our initial token, and `temp` empty
373-
// * If we are able to break any tokens in `out`, then `out` will have
374-
// at least one more element than 'temp', so we will try to break tokens
375-
// again.
376-
// * If we cannot break any tokens in 'out', we are done
377-
loop {
378-
let mut temp = SmallVec::<[_; 2]>::new();
379-
let mut changed = false;
380-
381-
for token in out.into_iter() {
382-
if let Some((first, second)) = token.kind.break_two_token_op() {
383-
temp.push(Token::new(first, DUMMY_SP));
384-
temp.push(Token::new(second, DUMMY_SP));
385-
changed = true;
386-
} else {
387-
temp.push(token);
388-
}
389-
}
390-
out = temp;
391-
if !changed {
392-
break;
393-
}
394-
}
395-
token_trees = out.into_iter().map(|t| TokenTree::Token(t)).collect();
396-
if token_trees.len() != 1 {
397-
debug!("break_tokens: broke {:?} to {:?}", tree, token_trees);
398-
}
399-
} else {
400-
token_trees = SmallVec::new();
401-
token_trees.push(tree);
402-
}
403-
token_trees.into_iter()
404-
}
405-
406-
let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens);
407-
let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens);
408-
for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
409-
if !t1.probably_equal_for_proc_macro(&t2) {
410-
return false;
411-
}
412-
}
413-
t1.next().is_none() && t2.next().is_none()
414-
}
415294

416295
pub fn map_enumerated<F: FnMut(usize, TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
417296
TokenStream(Lrc::new(

‎src/librustc_parse/lib.rs

Lines changed: 123 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,18 @@
77
#![feature(or_patterns)]
88

99
use rustc_ast::ast;
10-
use rustc_ast::token::{self, Nonterminal};
10+
use rustc_ast::token::{self, Nonterminal, Token, TokenKind, DelimToken};
1111
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
1212
use rustc_ast_pretty::pprust;
1313
use rustc_data_structures::sync::Lrc;
1414
use rustc_errors::{Diagnostic, FatalError, Level, PResult};
1515
use rustc_session::parse::ParseSess;
1616
use rustc_span::{FileName, SourceFile, Span};
17+
use rustc_span::symbol::kw;
1718

1819
use std::path::Path;
1920
use std::str;
21+
use std::mem;
2022

2123
use log::info;
2224

@@ -300,7 +302,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
300302
// modifications, including adding/removing typically non-semantic
301303
// tokens such as extra braces and commas, don't happen.
302304
if let Some(tokens) = tokens {
303-
if tokens.probably_equal_for_proc_macro(&tokens_for_real) {
305+
if tokenstream_probably_equal_for_proc_macro(&tokens, &tokens_for_real) {
304306
return tokens;
305307
}
306308
info!(
@@ -373,3 +375,122 @@ fn prepend_attrs(
373375
builder.push(tokens.clone());
374376
Some(builder.build())
375377
}
378+
379+
// See comments in `Nonterminal::to_tokenstream` for why we care about
380+
// *probably* equal here rather than actual equality
381+
//
382+
// This is otherwise the same as `eq_unspanned`, only recursing with a
383+
// different method.
384+
pub fn tokenstream_probably_equal_for_proc_macro(first: &TokenStream, other: &TokenStream) -> bool {
385+
// When checking for `probably_eq`, we ignore certain tokens that aren't
386+
// preserved in the AST. Because they are not preserved, the pretty
387+
// printer arbitrarily adds or removes them when printing as token
388+
// streams, making a comparison between a token stream generated from an
389+
// AST and a token stream which was parsed into an AST more reliable.
390+
fn semantic_tree(tree: &TokenTree) -> bool {
391+
if let TokenTree::Token(token) = tree {
392+
if let
393+
// The pretty printer tends to add trailing commas to
394+
// everything, and in particular, after struct fields.
395+
| token::Comma
396+
// The pretty printer emits `NoDelim` as whitespace.
397+
| token::OpenDelim(DelimToken::NoDelim)
398+
| token::CloseDelim(DelimToken::NoDelim)
399+
// The pretty printer collapses many semicolons into one.
400+
| token::Semi
401+
// The pretty printer collapses whitespace arbitrarily and can
402+
// introduce whitespace from `NoDelim`.
403+
| token::Whitespace
404+
// The pretty printer can turn `$crate` into `::crate_name`
405+
| token::ModSep = token.kind {
406+
return false;
407+
}
408+
}
409+
true
410+
}
411+
412+
let mut t1 = first.trees().filter(semantic_tree);
413+
let mut t2 = other.trees().filter(semantic_tree);
414+
for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
415+
if !tokentree_probably_equal_for_proc_macro(&t1, &t2) {
416+
return false;
417+
}
418+
}
419+
t1.next().is_none() && t2.next().is_none()
420+
}
421+
422+
// See comments in `Nonterminal::to_tokenstream` for why we care about
423+
// *probably* equal here rather than actual equality
424+
crate fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
425+
use TokenKind::*;
426+
427+
if mem::discriminant(&first.kind) != mem::discriminant(&other.kind) {
428+
return false;
429+
}
430+
match (&first.kind, &other.kind) {
431+
(&Eq, &Eq)
432+
| (&Lt, &Lt)
433+
| (&Le, &Le)
434+
| (&EqEq, &EqEq)
435+
| (&Ne, &Ne)
436+
| (&Ge, &Ge)
437+
| (&Gt, &Gt)
438+
| (&AndAnd, &AndAnd)
439+
| (&OrOr, &OrOr)
440+
| (&Not, &Not)
441+
| (&Tilde, &Tilde)
442+
| (&At, &At)
443+
| (&Dot, &Dot)
444+
| (&DotDot, &DotDot)
445+
| (&DotDotDot, &DotDotDot)
446+
| (&DotDotEq, &DotDotEq)
447+
| (&Comma, &Comma)
448+
| (&Semi, &Semi)
449+
| (&Colon, &Colon)
450+
| (&ModSep, &ModSep)
451+
| (&RArrow, &RArrow)
452+
| (&LArrow, &LArrow)
453+
| (&FatArrow, &FatArrow)
454+
| (&Pound, &Pound)
455+
| (&Dollar, &Dollar)
456+
| (&Question, &Question)
457+
| (&Whitespace, &Whitespace)
458+
| (&Comment, &Comment)
459+
| (&Eof, &Eof) => true,
460+
461+
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
462+
463+
(&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
464+
465+
(&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
466+
467+
(&Literal(a), &Literal(b)) => a == b,
468+
469+
(&Lifetime(a), &Lifetime(b)) => a == b,
470+
(&Ident(a, b), &Ident(c, d)) => {
471+
b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
472+
}
473+
474+
(&Interpolated(_), &Interpolated(_)) => false,
475+
476+
_ => panic!("forgot to add a token?"),
477+
}
478+
}
479+
480+
481+
// See comments in `Nonterminal::to_tokenstream` for why we care about
482+
// *probably* equal here rather than actual equality
483+
//
484+
// This is otherwise the same as `eq_unspanned`, only recursing with a
485+
// different method.
486+
pub fn tokentree_probably_equal_for_proc_macro(first: &TokenTree, other: &TokenTree) -> bool {
487+
match (first, other) {
488+
(TokenTree::Token(token), TokenTree::Token(token2)) => {
489+
token_probably_equal_for_proc_macro(token, token2)
490+
}
491+
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
492+
delim == delim2 && tokenstream_probably_equal_for_proc_macro(&tts, &tts2)
493+
}
494+
_ => false,
495+
}
496+
}

0 commit comments

Comments
 (0)