Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 0f071b2

Browse files
borsbors
committed
Auto merge of rust-lang#118671 - petrochenkov:span3, r=<try>
[PERF-EXPERIMENT] Add second span to TokenTree cc rust-lang#118517
2 parents 15bb3e2 + cca1dd4 commit 0f071b2

File tree

20 files changed

+180
-119
lines changed

20 files changed

+180
-119
lines changed

‎compiler/rustc_ast/src/attr/mod.rs‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -310,9 +310,10 @@ impl MetaItem {
310310
Some(&TokenTree::Token(
311311
Token { kind: ref kind @ (token::Ident(..) | token::ModSep), span },
312312
_,
313+
_,
313314
)) => 'arm: {
314315
let mut segments = if let &token::Ident(name, _) = kind {
315-
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
316+
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _, _)) =
316317
tokens.peek()
317318
{
318319
tokens.next();
@@ -324,14 +325,17 @@ impl MetaItem {
324325
thin_vec![PathSegment::path_root(span)]
325326
};
326327
loop {
327-
if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
328-
tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
328+
if let Some(&TokenTree::Token(
329+
Token { kind: token::Ident(name, _), span },
330+
_,
331+
_,
332+
)) = tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
329333
{
330334
segments.push(PathSegment::from_ident(Ident::new(name, span)));
331335
} else {
332336
return None;
333337
}
334-
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
338+
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _, _)) =
335339
tokens.peek()
336340
{
337341
tokens.next();
@@ -342,11 +346,13 @@ impl MetaItem {
342346
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
343347
Path { span, segments, tokens: None }
344348
}
345-
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &nt.0 {
346-
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
347-
token::Nonterminal::NtPath(path) => (**path).clone(),
348-
_ => return None,
349-
},
349+
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _, _)) => {
350+
match &nt.0 {
351+
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
352+
token::Nonterminal::NtPath(path) => (**path).clone(),
353+
_ => return None,
354+
}
355+
}
350356
_ => return None,
351357
};
352358
let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());
@@ -376,7 +382,7 @@ impl MetaItemKind {
376382
let item = NestedMetaItem::from_tokens(&mut tokens)?;
377383
result.push(item);
378384
match tokens.next() {
379-
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
385+
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _, _)) => {}
380386
_ => return None,
381387
}
382388
}
@@ -387,10 +393,10 @@ impl MetaItemKind {
387393
tokens: &mut impl Iterator<Item = &'a TokenTree>,
388394
) -> Option<MetaItemKind> {
389395
match tokens.next() {
390-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
396+
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens, _)) => {
391397
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
392398
}
393-
Some(TokenTree::Token(token, _)) => {
399+
Some(TokenTree::Token(token, _, _)) => {
394400
MetaItemLit::from_token(token).map(MetaItemKind::NameValue)
395401
}
396402
_ => None,
@@ -401,13 +407,13 @@ impl MetaItemKind {
401407
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
402408
) -> Option<MetaItemKind> {
403409
match tokens.peek() {
404-
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
410+
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens, _)) => {
405411
let inner_tokens = inner_tokens.clone();
406412
tokens.next();
407413
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
408414
}
409415
Some(TokenTree::Delimited(..)) => None,
410-
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
416+
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _, _)) => {
411417
tokens.next();
412418
MetaItemKind::name_value_from_tokens(tokens)
413419
}
@@ -520,11 +526,13 @@ impl NestedMetaItem {
520526
I: Iterator<Item = &'a TokenTree>,
521527
{
522528
match tokens.peek() {
523-
Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
529+
Some(TokenTree::Token(token, _, _))
530+
if let Some(lit) = MetaItemLit::from_token(token) =>
531+
{
524532
tokens.next();
525533
return Some(NestedMetaItem::Lit(lit));
526534
}
527-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
535+
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens, _)) => {
528536
tokens.next();
529537
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
530538
}
@@ -585,6 +593,7 @@ pub fn mk_attr_nested_word(
585593
let inner_tokens = TokenStream::new(vec![TokenTree::Token(
586594
Token::from_ast_ident(Ident::new(inner, span)),
587595
Spacing::Alone,
596+
span,
588597
)]);
589598
let outer_ident = Ident::new(outer, span);
590599
let path = Path::from_ident(outer_ident);

‎compiler/rustc_ast/src/mut_visit.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -706,10 +706,10 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
706706
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
707707
pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
708708
match tt {
709-
TokenTree::Token(token, _) => {
709+
TokenTree::Token(token, _, _) => {
710710
visit_token(token, vis);
711711
}
712-
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
712+
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts, _) => {
713713
vis.visit_span(open);
714714
vis.visit_span(close);
715715
visit_tts(tts, vis);

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 54 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ use std::{cmp, fmt, iter, mem};
4444
pub enum TokenTree {
4545
/// A single token. Should never be `OpenDelim` or `CloseDelim`, because
4646
/// delimiters are implicitly represented by `Delimited`.
47-
Token(Token, Spacing),
47+
Token(Token, Spacing, Span),
4848
/// A delimited sequence of token trees.
49-
Delimited(DelimSpan, Delimiter, TokenStream),
49+
Delimited(DelimSpan, Delimiter, TokenStream, Span),
5050
}
5151

5252
// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
@@ -65,8 +65,10 @@ impl TokenTree {
6565
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
6666
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
6767
match (self, other) {
68-
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
69-
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
68+
(TokenTree::Token(token, _, _), TokenTree::Token(token2, _, _)) => {
69+
token.kind == token2.kind
70+
}
71+
(TokenTree::Delimited(_, delim, tts, _), TokenTree::Delimited(_, delim2, tts2, _)) => {
7072
delim == delim2 && tts.eq_unspanned(tts2)
7173
}
7274
_ => false,
@@ -76,33 +78,33 @@ impl TokenTree {
7678
/// Retrieves the `TokenTree`'s span.
7779
pub fn span(&self) -> Span {
7880
match self {
79-
TokenTree::Token(token, _) => token.span,
81+
TokenTree::Token(token, _, _) => token.span,
8082
TokenTree::Delimited(sp, ..) => sp.entire(),
8183
}
8284
}
8385

8486
/// Modify the `TokenTree`'s span in-place.
8587
pub fn set_span(&mut self, span: Span) {
8688
match self {
87-
TokenTree::Token(token, _) => token.span = span,
89+
TokenTree::Token(token, _, _) => token.span = span,
8890
TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span),
8991
}
9092
}
9193

9294
/// Create a `TokenTree::Token` with alone spacing.
9395
pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
94-
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
96+
TokenTree::Token(Token::new(kind, span), Spacing::Alone, span)
9597
}
9698

9799
/// Create a `TokenTree::Token` with joint spacing.
98100
pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
99-
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
101+
TokenTree::Token(Token::new(kind, span), Spacing::Joint, span)
100102
}
101103

102104
pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
103105
match self {
104-
TokenTree::Token(token, spacing) => match token.uninterpolate() {
105-
Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
106+
TokenTree::Token(token, spacing, span) => match token.uninterpolate() {
107+
Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing, *span)),
106108
Cow::Borrowed(_) => Cow::Borrowed(self),
107109
},
108110
_ => Cow::Borrowed(self),
@@ -206,12 +208,15 @@ impl AttrTokenStream {
206208
.iter()
207209
.flat_map(|tree| match &tree {
208210
AttrTokenTree::Token(inner, spacing) => {
209-
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
210-
}
211-
AttrTokenTree::Delimited(span, delim, stream) => {
212-
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
213-
.into_iter()
211+
smallvec![TokenTree::Token(inner.clone(), *spacing, inner.span)].into_iter()
214212
}
213+
AttrTokenTree::Delimited(span, delim, stream) => smallvec![TokenTree::Delimited(
214+
*span,
215+
*delim,
216+
stream.to_tokenstream(),
217+
span.entire()
218+
),]
219+
.into_iter(),
215220
AttrTokenTree::Attributes(data) => {
216221
let idx = data
217222
.attrs
@@ -230,7 +235,7 @@ impl AttrTokenStream {
230235
let mut found = false;
231236
// Check the last two trees (to account for a trailing semi)
232237
for tree in target_tokens.iter_mut().rev().take(2) {
233-
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
238+
if let TokenTree::Delimited(span, delim, delim_tokens, _) = tree {
234239
// Inner attributes are only supported on extern blocks, functions,
235240
// impls, and modules. All of these have their inner attributes
236241
// placed at the beginning of the rightmost outermost braced group:
@@ -250,7 +255,7 @@ impl AttrTokenStream {
250255
stream.push_stream(inner_attr.tokens());
251256
}
252257
stream.push_stream(delim_tokens.clone());
253-
*tree = TokenTree::Delimited(*span, *delim, stream);
258+
*tree = TokenTree::Delimited(*span, *delim, stream, span.entire());
254259
found = true;
255260
break;
256261
}
@@ -330,10 +335,10 @@ impl TokenStream {
330335
while let Some((pos, ts)) = iter.next() {
331336
if let Some((_, next)) = iter.peek() {
332337
let sp = match (&ts, &next) {
333-
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
338+
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _, _)) => continue,
334339
(
335-
TokenTree::Token(token_left, Spacing::Alone),
336-
TokenTree::Token(token_right, _),
340+
TokenTree::Token(token_left, Spacing::Alone, _),
341+
TokenTree::Token(token_right, _, _),
337342
) if ((token_left.is_ident() && !token_left.is_reserved_ident())
338343
|| token_left.is_lit())
339344
&& ((token_right.is_ident() && !token_right.is_reserved_ident())
@@ -433,7 +438,7 @@ impl TokenStream {
433438

434439
/// Create a token stream containing a single `Delimited`.
435440
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
436-
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
441+
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts, span.entire())])
437442
}
438443

439444
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
@@ -475,25 +480,32 @@ impl TokenStream {
475480
}
476481
}
477482

478-
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
483+
fn flatten_token(token: &Token, spacing: Spacing, span: Span) -> TokenTree {
479484
match &token.kind {
480485
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => {
481-
TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
486+
TokenTree::Token(
487+
Token::new(token::Ident(ident.name, is_raw), ident.span),
488+
spacing,
489+
token.span,
490+
)
482491
}
483492
token::Interpolated(nt) => TokenTree::Delimited(
484493
DelimSpan::from_single(token.span),
485494
Delimiter::Invisible,
486495
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
496+
token.span,
487497
),
488-
_ => TokenTree::Token(token.clone(), spacing),
498+
_ => TokenTree::Token(token.clone(), spacing, span),
489499
}
490500
}
491501

492502
fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
493503
match tree {
494-
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
495-
TokenTree::Delimited(span, delim, tts) => {
496-
TokenTree::Delimited(*span, *delim, tts.flattened())
504+
TokenTree::Token(token, spacing, span) => {
505+
TokenStream::flatten_token(token, *spacing, *span)
506+
}
507+
TokenTree::Delimited(span, delim, tts, span2) => {
508+
TokenTree::Delimited(*span, *delim, tts.flattened(), *span2)
497509
}
498510
}
499511
}
@@ -502,8 +514,8 @@ impl TokenStream {
502514
pub fn flattened(&self) -> TokenStream {
503515
fn can_skip(stream: &TokenStream) -> bool {
504516
stream.trees().all(|tree| match tree {
505-
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
506-
TokenTree::Delimited(_, _, inner) => can_skip(inner),
517+
TokenTree::Token(token, _, _) => !matches!(token.kind, token::Interpolated(_)),
518+
TokenTree::Delimited(_, _, inner, _) => can_skip(inner),
507519
})
508520
}
509521

@@ -517,13 +529,13 @@ impl TokenStream {
517529
// If `vec` is not empty, try to glue `tt` onto its last token. The return
518530
// value indicates if gluing took place.
519531
fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
520-
if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
521-
&& let TokenTree::Token(tok, spacing) = tt
532+
if let Some(TokenTree::Token(last_tok, Spacing::Joint, _)) = vec.last()
533+
&& let TokenTree::Token(tok, spacing, span) = tt
522534
&& let Some(glued_tok) = last_tok.glue(tok)
523535
{
524536
// ...then overwrite the last token tree in `vec` with the
525537
// glued token, and skip the first token tree from `stream`.
526-
*vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
538+
*vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing, *span);
527539
true
528540
} else {
529541
false
@@ -582,6 +594,7 @@ impl TokenStream {
582594
&TokenTree::Token(
583595
Token { kind: token::DocComment(_, attr_style, data), span },
584596
_spacing,
597+
_,
585598
) => {
586599
let desugared = desugared_tts(attr_style, data, span);
587600
let desugared_len = desugared.len();
@@ -592,9 +605,14 @@ impl TokenStream {
592605

593606
&TokenTree::Token(..) => i += 1,
594607

595-
&TokenTree::Delimited(sp, delim, ref delim_stream) => {
608+
&TokenTree::Delimited(sp, delim, ref delim_stream, _) => {
596609
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
597-
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
610+
let new_tt = TokenTree::Delimited(
611+
sp,
612+
delim,
613+
desugared_delim_stream,
614+
sp.entire(),
615+
);
598616
Lrc::make_mut(&mut stream.0)[i] = new_tt;
599617
modified = true;
600618
}
@@ -637,6 +655,7 @@ impl TokenStream {
637655
]
638656
.into_iter()
639657
.collect::<TokenStream>(),
658+
delim_span.entire(),
640659
);
641660

642661
if attr_style == AttrStyle::Inner {
@@ -748,6 +767,6 @@ mod size_asserts {
748767
static_assert_size!(AttrTokenTree, 32);
749768
static_assert_size!(LazyAttrTokenStream, 8);
750769
static_assert_size!(TokenStream, 8);
751-
static_assert_size!(TokenTree, 32);
770+
static_assert_size!(TokenTree, 40);
752771
// tidy-alphabetical-end
753772
}

0 commit comments

Comments
 (0)