@@ -19,9 +19,8 @@ pub use path::PathStyle;
19
19
20
20
use rustc_ast:: ptr:: P ;
21
21
use rustc_ast:: token:: { self , Delimiter , Nonterminal , Token , TokenKind } ;
22
- use rustc_ast:: tokenstream:: AttributesData ;
23
- use rustc_ast:: tokenstream:: { self , DelimSpan , Spacing } ;
24
- use rustc_ast:: tokenstream:: { TokenStream , TokenTree } ;
22
+ use rustc_ast:: tokenstream:: { AttributesData , DelimSpan , Spacing } ;
23
+ use rustc_ast:: tokenstream:: { TokenStream , TokenTree , TokenTreeCursor } ;
25
24
use rustc_ast:: util:: case:: Case ;
26
25
use rustc_ast:: AttrId ;
27
26
use rustc_ast:: DUMMY_NODE_ID ;
@@ -168,7 +167,7 @@ pub struct Parser<'a> {
168
167
// This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
169
168
// it doesn't unintentionally get bigger.
170
169
#[ cfg( all( target_arch = "x86_64" , target_pointer_width = "64" ) ) ]
171
- rustc_data_structures:: static_assert_size!( Parser <' _>, 336 ) ;
170
+ rustc_data_structures:: static_assert_size!( Parser <' _>, 312 ) ;
172
171
173
172
/// Stores span information about a closure.
174
173
#[ derive( Clone ) ]
@@ -221,18 +220,27 @@ impl<'a> Drop for Parser<'a> {
221
220
}
222
221
}
223
222
223
+ /// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that
224
+ /// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b)
225
+ /// use this type to emit them as a linear sequence. But a linear sequence is
226
+ /// what the parser expects, for the most part.
224
227
#[ derive( Clone ) ]
225
228
struct TokenCursor {
226
- // The current (innermost) frame. `frame` and `stack` could be combined,
227
- // but it's faster to have them separately to access `frame` directly
228
- // rather than via something like `stack.last().unwrap()` or
229
- // `stack[stack.len() - 1]`.
230
- frame : TokenCursorFrame ,
231
- // Additional frames that enclose `frame`.
232
- stack : Vec < TokenCursorFrame > ,
229
+ // Cursor for the current (innermost) token stream. The delimiters for this
230
+ // token stream are found in `self.stack.last()`; when that is `None` then
231
+ // we are in the outermost token stream which never has delimiters.
232
+ tree_cursor : TokenTreeCursor ,
233
+
234
+ // Token streams surrounding the current one. The delimiters for stack[n]'s
235
+ // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
236
+ // because it's the outermost token stream which never has delimiters.
237
+ stack : Vec < ( TokenTreeCursor , Delimiter , DelimSpan ) > ,
238
+
233
239
desugar_doc_comments : bool ,
240
+
234
241
// Counts the number of calls to `{,inlined_}next`.
235
242
num_next_calls : usize ,
243
+
236
244
// During parsing, we may sometimes need to 'unglue' a
237
245
// glued token into two component tokens
238
246
// (e.g. '>>' into '>' and '>), so that the parser
@@ -257,18 +265,6 @@ struct TokenCursor {
257
265
break_last_token : bool ,
258
266
}
259
267
260
- #[ derive( Clone ) ]
261
- struct TokenCursorFrame {
262
- delim_sp : Option < ( Delimiter , DelimSpan ) > ,
263
- tree_cursor : tokenstream:: Cursor ,
264
- }
265
-
266
- impl TokenCursorFrame {
267
- fn new ( delim_sp : Option < ( Delimiter , DelimSpan ) > , tts : TokenStream ) -> Self {
268
- TokenCursorFrame { delim_sp, tree_cursor : tts. into_trees ( ) }
269
- }
270
- }
271
-
272
268
impl TokenCursor {
273
269
fn next ( & mut self , desugar_doc_comments : bool ) -> ( Token , Spacing ) {
274
270
self . inlined_next ( desugar_doc_comments)
@@ -281,38 +277,47 @@ impl TokenCursor {
281
277
// FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will
282
278
// need to, whereupon the `delim != Delimiter::Invisible` conditions below can be
283
279
// removed.
284
- if let Some ( tree) = self . frame . tree_cursor . next_ref ( ) {
280
+ if let Some ( tree) = self . tree_cursor . next_ref ( ) {
285
281
match tree {
286
282
& TokenTree :: Token ( ref token, spacing) => match ( desugar_doc_comments, token) {
287
283
( true , & Token { kind : token:: DocComment ( _, attr_style, data) , span } ) => {
288
- return self . desugar ( attr_style, data, span) ;
284
+ let desugared = self . desugar ( attr_style, data, span) ;
285
+ self . tree_cursor . replace_prev_and_rewind ( desugared) ;
286
+ // Continue to get the first token of the desugared doc comment.
287
+ }
288
+ _ => {
289
+ debug_assert ! ( !matches!(
290
+ token. kind,
291
+ token:: OpenDelim ( _) | token:: CloseDelim ( _)
292
+ ) ) ;
293
+ return ( token. clone ( ) , spacing) ;
289
294
}
290
- _ => return ( token. clone ( ) , spacing) ,
291
295
} ,
292
296
& TokenTree :: Delimited ( sp, delim, ref tts) => {
293
- // Set `open_delim` to true here because we deal with it immediately.
294
- let frame = TokenCursorFrame :: new ( Some ( ( delim, sp) ) , tts. clone ( ) ) ;
295
- self . stack . push ( mem:: replace ( & mut self . frame , frame) ) ;
297
+ let trees = tts. clone ( ) . into_trees ( ) ;
298
+ self . stack . push ( ( mem:: replace ( & mut self . tree_cursor , trees) , delim, sp) ) ;
296
299
if delim != Delimiter :: Invisible {
297
300
return ( Token :: new ( token:: OpenDelim ( delim) , sp. open ) , Spacing :: Alone ) ;
298
301
}
299
302
// No open delimiter to return; continue on to the next iteration.
300
303
}
301
304
} ;
302
- } else if let Some ( frame) = self . stack . pop ( ) {
303
- if let Some ( ( delim, span) ) = self . frame . delim_sp && delim != Delimiter :: Invisible {
304
- self . frame = frame;
305
+ } else if let Some ( ( tree_cursor, delim, span) ) = self . stack . pop ( ) {
306
+ // We have exhausted this token stream. Move back to its parent token stream.
307
+ self . tree_cursor = tree_cursor;
308
+ if delim != Delimiter :: Invisible {
305
309
return ( Token :: new ( token:: CloseDelim ( delim) , span. close ) , Spacing :: Alone ) ;
306
310
}
307
- self . frame = frame;
308
311
// No close delimiter to return; continue on to the next iteration.
309
312
} else {
313
+ // We have exhausted the outermost token stream.
310
314
return ( Token :: new ( token:: Eof , DUMMY_SP ) , Spacing :: Alone ) ;
311
315
}
312
316
}
313
317
}
314
318
315
- fn desugar ( & mut self , attr_style : AttrStyle , data : Symbol , span : Span ) -> ( Token , Spacing ) {
319
+ // Desugar a doc comment into something like `#[doc = r"foo"]`.
320
+ fn desugar ( & mut self , attr_style : AttrStyle , data : Symbol , span : Span ) -> Vec < TokenTree > {
316
321
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
317
322
// required to wrap the text. E.g.
318
323
// - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
@@ -346,27 +351,15 @@ impl TokenCursor {
346
351
. collect :: < TokenStream > ( ) ,
347
352
) ;
348
353
349
- self . stack . push ( mem:: replace (
350
- & mut self . frame ,
351
- TokenCursorFrame :: new (
352
- None ,
353
- if attr_style == AttrStyle :: Inner {
354
- [
355
- TokenTree :: token_alone ( token:: Pound , span) ,
356
- TokenTree :: token_alone ( token:: Not , span) ,
357
- body,
358
- ]
359
- . into_iter ( )
360
- . collect :: < TokenStream > ( )
361
- } else {
362
- [ TokenTree :: token_alone ( token:: Pound , span) , body]
363
- . into_iter ( )
364
- . collect :: < TokenStream > ( )
365
- } ,
366
- ) ,
367
- ) ) ;
368
-
369
- self . next ( /* desugar_doc_comments */ false )
354
+ if attr_style == AttrStyle :: Inner {
355
+ vec ! [
356
+ TokenTree :: token_alone( token:: Pound , span) ,
357
+ TokenTree :: token_alone( token:: Not , span) ,
358
+ body,
359
+ ]
360
+ } else {
361
+ vec ! [ TokenTree :: token_alone( token:: Pound , span) , body]
362
+ }
370
363
}
371
364
}
372
365
@@ -475,7 +468,7 @@ impl<'a> Parser<'a> {
475
468
restrictions : Restrictions :: empty ( ) ,
476
469
expected_tokens : Vec :: new ( ) ,
477
470
token_cursor : TokenCursor {
478
- frame : TokenCursorFrame :: new ( None , tokens) ,
471
+ tree_cursor : tokens. into_trees ( ) ,
479
472
stack : Vec :: new ( ) ,
480
473
num_next_calls : 0 ,
481
474
desugar_doc_comments,
@@ -1142,14 +1135,16 @@ impl<'a> Parser<'a> {
1142
1135
return looker ( & self . token ) ;
1143
1136
}
1144
1137
1145
- let frame = & self . token_cursor . frame ;
1146
- if let Some ( ( delim, span) ) = frame. delim_sp && delim != Delimiter :: Invisible {
1138
+ let tree_cursor = & self . token_cursor . tree_cursor ;
1139
+ if let Some ( & ( _, delim, span) ) = self . token_cursor . stack . last ( )
1140
+ && delim != Delimiter :: Invisible
1141
+ {
1147
1142
let all_normal = ( 0 ..dist) . all ( |i| {
1148
- let token = frame . tree_cursor . look_ahead ( i) ;
1143
+ let token = tree_cursor. look_ahead ( i) ;
1149
1144
!matches ! ( token, Some ( TokenTree :: Delimited ( _, Delimiter :: Invisible , _) ) )
1150
1145
} ) ;
1151
1146
if all_normal {
1152
- return match frame . tree_cursor . look_ahead ( dist - 1 ) {
1147
+ return match tree_cursor. look_ahead ( dist - 1 ) {
1153
1148
Some ( tree) => match tree {
1154
1149
TokenTree :: Token ( token, _) => looker ( token) ,
1155
1150
TokenTree :: Delimited ( dspan, delim, _) => {
@@ -1310,10 +1305,10 @@ impl<'a> Parser<'a> {
1310
1305
pub ( crate ) fn parse_token_tree ( & mut self ) -> TokenTree {
1311
1306
match self . token . kind {
1312
1307
token:: OpenDelim ( ..) => {
1313
- // Grab the tokens from this frame .
1314
- let frame = & self . token_cursor . frame ;
1315
- let stream = frame . tree_cursor . stream . clone ( ) ;
1316
- let ( delim, span) = frame . delim_sp . unwrap ( ) ;
1308
+ // Grab the tokens within the delimiters .
1309
+ let tree_cursor = & self . token_cursor . tree_cursor ;
1310
+ let stream = tree_cursor. stream . clone ( ) ;
1311
+ let ( _ , delim, span) = * self . token_cursor . stack . last ( ) . unwrap ( ) ;
1317
1312
1318
1313
// Advance the token cursor through the entire delimited
1319
1314
// sequence. After getting the `OpenDelim` we are *within* the
0 commit comments