Remove Nonterminal::NtTT.

It's only needed for macro expansion, not as a general element in the
AST. This commit removes it, adds `NtOrTt` for the parser and macro
expansion cases, and renames the variants in `NamedMatch` to better
match the new type.

Author: nnethercote

compiler/rustc_ast/src/ast_like.rs

@@ -51,7 +51,6 @@ impl AstLike for crate::token::Nonterminal {
             | Nonterminal::NtMeta(_)
             | Nonterminal::NtPath(_)
             | Nonterminal::NtVis(_)
-            | Nonterminal::NtTT(_)
             | Nonterminal::NtBlock(_)
             | Nonterminal::NtIdent(..)
             | Nonterminal::NtLifetime(_) => &[],
@@ -67,7 +66,6 @@ impl AstLike for crate::token::Nonterminal {
             | Nonterminal::NtMeta(_)
             | Nonterminal::NtPath(_)
             | Nonterminal::NtVis(_)
-            | Nonterminal::NtTT(_)
             | Nonterminal::NtBlock(_)
             | Nonterminal::NtIdent(..)
             | Nonterminal::NtLifetime(_) => {}
@@ -84,7 +82,7 @@ impl AstLike for crate::token::Nonterminal {
             Nonterminal::NtPath(path) => path.tokens_mut(),
             Nonterminal::NtVis(vis) => vis.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
-            Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None,
+            Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -787,7 +787,6 @@ pub fn visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut
             visit_lazy_tts(tokens, vis);
         }
         token::NtPath(path) => vis.visit_path(path),
-        token::NtTT(tt) => visit_tt(tt, vis),
         token::NtVis(visib) => vis.visit_vis(visib),
     }
 }

compiler/rustc_ast/src/token.rs

@@ -6,7 +6,6 @@ pub use TokenKind::*;
 
 use crate::ast;
 use crate::ptr::P;
-use crate::tokenstream::TokenTree;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
@@ -680,7 +679,6 @@ pub enum Nonterminal {
     NtMeta(P<ast::AttrItem>),
     NtPath(ast::Path),
     NtVis(ast::Visibility),
-    NtTT(TokenTree),
 }
 
 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
@@ -778,7 +776,6 @@ impl Nonterminal {
             NtMeta(attr_item) => attr_item.span(),
             NtPath(path) => path.span,
             NtVis(vis) => vis.span,
-            NtTT(tt) => tt.span(),
         }
     }
 }
@@ -790,7 +787,6 @@ impl PartialEq for Nonterminal {
                 ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
             }
             (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
-            (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
             // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
             // correctly based on data from AST. This will prevent them from matching each other
             // in macros. The comparison will become possible only when each nonterminal has an
@@ -813,7 +809,6 @@ impl fmt::Debug for Nonterminal {
             NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtMeta(..) => f.pad("NtMeta(..)"),
             NtPath(..) => f.pad("NtPath(..)"),
-            NtTT(..) => f.pad("NtTT(..)"),
             NtVis(..) => f.pad("NtVis(..)"),
             NtLifetime(..) => f.pad("NtLifetime(..)"),
         }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -714,7 +714,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(),
             token::NtLifetime(e) => e.to_string(),
             token::NtLiteral(ref e) => self.expr_to_string(e),
-            token::NtTT(ref tree) => self.tt_to_string(tree),
             token::NtVis(ref e) => self.vis_to_string(e),
         }
     }

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -76,7 +76,7 @@ crate use ParseResult::*;
 use crate::mbe::{self, SequenceRepetition, TokenTree};
 
 use rustc_ast::token::{self, DocComment, Nonterminal, Token};
-use rustc_parse::parser::Parser;
+use rustc_parse::parser::{NtOrTt, Parser};
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::MacroRulesNormalizedIdent;
 
@@ -275,7 +275,7 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
 }
 
 /// `NamedMatch` is a pattern-match result for a single metavar. All
-/// `MatchedNtNonTt`s in the `NamedMatch` have the same non-terminal type
+/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type
 /// (expr, item, etc).
 ///
 /// The in-memory structure of a particular `NamedMatch` represents the match
@@ -306,32 +306,29 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
 /// ```rust
 /// MatchedSeq([
 ///   MatchedSeq([
-///     MatchedNtNonTt(a),
-///     MatchedNtNonTt(b),
-///     MatchedNtNonTt(c),
-///     MatchedNtNonTt(d),
+///     MatchedNonterminal(a),
+///     MatchedNonterminal(b),
+///     MatchedNonterminal(c),
+///     MatchedNonterminal(d),
 ///   ]),
 ///   MatchedSeq([
-///     MatchedNtNonTt(a),
-///     MatchedNtNonTt(b),
-///     MatchedNtNonTt(c),
-///     MatchedNtNonTt(d),
-///     MatchedNtNonTt(e),
+///     MatchedNonterminal(a),
+///     MatchedNonterminal(b),
+///     MatchedNonterminal(c),
+///     MatchedNonterminal(d),
+///     MatchedNonterminal(e),
 ///   ])
 /// ])
 /// ```
 #[derive(Debug, Clone)]
 crate enum NamedMatch {
     MatchedSeq(Lrc<NamedMatchVec>),
 
-    // This variant should never hold an `NtTT`. `MatchedNtTt` should be used
-    // for that case.
-    MatchedNtNonTt(Lrc<Nonterminal>),
+    // A metavar match of type `tt`.
+    MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
 
-    // `NtTT` is handled without any cloning when transcribing, unlike other
-    // nonterminals. Therefore, an `Lrc` isn't helpful and causes unnecessary
-    // allocations. Hence this separate variant.
-    MatchedNtTt(rustc_ast::tokenstream::TokenTree),
+    // A metavar match of any type other than `tt`.
+    MatchedNonterminal(Lrc<Nonterminal>),
 }
 
 /// Takes a slice of token trees `ms` representing a matcher which successfully matched input
@@ -677,8 +674,8 @@ impl<'tt> TtParser<'tt> {
                             Ok(nt) => nt,
                         };
                         let m = match nt {
-                            Nonterminal::NtTT(tt) => MatchedNtTt(tt),
-                            _ => MatchedNtNonTt(Lrc::new(nt)),
+                            NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
+                            NtOrTt::Tt(tt) => MatchedTokenTree(tt),
                         };
                         item.push_match(match_cur, m);
                         item.idx += 1;

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -4,7 +4,7 @@ use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstF
 use crate::mbe;
 use crate::mbe::macro_check;
 use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success, TtParser};
-use crate::mbe::macro_parser::{MatchedNtTt, MatchedSeq};
+use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree};
 use crate::mbe::transcribe::transcribe;
 
 use rustc_ast as ast;
@@ -470,7 +470,7 @@ pub fn compile_declarative_macro(
         MatchedSeq(ref s) => s
             .iter()
             .map(|m| {
-                if let MatchedNtTt(ref tt) = *m {
+                if let MatchedTokenTree(ref tt) = *m {
                     let mut tts = vec![];
                     mbe::quoted::parse(
                         tt.clone().into(),
@@ -495,7 +495,7 @@ pub fn compile_declarative_macro(
         MatchedSeq(ref s) => s
             .iter()
             .map(|m| {
-                if let MatchedNtTt(ref tt) = *m {
+                if let MatchedTokenTree(ref tt) = *m {
                     let mut tts = vec![];
                     mbe::quoted::parse(
                         tt.clone().into(),

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -1,8 +1,8 @@
 use crate::base::ExtCtxt;
-use crate::mbe::macro_parser::{MatchedNtNonTt, MatchedNtTt, MatchedSeq, NamedMatch};
+use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
 use crate::mbe::{self, MetaVarExpr};
 use rustc_ast::mut_visit::{self, MutVisitor};
-use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
+use rustc_ast::token::{self, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -234,17 +234,16 @@ pub(super) fn transcribe<'a>(
                 let ident = MacroRulesNormalizedIdent::new(orignal_ident);
                 if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
                     match cur_matched {
-                        MatchedNtTt(ref tt) => {
+                        MatchedTokenTree(ref tt) => {
                             // `tt`s are emitted into the output stream directly as "raw tokens",
                             // without wrapping them into groups.
                             let token = tt.clone();
                             result.push(token.into());
                         }
-                        MatchedNtNonTt(ref nt) => {
+                        MatchedNonterminal(ref nt) => {
                             // Other variables are emitted into the output stream as groups with
                             // `Delimiter::None` to maintain parsing priorities.
                             // `Interpolated` is currently used for such groups in rustc parser.
-                            debug_assert!(!matches!(**nt, Nonterminal::NtTT(_)));
                             marker.visit_span(&mut sp);
                             let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
                             result.push(token.into());
@@ -312,7 +311,7 @@ fn lookup_cur_matched<'a>(
         let mut matched = matched;
         for &(idx, _) in repeats {
             match matched {
-                MatchedNtTt(_) | MatchedNtNonTt(_) => break,
+                MatchedTokenTree(_) | MatchedNonterminal(_) => break,
                 MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(),
             }
         }
@@ -402,7 +401,7 @@ fn lockstep_iter_size(
             let name = MacroRulesNormalizedIdent::new(name);
             match lookup_cur_matched(name, interpolations, repeats) {
                 Some(matched) => match matched {
-                    MatchedNtTt(_) | MatchedNtNonTt(_) => LockstepIterSize::Unconstrained,
+                    MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
                     MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name),
                 },
                 _ => LockstepIterSize::Unconstrained,
@@ -449,7 +448,7 @@ fn count_repetitions<'a>(
         sp: &DelimSpan,
     ) -> PResult<'a, usize> {
         match matched {
-            MatchedNtTt(_) | MatchedNtNonTt(_) => {
+            MatchedTokenTree(_) | MatchedNonterminal(_) => {
                 if declared_lhs_depth == 0 {
                     return Err(cx.struct_span_err(
                         sp.entire(),

compiler/rustc_parse/src/lib.rs

@@ -289,7 +289,6 @@ pub fn nt_to_tokenstream(
         Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
         Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
         Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
-        Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
         Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
             prepend_attrs(&expr.attrs, expr.tokens.as_ref())
         }

compiler/rustc_parse/src/parser/mod.rs

@@ -19,7 +19,7 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
 pub use path::PathStyle;
 
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, DelimToken, Token, TokenKind};
+use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind};
 use rustc_ast::tokenstream::AttributesData;
 use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
 use rustc_ast::tokenstream::{TokenStream, TokenTree};
@@ -1507,3 +1507,9 @@ pub enum FlatToken {
     /// handling of replace ranges.
     Empty,
 }
+
+#[derive(Debug)]
+pub enum NtOrTt {
+    Nt(Nonterminal),
+    Tt(TokenTree),
+}

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -1,12 +1,12 @@
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token};
+use rustc_ast::token::{self, NonterminalKind, Token};
 use rustc_ast::AstLike;
 use rustc_ast_pretty::pprust;
 use rustc_errors::PResult;
 use rustc_span::symbol::{kw, Ident};
 
 use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
-use crate::parser::{FollowedByType, ForceCollect, Parser, PathStyle};
+use crate::parser::{FollowedByType, ForceCollect, NtOrTt, Parser, PathStyle};
 
 impl<'a> Parser<'a> {
     /// Checks whether a non-terminal may begin with a particular token.
@@ -85,7 +85,7 @@ impl<'a> Parser<'a> {
             NonterminalKind::Lifetime => match token.kind {
                 token::Lifetime(_) => true,
                 token::Interpolated(ref nt) => {
-                    matches!(**nt, token::NtLifetime(_) | token::NtTT(_))
+                    matches!(**nt, token::NtLifetime(_))
                 }
                 _ => false,
             },
@@ -96,14 +96,16 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a non-terminal (e.g. MBE `:pat` or `:ident`).
-    pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonterminal> {
+    pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, NtOrTt> {
         // Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
         // needs to have them force-captured here.
         // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
         // which requires having captured tokens available. Since we cannot determine
         // in advance whether or not a proc-macro will be (transitively) invoked,
         // we always capture tokens for any `Nonterminal` which needs them.
         let mut nt = match kind {
+            // Note that TT is treated differently to all the others.
+            NonterminalKind::TT => return Ok(NtOrTt::Tt(self.parse_token_tree())),
             NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
                 Some(item) => token::NtItem(item),
                 None => {
@@ -124,9 +126,12 @@ impl<'a> Parser<'a> {
             NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr { .. } => {
                 token::NtPat(self.collect_tokens_no_attrs(|this| match kind {
                     NonterminalKind::PatParam { .. } => this.parse_pat_no_top_alt(None),
-                    NonterminalKind::PatWithOr { .. } => {
-                        this.parse_pat_allow_top_alt(None, RecoverComma::No, RecoverColon::No, CommaRecoveryMode::EitherTupleOrPipe)
-                    }
+                    NonterminalKind::PatWithOr { .. } => this.parse_pat_allow_top_alt(
+                        None,
+                        RecoverComma::No,
+                        RecoverColon::No,
+                        CommaRecoveryMode::EitherTupleOrPipe,
+                    ),
                     _ => unreachable!(),
                 })?)
             }
@@ -139,9 +144,10 @@ impl<'a> Parser<'a> {
                 )
             }
 
-            NonterminalKind::Ty => {
-                token::NtTy(self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?)
-            }
+            NonterminalKind::Ty => token::NtTy(
+                self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?,
+            ),
+
             // this could be handled like a token, since it is one
             NonterminalKind::Ident
                 if let Some((ident, is_raw)) = get_macro_ident(&self.token) =>
@@ -158,7 +164,6 @@ impl<'a> Parser<'a> {
                 self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?,
             ),
             NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item(true)?)),
-            NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
             NonterminalKind::Vis => token::NtVis(
                 self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?,
             ),
@@ -183,7 +188,7 @@ impl<'a> Parser<'a> {
             );
         }
 
-        Ok(nt)
+        Ok(NtOrTt::Nt(nt))
     }
 }