Rollup merge of #125815 - nnethercote:rustc_parse-top-level-cleanups, r=spastorino

`rustc_parse` top-level cleanups

A bunch of improvements in and around `compiler/rustc_parse/src/lib.rs`. Many of the changes streamline the API in that file from this (12 functions and one macro):
```
    name                              args                  return type
    ----                              ----                  -----------
    panictry_buffer!                  Result<T, Vec<Diag>>  T

pub parse_crate_from_file             path                  PResult<Crate>
pub parse_crate_attrs_from_file       path                  PResult<AttrVec>
pub parse_crate_from_source_str       name,src              PResult<Crate>
pub parse_crate_attrs_from_source_str name,src              PResult<AttrVec>

pub new_parser_from_source_str        name,src              Parser
pub maybe_new_parser_from_source_str  name,src              Result<Parser, Vec<Diag>>
pub new_parser_from_file              path,error_sp         Parser
    maybe_source_file_to_parser       srcfile               Result<Parser, Vec<Diag>>

pub parse_stream_from_source_str      name,src,override_sp  TokenStream
pub source_file_to_stream             srcfile,override_sp   TokenStream
    maybe_file_to_stream              srcfile,override_sp   Result<TokenStream, Vec<Diag>>

pub stream_to_parser                  stream,subparser_name Parser
```
to this:
```
    name                              args                  return type
    ----                              ----                  -----------
    unwrap_or_emit_fatal              Result<T, Vec<Diag>>  T

pub new_parser_from_source_str        name,src              Result<Parser, Vec<Diag>>
pub new_parser_from_file              path,error_sp         Result<Parser, Vec<Diag>>
    new_parser_from_source_file       srcfile               Result<Parser, Vec<Diag>>

pub source_str_to_stream              name,src,override_sp  Result<TokenStream, Vec<Diag>>
    source_file_to_stream             srcfile,override_sp   Result<TokenStream, Vec<Diag>>
```
I found the old API quite confusing, with lots of similar-sounding function names and no clear structure. I think the new API is much better.

r? `@spastorino`

Author: workingjubilee

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -196,7 +196,7 @@ impl CfgEval<'_, '_> {
         // Re-parse the tokens, setting the `capture_cfg` flag to save extra information
         // to the captured `AttrTokenStream` (specifically, we capture
         // `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`)
-        let mut parser = rustc_parse::stream_to_parser(&self.cfg.sess.psess, orig_tokens, None);
+        let mut parser = Parser::new(&self.cfg.sess.psess, orig_tokens, None);
         parser.capture_cfg = true;
         match parse_annotatable_with(&mut parser) {
             Ok(a) => annotatable = a,

compiler/rustc_builtin_macros/src/cmdline_attrs.rs

@@ -4,16 +4,17 @@ use crate::errors;
 use rustc_ast::attr::mk_attr;
 use rustc_ast::token;
 use rustc_ast::{self as ast, AttrItem, AttrStyle};
+use rustc_parse::{new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_session::parse::ParseSess;
 use rustc_span::FileName;
 
 pub fn inject(krate: &mut ast::Crate, psess: &ParseSess, attrs: &[String]) {
     for raw_attr in attrs {
-        let mut parser = rustc_parse::new_parser_from_source_str(
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
             psess,
             FileName::cli_crate_attr_source_code(raw_attr),
             raw_attr.clone(),
-        );
+        ));
 
         let start_span = parser.token.span;
         let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item(false) {

compiler/rustc_builtin_macros/src/source_util.rs

@@ -12,8 +12,8 @@ use rustc_expand::base::{
 };
 use rustc_expand::module::DirOwnership;
 use rustc_lint_defs::BuiltinLintDiag;
-use rustc_parse::new_parser_from_file;
 use rustc_parse::parser::{ForceCollect, Parser};
+use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::Symbol;
@@ -126,7 +126,7 @@ pub(crate) fn expand_include<'cx>(
             return ExpandResult::Ready(DummyResult::any(sp, guar));
         }
     };
-    let p = new_parser_from_file(cx.psess(), &file, Some(sp));
+    let p = unwrap_or_emit_fatal(new_parser_from_file(cx.psess(), &file, Some(sp)));
 
     // If in the included file we have e.g., `mod bar;`,
     // then the path of `bar.rs` should be relative to the directory of `file`.

compiler/rustc_driver_impl/src/lib.rs

@@ -32,6 +32,7 @@ use rustc_interface::{interface, Queries};
 use rustc_lint::unerased_lint_store;
 use rustc_metadata::creader::MetadataLoader;
 use rustc_metadata::locator;
+use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS};
 use rustc_session::config::{ErrorOutputType, Input, OutFileName, OutputType};
 use rustc_session::getopts::{self, Matches};
@@ -1264,12 +1265,13 @@ pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> Option<geto
 }
 
 fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> {
-    match &sess.io.input {
-        Input::File(ifile) => rustc_parse::parse_crate_attrs_from_file(ifile, &sess.psess),
+    let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
+        Input::File(file) => new_parser_from_file(&sess.psess, file, None),
         Input::Str { name, input } => {
-            rustc_parse::parse_crate_attrs_from_source_str(name.clone(), input.clone(), &sess.psess)
+            new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
         }
-    }
+    });
+    parser.parse_inner_attributes()
 }
 
 /// Runs a closure and catches unwinds triggered by fatal errors.

compiler/rustc_expand/src/base.rs

@@ -15,7 +15,7 @@ use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagCtxt, ErrorGuaranteed, PResult};
 use rustc_feature::Features;
 use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools};
-use rustc_parse::{parser, MACRO_ARGUMENTS};
+use rustc_parse::{parser::Parser, MACRO_ARGUMENTS};
 use rustc_session::config::CollapseMacroDebuginfo;
 use rustc_session::{parse::ParseSess, Limit, Session};
 use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
@@ -1149,8 +1149,8 @@ impl<'a> ExtCtxt<'a> {
     pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> {
         expand::MacroExpander::new(self, true)
     }
-    pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> {
-        rustc_parse::stream_to_parser(&self.sess.psess, stream, MACRO_ARGUMENTS)
+    pub fn new_parser_from_tts(&self, stream: TokenStream) -> Parser<'a> {
+        Parser::new(&self.sess.psess, stream, MACRO_ARGUMENTS)
     }
     pub fn source_map(&self) -> &'a SourceMap {
         self.sess.psess.source_map()

compiler/rustc_expand/src/module.rs

@@ -5,8 +5,8 @@ use crate::errors::{
 use rustc_ast::ptr::P;
 use rustc_ast::{token, AttrVec, Attribute, Inline, Item, ModSpans};
 use rustc_errors::{Diag, ErrorGuaranteed};
-use rustc_parse::new_parser_from_file;
 use rustc_parse::validate_attr;
+use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal};
 use rustc_session::parse::ParseSess;
 use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident};
@@ -66,7 +66,8 @@ pub(crate) fn parse_external_mod(
         }
 
         // Actually parse the external file as a module.
-        let mut parser = new_parser_from_file(&sess.psess, &mp.file_path, Some(span));
+        let mut parser =
+            unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span)));
         let (inner_attrs, items, inner_span) =
             parser.parse_mod(&token::Eof).map_err(|err| ModError::ParserError(err))?;
         attrs.extend(inner_attrs);

compiler/rustc_expand/src/proc_macro.rs

@@ -6,7 +6,7 @@ use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::ErrorGuaranteed;
-use rustc_parse::parser::ForceCollect;
+use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_session::config::ProcMacroExecutionStrategy;
 use rustc_span::profiling::SpannedEventArgRecorder;
 use rustc_span::Span;
@@ -154,8 +154,7 @@ impl MultiItemModifier for DeriveProcMacro {
         };
 
         let error_count_before = ecx.dcx().err_count();
-        let mut parser =
-            rustc_parse::stream_to_parser(&ecx.sess.psess, stream, Some("proc-macro derive"));
+        let mut parser = Parser::new(&ecx.sess.psess, stream, Some("proc-macro derive"));
         let mut items = vec![];
 
         loop {

compiler/rustc_expand/src/proc_macro_server.rs

@@ -13,7 +13,8 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
 use rustc_parse::lexer::nfc_normalize;
-use rustc_parse::parse_stream_from_source_str;
+use rustc_parse::parser::Parser;
+use rustc_parse::{new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
 use rustc_session::parse::ParseSess;
 use rustc_span::def_id::CrateNum;
 use rustc_span::symbol::{self, sym, Symbol};
@@ -466,7 +467,8 @@ impl server::FreeFunctions for Rustc<'_, '_> {
 
     fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
         let name = FileName::proc_macro_source_code(s);
-        let mut parser = rustc_parse::new_parser_from_source_str(self.psess(), name, s.to_owned());
+        let mut parser =
+            unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned()));
 
         let first_span = parser.token.span.data();
         let minus_present = parser.eat(&token::BinOp(token::Minus));
@@ -538,12 +540,12 @@ impl server::TokenStream for Rustc<'_, '_> {
     }
 
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
-        parse_stream_from_source_str(
+        unwrap_or_emit_fatal(source_str_to_stream(
+            self.psess(),
             FileName::proc_macro_source_code(src),
             src.to_string(),
-            self.psess(),
             Some(self.call_site),
-        )
+        ))
     }
 
     fn to_string(&mut self, stream: &Self::TokenStream) -> String {
@@ -553,11 +555,7 @@ impl server::TokenStream for Rustc<'_, '_> {
     fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStream, ()> {
         // Parse the expression from our tokenstream.
         let expr: PResult<'_, _> = try {
-            let mut p = rustc_parse::stream_to_parser(
-                self.psess(),
-                stream.clone(),
-                Some("proc_macro expand expr"),
-            );
+            let mut p = Parser::new(self.psess(), stream.clone(), Some("proc_macro expand expr"));
             let expr = p.parse_expr()?;
             if p.token != token::Eof {
                 p.unexpected()?;

compiler/rustc_interface/src/interface.rs

@@ -14,7 +14,7 @@ use rustc_lint::LintStore;
 use rustc_middle::ty;
 use rustc_middle::ty::CurrentGcx;
 use rustc_middle::util::Providers;
-use rustc_parse::maybe_new_parser_from_source_str;
+use rustc_parse::new_parser_from_source_str;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
 use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
@@ -67,7 +67,7 @@ pub(crate) fn parse_cfg(dcx: &DiagCtxt, cfgs: Vec<String>) -> Cfg {
                 };
             }
 
-            match maybe_new_parser_from_source_str(&psess, filename, s.to_string()) {
+            match new_parser_from_source_str(&psess, filename, s.to_string()) {
                 Ok(mut parser) => match parser.parse_meta_item() {
                     Ok(meta_item) if parser.token == token::Eof => {
                         if meta_item.path.segments.len() != 1 {
@@ -166,7 +166,7 @@ pub(crate) fn parse_check_cfg(dcx: &DiagCtxt, specs: Vec<String>) -> CheckCfg {
             error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
         };
 
-        let mut parser = match maybe_new_parser_from_source_str(&psess, filename, s.to_string()) {
+        let mut parser = match new_parser_from_source_str(&psess, filename, s.to_string()) {
             Ok(parser) => parser,
             Err(errs) => {
                 errs.into_iter().for_each(|err| err.cancel());

compiler/rustc_interface/src/passes.rs

@@ -19,7 +19,9 @@ use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
 use rustc_middle::ty::{self, GlobalCtxt, RegisteredTools, TyCtxt};
 use rustc_middle::util::Providers;
-use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str, validate_attr};
+use rustc_parse::{
+    new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal, validate_attr,
+};
 use rustc_passes::{abi_test, hir_stats, layout_test};
 use rustc_resolve::Resolver;
 use rustc_session::code_stats::VTableSizeInfo;
@@ -42,11 +44,14 @@ use std::{env, fs, iter};
 use tracing::{info, instrument};
 
 pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> {
-    let krate = sess.time("parse_crate", || match &sess.io.input {
-        Input::File(file) => parse_crate_from_file(file, &sess.psess),
-        Input::Str { input, name } => {
-            parse_crate_from_source_str(name.clone(), input.clone(), &sess.psess)
-        }
+    let krate = sess.time("parse_crate", || {
+        let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
+            Input::File(file) => new_parser_from_file(&sess.psess, file, None),
+            Input::Str { input, name } => {
+                new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
+            }
+        });
+        parser.parse_crate_mod()
     })?;
 
     if sess.opts.unstable_opts.input_stats {