Rollup merge of rust-lang#120218 - compiler-errors:parse_macro_arg, r…

…=calebcartwright,ytmimi

rustfmt: Check that a token can begin a nonterminal kind before parsing it as a macro arg

r? ``@ytmimi`` and/or ``@calebcartwright``
cc ``@fmease``

I'm putting this on r-l/rust since it should fix the nightly rustfmt version. If you don't care about having this regression until the next rustfmt->rust sync, then I can move that PR over to r-l/rustfmt.

---

> Any idea why the formatting would have changed [from rust-lang#119099]?

**Copied over explanation:**

This has to do with the weirdness of the way that `parse_macro_arg` works. Unlike parsing nonterminal args in a macro-by-example, it eagerly tries, for example, to parse a type without checking that the beginning token may begin a type:

https://github.com/rust-lang/rustfmt/blob/bf967319e258acb9b1648a952bba52665eceaf52/src/parse/macros/mod.rs#L54

Contrast this to the nonterminal parsing code, which first checks that the nonterminal may begin with a given token:

https://github.com/rust-lang/rust/blob/ef71f1047e04438181d7cb925a833e2ada6ab390/compiler/rustc_parse/src/parser/nonterminal.rs#L47

In rust-lang#119099, ``@fmease`` implemented a change so that `const Tr` would be parsed as `dyn const Tr` (a trait object to a const trait) in edition 2015.

This is okay for the purposes of macros, because he explicitly made sure that `const` did not get added to the list of tokens that may begin a `:ty` nonterminal kind: rust-lang#119099 (comment)

However, since rustfmt is not so careful about eagerly parsing macro args before checking that they're legal in macro position, this changed the way that the string of tokens is being parsed into macro args.

src/tools/rustfmt/src/parse/macros/mod.rs

@@ -1,7 +1,7 @@
-use rustc_ast::token::{Delimiter, TokenKind};
+use rustc_ast::token::{Delimiter, NonterminalKind, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::{ast, ptr};
-use rustc_parse::parser::{ForceCollect, Parser};
+use rustc_parse::parser::{ForceCollect, Parser, Recovery};
 use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS};
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::{self, kw};
@@ -15,7 +15,7 @@ pub(crate) mod cfg_if;
 pub(crate) mod lazy_static;
 
 fn build_stream_parser<'a>(sess: &'a ParseSess, tokens: TokenStream) -> Parser<'a> {
-    stream_to_parser(sess, tokens, MACRO_ARGUMENTS)
+    stream_to_parser(sess, tokens, MACRO_ARGUMENTS).recovery(Recovery::Forbidden)
 }
 
 fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> {
@@ -24,45 +24,51 @@ fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser
 
 fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option<MacroArg> {
     macro_rules! parse_macro_arg {
-        ($macro_arg:ident, $parser:expr, $f:expr) => {
+        ($macro_arg:ident, $nt_kind:expr, $try_parse:expr, $then:expr) => {
             let mut cloned_parser = (*parser).clone();
-            match $parser(&mut cloned_parser) {
-                Ok(x) => {
-                    if parser.sess.dcx.has_errors().is_some() {
+            if Parser::nonterminal_may_begin_with($nt_kind, &cloned_parser.token) {
+                match $try_parse(&mut cloned_parser) {
+                    Ok(x) => {
+                        if parser.sess.dcx.has_errors().is_some() {
+                            parser.sess.dcx.reset_err_count();
+                        } else {
+                            // Parsing succeeded.
+                            *parser = cloned_parser;
+                            return Some(MacroArg::$macro_arg($then(x)?));
+                        }
+                    }
+                    Err(e) => {
+                        e.cancel();
                         parser.sess.dcx.reset_err_count();
-                    } else {
-                        // Parsing succeeded.
-                        *parser = cloned_parser;
-                        return Some(MacroArg::$macro_arg($f(x)?));
                     }
                 }
-                Err(e) => {
-                    e.cancel();
-                    parser.sess.dcx.reset_err_count();
-                }
             }
         };
     }
 
     parse_macro_arg!(
         Expr,
-        |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_expr(),
+        NonterminalKind::Expr,
+        |parser: &mut Parser<'b>| parser.parse_expr(),
         |x: ptr::P<ast::Expr>| Some(x)
     );
     parse_macro_arg!(
         Ty,
-        |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_ty(),
+        NonterminalKind::Ty,
+        |parser: &mut Parser<'b>| parser.parse_ty(),
         |x: ptr::P<ast::Ty>| Some(x)
     );
     parse_macro_arg!(
         Pat,
-        |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_pat_no_top_alt(None, None),
+        NonterminalKind::PatParam { inferred: false },
+        |parser: &mut Parser<'b>| parser.parse_pat_no_top_alt(None, None),
         |x: ptr::P<ast::Pat>| Some(x)
     );
     // `parse_item` returns `Option<ptr::P<ast::Item>>`.
     parse_macro_arg!(
         Item,
-        |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_item(ForceCollect::No),
+        NonterminalKind::Item,
+        |parser: &mut Parser<'b>| parser.parse_item(ForceCollect::No),
         |x: Option<ptr::P<ast::Item>>| x
     );
 

src/tools/rustfmt/tests/source/macros/rewrite-const-item.rs

@@ -0,0 +1 @@
+m!(const N: usize = 0;);

src/tools/rustfmt/tests/target/macros/rewrite-const-item.rs

@@ -0,0 +1,3 @@
+m!(
+    const N: usize = 0;
+);