Adjust a lot of FollowedBy values.

The previous commit was conservative in the `Spacing` to `FollowedBy`
conversion. This commit makes many `FollowedBy` values more precise.

In particular, it removes `TokenStream::token_fby_{space,punct}`,
because for a token stream with a single token in it,
`FollowedBy::Other` is the only value that makes sense.

compiler/rustc_ast/src/tokenstream.rs

@@ -450,19 +450,8 @@ impl TokenStream {
     }
 
     /// Create a token stream containing a single token with
-    /// `FollowedBy::Space`.
-    pub fn token_fby_space(kind: TokenKind, span: Span) -> TokenStream {
-        TokenStream::new(vec![TokenTree::token_fby_space(kind, span)])
-    }
-
-    /// Create a token stream containing a single token with
-    /// `FollowedBy::Punct`.
-    pub fn token_fby_punct(kind: TokenKind, span: Span) -> TokenStream {
-        TokenStream::new(vec![TokenTree::token_fby_punct(kind, span)])
-    }
-
-    /// Create a token stream containing a single token with
-    /// `FollowedBy::Other`.
+    /// `FollowedBy::Other`. This is the only `FollowedBy` value that makes
+    /// sense for a token stream containing a single token.
     pub fn token_fby_other(kind: TokenKind, span: Span) -> TokenStream {
         TokenStream::new(vec![TokenTree::token_fby_other(kind, span)])
     }
@@ -490,16 +479,16 @@ impl TokenStream {
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
         match nt {
             Nonterminal::NtIdent(ident, is_raw) => {
-                TokenStream::token_fby_space(token::Ident(ident.name, *is_raw), ident.span)
+                TokenStream::token_fby_other(token::Ident(ident.name, *is_raw), ident.span)
             }
             Nonterminal::NtLifetime(ident) => {
-                TokenStream::token_fby_space(token::Lifetime(ident.name), ident.span)
+                TokenStream::token_fby_other(token::Lifetime(ident.name), ident.span)
             }
             Nonterminal::NtItem(item) => TokenStream::from_ast(item),
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
             Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
                 // FIXME: Properly collect tokens for empty statements.
-                TokenStream::token_fby_space(token::Semi, stmt.span)
+                TokenStream::token_fby_other(token::Semi, stmt.span)
             }
             Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
             Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
@@ -664,7 +653,7 @@ impl TokenStream {
                 [
                     TokenTree::token_fby_space(token::Ident(sym::doc, false), span),
                     TokenTree::token_fby_space(token::Eq, span),
-                    TokenTree::token_fby_space(
+                    TokenTree::token_fby_other(
                         TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
                         span,
                     ),
@@ -675,12 +664,12 @@ impl TokenStream {
 
             if attr_style == AttrStyle::Inner {
                 vec![
-                    TokenTree::token_fby_space(token::Pound, span),
-                    TokenTree::token_fby_space(token::Not, span),
+                    TokenTree::token_fby_punct(token::Pound, span),
+                    TokenTree::token_fby_other(token::Not, span),
                     body,
                 ]
             } else {
-                vec![TokenTree::token_fby_space(token::Pound, span), body]
+                vec![TokenTree::token_fby_other(token::Pound, span), body]
             }
         }
     }

compiler/rustc_builtin_macros/src/assert/context.rs

@@ -151,7 +151,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
     fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
         let escaped_expr_str = escape_to_fmt(expr_str);
         let initial = [
-            TokenTree::token_fby_space(
+            TokenTree::token_fby_other(
                 token::Literal(token::Lit {
                     kind: token::LitKind::Str,
                     symbol: Symbol::intern(&if self.fmt_string.is_empty() {
@@ -170,7 +170,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
         ];
         let captures = self.capture_decls.iter().flat_map(|cap| {
             [
-                TokenTree::token_fby_space(token::Ident(cap.ident.name, false), cap.ident.span),
+                TokenTree::token_fby_other(token::Ident(cap.ident.name, false), cap.ident.span),
                 TokenTree::token_fby_space(token::Comma, self.span),
             ]
         });

compiler/rustc_expand/src/config.rs

@@ -373,16 +373,20 @@ impl<'a> StripUnconfigured<'a> {
         };
         let pound_span = pound_token.span;
 
-        let mut trees = vec![AttrTokenTree::Token(pound_token, FollowedBy::Space)];
-        if attr.style == AttrStyle::Inner {
+        let mut trees = if attr.style == AttrStyle::Inner {
             // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
             let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
                 orig_trees.next().unwrap().clone()
             else {
                 panic!("Bad tokens for attribute {attr:?}");
             };
-            trees.push(AttrTokenTree::Token(bang_token, FollowedBy::Space));
-        }
+            vec![
+                AttrTokenTree::Token(pound_token, FollowedBy::Punct),
+                AttrTokenTree::Token(bang_token, FollowedBy::Other),
+            ]
+        } else {
+            vec![AttrTokenTree::Token(pound_token, FollowedBy::Other)]
+        };
         // We don't really have a good span to use for the synthesized `[]`
         // in `#[attr]`, so just use the span of the `#` token.
         let bracket_group = AttrTokenTree::Delimited(

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -242,7 +242,7 @@ pub(super) fn transcribe<'a>(
                     // with modified syntax context. (I believe this supports nested macros).
                     marker.visit_span(&mut sp);
                     marker.visit_ident(&mut original_ident);
-                    result.push(TokenTree::token_fby_space(token::Dollar, sp));
+                    result.push(TokenTree::token_fby_other(token::Dollar, sp));
                     result.push(TokenTree::Token(
                         Token::from_ast_ident(original_ident),
                         FollowedBy::Space,

compiler/rustc_expand/src/proc_macro.rs

@@ -126,7 +126,7 @@ impl MultiItemModifier for DeriveProcMacro {
                 Annotatable::Stmt(stmt) => token::NtStmt(stmt),
                 _ => unreachable!(),
             };
-            TokenStream::token_fby_space(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
+            TokenStream::token_fby_other(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
         } else {
             item.to_tokens()
         };

compiler/rustc_expand/src/proc_macro_server.rs

@@ -219,14 +219,13 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     for ch in data.as_str().chars() {
                         escaped.extend(ch.escape_debug());
                     }
-                    let stream = [
-                        Ident(sym::doc, false),
-                        Eq,
-                        TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
-                    ]
-                    .into_iter()
-                    .map(|kind| tokenstream::TokenTree::token_fby_space(kind, span))
-                    .collect();
+                    let stream = tokenstream::TokenStream::from_iter([
+                        tokenstream::TokenTree::token_fby_space(Ident(sym::doc, false), span),
+                        tokenstream::TokenTree::token_fby_space(Eq, span),
+                        tokenstream::TokenTree::token_fby_other(
+                            TokenKind::lit(token::Str, Symbol::intern(&escaped), None), span
+                        ),
+                    ]);
                     trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
                     if attr_style == ast::AttrStyle::Inner {
                         trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
@@ -339,7 +338,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
                 let minus = BinOp(BinOpToken::Minus);
                 let symbol = Symbol::intern(&symbol.as_str()[1..]);
                 let integer = TokenKind::lit(token::Integer, symbol, suffix);
-                let a = tokenstream::TokenTree::token_fby_space(minus, span);
+                let a = tokenstream::TokenTree::token_fby_other(minus, span);
                 let b = tokenstream::TokenTree::token_fby_space(integer, span);
                 smallvec![a, b]
             }
@@ -352,7 +351,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
                 let minus = BinOp(BinOpToken::Minus);
                 let symbol = Symbol::intern(&symbol.as_str()[1..]);
                 let float = TokenKind::lit(token::Float, symbol, suffix);
-                let a = tokenstream::TokenTree::token_fby_space(minus, span);
+                let a = tokenstream::TokenTree::token_fby_other(minus, span);
                 let b = tokenstream::TokenTree::token_fby_space(float, span);
                 smallvec![a, b]
             }
@@ -545,17 +544,17 @@ impl server::TokenStream for Rustc<'_, '_> {
         // be recovered in the general case.
         match &expr.kind {
             ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => {
-                Ok(tokenstream::TokenStream::token_fby_space(
+                Ok(tokenstream::TokenStream::token_fby_other(
                     token::Ident(token_lit.symbol, false),
                     expr.span,
                 ))
             }
             ast::ExprKind::Lit(token_lit) => {
-                Ok(tokenstream::TokenStream::token_fby_space(token::Literal(*token_lit), expr.span))
+                Ok(tokenstream::TokenStream::token_fby_other(token::Literal(*token_lit), expr.span))
             }
             ast::ExprKind::IncludedBytes(bytes) => {
                 let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None);
-                Ok(tokenstream::TokenStream::token_fby_space(
+                Ok(tokenstream::TokenStream::token_fby_other(
                     token::TokenKind::Literal(lit),
                     expr.span,
                 ))
@@ -566,11 +565,11 @@ impl server::TokenStream for Rustc<'_, '_> {
                         Ok(Self::TokenStream::from_iter([
                             // FIXME: The span of the `-` token is lost when
                             // parsing, so we cannot faithfully recover it here.
-                            tokenstream::TokenTree::token_fby_space(
+                            tokenstream::TokenTree::token_fby_other(
                                 token::BinOp(token::Minus),
                                 e.span,
                             ),
-                            tokenstream::TokenTree::token_fby_space(
+                            tokenstream::TokenTree::token_fby_other(
                                 token::Literal(*token_lit),
                                 e.span,
                             ),

compiler/rustc_expand/src/tokenstream/tests.rs

@@ -87,7 +87,7 @@ fn test_is_empty() {
     create_default_session_globals_then(|| {
         let test0 = TokenStream::default();
         let test1 =
-            TokenStream::token_fby_space(token::Ident(Symbol::intern("a"), false), sp(0, 1));
+            TokenStream::token_fby_other(token::Ident(Symbol::intern("a"), false), sp(0, 1));
         let test2 = string_to_ts("foo(bar::baz)");
 
         assert_eq!(test0.is_empty(), true);
@@ -102,7 +102,7 @@ fn test_dotdotdot() {
         let mut stream = TokenStream::default();
         stream.push_tree(TokenTree::token_fby_punct(token::Dot, sp(0, 1)));
         stream.push_tree(TokenTree::token_fby_punct(token::Dot, sp(1, 2)));
-        stream.push_tree(TokenTree::token_fby_space(token::Dot, sp(2, 3)));
+        stream.push_tree(TokenTree::token_fby_other(token::Dot, sp(2, 3)));
         assert!(stream.eq_unspanned(&string_to_ts("...")));
         assert_eq!(stream.trees().count(), 1);
     })

compiler/rustc_parse/src/parser/expr.rs

@@ -1139,9 +1139,7 @@ impl<'a> Parser<'a> {
             // 1.2 | 1.2e3
             DestructuredFloat::MiddleDot(symbol1, ident1_span, dot_span, symbol2, ident2_span) => {
                 self.token = Token::new(token::Ident(symbol1, false), ident1_span);
-                // This needs to be `FollowedBy::Space` to prevent regressions.
-                // See issue #76399 and PR #76285 for more details
-                let next_token1 = (Token::new(token::Dot, dot_span), FollowedBy::Space);
+                let next_token1 = (Token::new(token::Dot, dot_span), FollowedBy::Other);
                 let base1 =
                     self.parse_expr_tuple_field_access(lo, base, symbol1, None, Some(next_token1));
                 let next_token2 = Token::new(token::Ident(symbol2, false), ident2_span);

compiler/rustc_parse/src/parser/mod.rs

@@ -286,7 +286,7 @@ impl TokenCursor {
                 // No close delimiter to return; continue on to the next iteration.
             } else {
                 // We have exhausted the outermost token stream.
-                return (Token::new(token::Eof, DUMMY_SP), FollowedBy::Space);
+                return (Token::new(token::Eof, DUMMY_SP), FollowedBy::Other);
             }
         }
     }

tests/ui/proc-macro/auxiliary/expand-expr.rs

@@ -99,21 +99,12 @@ pub fn expand_expr_is(input: TokenStream) -> TokenStream {
     let expected = expected_tts.into_iter().collect::<TokenStream>();
     let expanded = iter.collect::<TokenStream>().expand_expr().expect("expand_expr failed");
 
-    // FIXME: Temporarily broken. The next commit will fix it.
-    //
-    // +       LL | simple_lit!(-3.14159);
-    // +          | --------------------- in this macro invocation
-    // +          |
-    // +          = help: message: assert failed
-    // +                  expected: `-3.14159`
-    // +                  expanded: `- 3.14159`
-    //
-    // assert!(
-    //     expected.to_string() == expanded.to_string(),
-    //     "assert failed\nexpected: `{}`\nexpanded: `{}`",
-    //     expected.to_string(),
-    //     expanded.to_string()
-    // );
+    assert!(
+        expected.to_string() == expanded.to_string(),
+        "assert failed\nexpected: `{}`\nexpanded: `{}`",
+        expected.to_string(),
+        expanded.to_string()
+    );
 
     // Also compare the raw tts to make sure they line up.
     assert_ts_eq(&expected, &expanded);

tests/ui/proc-macro/cfg-eval-inner.stdout

@@ -1,15 +1,15 @@
 PRINT-ATTR INPUT (DISPLAY): impl
 Foo<[u8;
 {
-    #! [rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner
-    { field: [u8; { #! [rustc_dummy(another_cursed_inner)] 1 }] } 0
-}] > { #! [rustc_dummy(evaluated_attr)] fn bar() {} }
+    #![rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner
+    { field: [u8; { #![rustc_dummy(another_cursed_inner)] 1 }] } 0
+}] > { #![rustc_dummy(evaluated_attr)] fn bar() {} }
 PRINT-ATTR RE-COLLECTED (DISPLAY): impl Foo <
 [u8;
 {
-    #! [rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner
-    { field: [u8; { #! [rustc_dummy(another_cursed_inner)] 1 }] } 0
-}] > { #! [rustc_dummy(evaluated_attr)] fn bar() {} }
+    #![rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner
+    { field: [u8; { #![rustc_dummy(another_cursed_inner)] 1 }] } 0
+}] > { #![rustc_dummy(evaluated_attr)] fn bar() {} }
 PRINT-ATTR DEEP-RE-COLLECTED (DISPLAY): impl Foo <
 [u8 ;
 {
@@ -47,7 +47,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
                 stream: TokenStream [
                     Punct {
                         ch: '#',
-                        spacing: Alone,
+                        spacing: Joint,
                         span: $DIR/cfg-eval-inner.rs:19:5: 19:6 (#0),
                     },
                     Punct {
@@ -142,7 +142,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
                                         stream: TokenStream [
                                             Punct {
                                                 ch: '#',
-                                                spacing: Alone,
+                                                spacing: Joint,
                                                 span: $DIR/cfg-eval-inner.rs:23:13: 23:14 (#0),
                                             },
                                             Punct {
@@ -207,7 +207,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
         stream: TokenStream [
             Punct {
                 ch: '#',
-                spacing: Alone,
+                spacing: Joint,
                 span: $DIR/cfg-eval-inner.rs:32:5: 32:6 (#0),
             },
             Punct {

tests/ui/proc-macro/inner-attrs.stdout

@@ -579,7 +579,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): struct MyDerivePrint
-{ field: [u8; { match true { _ => { #! [rustc_dummy(third)] true } } ; 0 }] }
+{ field: [u8; { match true { _ => { #![rustc_dummy(third)] true } } ; 0 }] }
 PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): struct MyDerivePrint
 {
     field :
@@ -651,7 +651,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
                                         stream: TokenStream [
                                             Punct {
                                                 ch: '#',
-                                                spacing: Alone,
+                                                spacing: Joint,
                                                 span: $DIR/inner-attrs.rs:40:17: 40:18 (#0),
                                             },
                                             Punct {

tests/ui/proc-macro/issue-78675-captured-inner-attrs.stdout

@@ -46,7 +46,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
                         stream: TokenStream [
                             Punct {
                                 ch: '#',
-                                spacing: Alone,
+                                spacing: Joint,
                                 span: $DIR/issue-78675-captured-inner-attrs.rs:28:9: 28:16 (#0),
                             },
                             Punct {

tests/ui/proc-macro/macro-rules-derive-cfg.stdout

@@ -1,4 +1,13 @@
 PRINT-DERIVE INPUT (DISPLAY): struct Foo
+{
+    val :
+    [bool ;
+    {
+        let a = #[rustc_dummy(first)] #[rustc_dummy(second)]
+        { #![allow(unused)] 30 } ; 0
+    }]
+}
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): struct Foo
 {
     val :
     [bool ;
@@ -111,7 +120,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
                                 stream: TokenStream [
                                     Punct {
                                         ch: '#',
-                                        spacing: Alone,
+                                        spacing: Joint,
                                         span: $DIR/macro-rules-derive-cfg.rs:27:5: 27:6 (#0),
                                     },
                                     Punct {