From 0a92053591aef1bf3f26ad678a0049897264d6a4 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 8 Aug 2023 14:41:38 +1000 Subject: [PATCH] Adjust a lot of `FollowedBy` values. The previous commit was conservative in the `Spacing` to `FollowedBy` conversion. This commit makes many `FollowedBy` values more precise. In particular, it removes `TokenStream::token_fby_{space,punct}`, because for a token stream with a single token in it, `FollowedBy::Other` is the only value that makes sense. --- compiler/rustc_ast/src/tokenstream.rs | 29 ++++++------------- .../src/assert/context.rs | 4 +-- compiler/rustc_expand/src/config.rs | 12 +++++--- compiler/rustc_expand/src/mbe/transcribe.rs | 2 +- compiler/rustc_expand/src/proc_macro.rs | 2 +- .../rustc_expand/src/proc_macro_server.rs | 29 +++++++++---------- .../rustc_expand/src/tokenstream/tests.rs | 4 +-- compiler/rustc_parse/src/parser/expr.rs | 4 +-- compiler/rustc_parse/src/parser/mod.rs | 2 +- tests/ui/proc-macro/auxiliary/expand-expr.rs | 21 ++++---------- tests/ui/proc-macro/cfg-eval-inner.stdout | 18 ++++++------ tests/ui/proc-macro/inner-attrs.stdout | 4 +-- .../issue-78675-captured-inner-attrs.stdout | 2 +- .../proc-macro/macro-rules-derive-cfg.stdout | 11 ++++++- 14 files changed, 67 insertions(+), 77 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 62accdf1f3b23..0270eb4467603 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -450,19 +450,8 @@ impl TokenStream { } /// Create a token stream containing a single token with - /// `FollowedBy::Space`. - pub fn token_fby_space(kind: TokenKind, span: Span) -> TokenStream { - TokenStream::new(vec![TokenTree::token_fby_space(kind, span)]) - } - - /// Create a token stream containing a single token with - /// `FollowedBy::Punct`. - pub fn token_fby_punct(kind: TokenKind, span: Span) -> TokenStream { - TokenStream::new(vec![TokenTree::token_fby_punct(kind, span)]) - } - - /// Create a token stream containing a single token with - /// `FollowedBy::Other`. + /// `FollowedBy::Other`. This is the only `FollowedBy` value that makes + /// sense for a token stream containing a single token. pub fn token_fby_other(kind: TokenKind, span: Span) -> TokenStream { TokenStream::new(vec![TokenTree::token_fby_other(kind, span)]) } @@ -490,16 +479,16 @@ impl TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { match nt { Nonterminal::NtIdent(ident, is_raw) => { - TokenStream::token_fby_space(token::Ident(ident.name, *is_raw), ident.span) + TokenStream::token_fby_other(token::Ident(ident.name, *is_raw), ident.span) } Nonterminal::NtLifetime(ident) => { - TokenStream::token_fby_space(token::Lifetime(ident.name), ident.span) + TokenStream::token_fby_other(token::Lifetime(ident.name), ident.span) } Nonterminal::NtItem(item) => TokenStream::from_ast(item), Nonterminal::NtBlock(block) => TokenStream::from_ast(block), Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { // FIXME: Properly collect tokens for empty statements. - TokenStream::token_fby_space(token::Semi, stmt.span) + TokenStream::token_fby_other(token::Semi, stmt.span) } Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt), Nonterminal::NtPat(pat) => TokenStream::from_ast(pat), @@ -664,7 +653,7 @@ impl TokenStream { [ TokenTree::token_fby_space(token::Ident(sym::doc, false), span), TokenTree::token_fby_space(token::Eq, span), - TokenTree::token_fby_space( + TokenTree::token_fby_other( TokenKind::lit(token::StrRaw(num_of_hashes), data, None), span, ), @@ -675,12 +664,12 @@ impl TokenStream { if attr_style == AttrStyle::Inner { vec![ - TokenTree::token_fby_space(token::Pound, span), - TokenTree::token_fby_space(token::Not, span), + TokenTree::token_fby_punct(token::Pound, span), + TokenTree::token_fby_other(token::Not, span), body, ] } else { - vec![TokenTree::token_fby_space(token::Pound, span), body] + vec![TokenTree::token_fby_other(token::Pound, span), body] } } } diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs index 5239c6e30fa02..d56d51da421ee 100644 --- a/compiler/rustc_builtin_macros/src/assert/context.rs +++ b/compiler/rustc_builtin_macros/src/assert/context.rs @@ -151,7 +151,7 @@ impl<'cx, 'a> Context<'cx, 'a> { fn build_panic(&self, expr_str: &str, panic_path: Path) -> P { let escaped_expr_str = escape_to_fmt(expr_str); let initial = [ - TokenTree::token_fby_space( + TokenTree::token_fby_other( token::Literal(token::Lit { kind: token::LitKind::Str, symbol: Symbol::intern(&if self.fmt_string.is_empty() { @@ -170,7 +170,7 @@ impl<'cx, 'a> Context<'cx, 'a> { ]; let captures = self.capture_decls.iter().flat_map(|cap| { [ - TokenTree::token_fby_space(token::Ident(cap.ident.name, false), cap.ident.span), + TokenTree::token_fby_other(token::Ident(cap.ident.name, false), cap.ident.span), TokenTree::token_fby_space(token::Comma, self.span), ] }); diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index d61cf9c705912..c9e462d8cc3e7 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -373,16 +373,20 @@ impl<'a> StripUnconfigured<'a> { }; let pound_span = pound_token.span; - let mut trees = vec![AttrTokenTree::Token(pound_token, FollowedBy::Space)]; - if attr.style == AttrStyle::Inner { + let mut trees = if attr.style == AttrStyle::Inner { // For inner attributes, we do the same thing for the `!` in `#![some_attr]` let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap().clone() else { panic!("Bad tokens for attribute {attr:?}"); }; - trees.push(AttrTokenTree::Token(bang_token, FollowedBy::Space)); - } + vec![ + AttrTokenTree::Token(pound_token, FollowedBy::Punct), + AttrTokenTree::Token(bang_token, FollowedBy::Other), + ] + } else { + vec![AttrTokenTree::Token(pound_token, FollowedBy::Other)] + }; // We don't really have a good span to use for the synthesized `[]` // in `#[attr]`, so just use the span of the `#` token. let bracket_group = AttrTokenTree::Delimited( diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index be8db3414ad7c..3abfdbedd4138 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -242,7 +242,7 @@ pub(super) fn transcribe<'a>( // with modified syntax context. (I believe this supports nested macros). marker.visit_span(&mut sp); marker.visit_ident(&mut original_ident); - result.push(TokenTree::token_fby_space(token::Dollar, sp)); + result.push(TokenTree::token_fby_other(token::Dollar, sp)); result.push(TokenTree::Token( Token::from_ast_ident(original_ident), FollowedBy::Space, diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 1fca1276a49ff..5289349cba3be 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -126,7 +126,7 @@ impl MultiItemModifier for DeriveProcMacro { Annotatable::Stmt(stmt) => token::NtStmt(stmt), _ => unreachable!(), }; - TokenStream::token_fby_space(token::Interpolated(Lrc::new(nt)), DUMMY_SP) + TokenStream::token_fby_other(token::Interpolated(Lrc::new(nt)), DUMMY_SP) } else { item.to_tokens() }; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 0758ec0b71652..852298c6957c2 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -219,14 +219,13 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token_fby_space(minus, span); + let a = tokenstream::TokenTree::token_fby_other(minus, span); let b = tokenstream::TokenTree::token_fby_space(integer, span); smallvec![a, b] } @@ -352,7 +351,7 @@ impl ToInternal> let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token_fby_space(minus, span); + let a = tokenstream::TokenTree::token_fby_other(minus, span); let b = tokenstream::TokenTree::token_fby_space(float, span); smallvec![a, b] } @@ -545,17 +544,17 @@ impl server::TokenStream for Rustc<'_, '_> { // be recovered in the general case. match &expr.kind { ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => { - Ok(tokenstream::TokenStream::token_fby_space( + Ok(tokenstream::TokenStream::token_fby_other( token::Ident(token_lit.symbol, false), expr.span, )) } ast::ExprKind::Lit(token_lit) => { - Ok(tokenstream::TokenStream::token_fby_space(token::Literal(*token_lit), expr.span)) + Ok(tokenstream::TokenStream::token_fby_other(token::Literal(*token_lit), expr.span)) } ast::ExprKind::IncludedBytes(bytes) => { let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None); - Ok(tokenstream::TokenStream::token_fby_space( + Ok(tokenstream::TokenStream::token_fby_other( token::TokenKind::Literal(lit), expr.span, )) @@ -566,11 +565,11 @@ impl server::TokenStream for Rustc<'_, '_> { Ok(Self::TokenStream::from_iter([ // FIXME: The span of the `-` token is lost when // parsing, so we cannot faithfully recover it here. - tokenstream::TokenTree::token_fby_space( + tokenstream::TokenTree::token_fby_other( token::BinOp(token::Minus), e.span, ), - tokenstream::TokenTree::token_fby_space( + tokenstream::TokenTree::token_fby_other( token::Literal(*token_lit), e.span, ), diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs index 8b0deced4456a..040dcedd17bd4 100644 --- a/compiler/rustc_expand/src/tokenstream/tests.rs +++ b/compiler/rustc_expand/src/tokenstream/tests.rs @@ -87,7 +87,7 @@ fn test_is_empty() { create_default_session_globals_then(|| { let test0 = TokenStream::default(); let test1 = - TokenStream::token_fby_space(token::Ident(Symbol::intern("a"), false), sp(0, 1)); + TokenStream::token_fby_other(token::Ident(Symbol::intern("a"), false), sp(0, 1)); let test2 = string_to_ts("foo(bar::baz)"); assert_eq!(test0.is_empty(), true); @@ -102,7 +102,7 @@ fn test_dotdotdot() { let mut stream = TokenStream::default(); stream.push_tree(TokenTree::token_fby_punct(token::Dot, sp(0, 1))); stream.push_tree(TokenTree::token_fby_punct(token::Dot, sp(1, 2))); - stream.push_tree(TokenTree::token_fby_space(token::Dot, sp(2, 3))); + stream.push_tree(TokenTree::token_fby_other(token::Dot, sp(2, 3))); assert!(stream.eq_unspanned(&string_to_ts("..."))); assert_eq!(stream.trees().count(), 1); }) diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index d19004314bb73..0889fc0ca953e 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1139,9 +1139,7 @@ impl<'a> Parser<'a> { // 1.2 | 1.2e3 DestructuredFloat::MiddleDot(symbol1, ident1_span, dot_span, symbol2, ident2_span) => { self.token = Token::new(token::Ident(symbol1, false), ident1_span); - // This needs to be `FollowedBy::Space` to prevent regressions. - // See issue #76399 and PR #76285 for more details - let next_token1 = (Token::new(token::Dot, dot_span), FollowedBy::Space); + let next_token1 = (Token::new(token::Dot, dot_span), FollowedBy::Other); let base1 = self.parse_expr_tuple_field_access(lo, base, symbol1, None, Some(next_token1)); let next_token2 = Token::new(token::Ident(symbol2, false), ident2_span); diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 70323798a342f..ebe10a24a7845 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -286,7 +286,7 @@ impl TokenCursor { // No close delimiter to return; continue on to the next iteration. } else { // We have exhausted the outermost token stream. - return (Token::new(token::Eof, DUMMY_SP), FollowedBy::Space); + return (Token::new(token::Eof, DUMMY_SP), FollowedBy::Other); } } } diff --git a/tests/ui/proc-macro/auxiliary/expand-expr.rs b/tests/ui/proc-macro/auxiliary/expand-expr.rs index edd3bb58e7b42..e7bd1353bc150 100644 --- a/tests/ui/proc-macro/auxiliary/expand-expr.rs +++ b/tests/ui/proc-macro/auxiliary/expand-expr.rs @@ -99,21 +99,12 @@ pub fn expand_expr_is(input: TokenStream) -> TokenStream { let expected = expected_tts.into_iter().collect::(); let expanded = iter.collect::().expand_expr().expect("expand_expr failed"); - // FIXME: Temporarily broken. The next commit will fix it. - // - // + LL | simple_lit!(-3.14159); - // + | --------------------- in this macro invocation - // + | - // + = help: message: assert failed - // + expected: `-3.14159` - // + expanded: `- 3.14159` - // - // assert!( - // expected.to_string() == expanded.to_string(), - // "assert failed\nexpected: `{}`\nexpanded: `{}`", - // expected.to_string(), - // expanded.to_string() - // ); + assert!( + expected.to_string() == expanded.to_string(), + "assert failed\nexpected: `{}`\nexpanded: `{}`", + expected.to_string(), + expanded.to_string() + ); // Also compare the raw tts to make sure they line up. assert_ts_eq(&expected, &expanded); diff --git a/tests/ui/proc-macro/cfg-eval-inner.stdout b/tests/ui/proc-macro/cfg-eval-inner.stdout index 1eba65ca73bba..e297783286f38 100644 --- a/tests/ui/proc-macro/cfg-eval-inner.stdout +++ b/tests/ui/proc-macro/cfg-eval-inner.stdout @@ -1,15 +1,15 @@ PRINT-ATTR INPUT (DISPLAY): impl Foo<[u8; { - #! [rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner - { field: [u8; { #! [rustc_dummy(another_cursed_inner)] 1 }] } 0 -}] > { #! [rustc_dummy(evaluated_attr)] fn bar() {} } + #![rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner + { field: [u8; { #![rustc_dummy(another_cursed_inner)] 1 }] } 0 +}] > { #![rustc_dummy(evaluated_attr)] fn bar() {} } PRINT-ATTR RE-COLLECTED (DISPLAY): impl Foo < [u8; { - #! [rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner - { field: [u8; { #! [rustc_dummy(another_cursed_inner)] 1 }] } 0 -}] > { #! [rustc_dummy(evaluated_attr)] fn bar() {} } + #![rustc_dummy(cursed_inner)] #![allow(unused)] struct Inner + { field: [u8; { #![rustc_dummy(another_cursed_inner)] 1 }] } 0 +}] > { #![rustc_dummy(evaluated_attr)] fn bar() {} } PRINT-ATTR DEEP-RE-COLLECTED (DISPLAY): impl Foo < [u8 ; { @@ -47,7 +47,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ stream: TokenStream [ Punct { ch: '#', - spacing: Alone, + spacing: Joint, span: $DIR/cfg-eval-inner.rs:19:5: 19:6 (#0), }, Punct { @@ -142,7 +142,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ stream: TokenStream [ Punct { ch: '#', - spacing: Alone, + spacing: Joint, span: $DIR/cfg-eval-inner.rs:23:13: 23:14 (#0), }, Punct { @@ -207,7 +207,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ stream: TokenStream [ Punct { ch: '#', - spacing: Alone, + spacing: Joint, span: $DIR/cfg-eval-inner.rs:32:5: 32:6 (#0), }, Punct { diff --git a/tests/ui/proc-macro/inner-attrs.stdout b/tests/ui/proc-macro/inner-attrs.stdout index c5cf37343497c..7c10388a9dcbf 100644 --- a/tests/ui/proc-macro/inner-attrs.stdout +++ b/tests/ui/proc-macro/inner-attrs.stdout @@ -579,7 +579,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [ }, ] PRINT-DERIVE INPUT (DISPLAY): struct MyDerivePrint -{ field: [u8; { match true { _ => { #! [rustc_dummy(third)] true } } ; 0 }] } +{ field: [u8; { match true { _ => { #![rustc_dummy(third)] true } } ; 0 }] } PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): struct MyDerivePrint { field : @@ -651,7 +651,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [ stream: TokenStream [ Punct { ch: '#', - spacing: Alone, + spacing: Joint, span: $DIR/inner-attrs.rs:40:17: 40:18 (#0), }, Punct { diff --git a/tests/ui/proc-macro/issue-78675-captured-inner-attrs.stdout b/tests/ui/proc-macro/issue-78675-captured-inner-attrs.stdout index ae5e94008094c..37ecf3a8df331 100644 --- a/tests/ui/proc-macro/issue-78675-captured-inner-attrs.stdout +++ b/tests/ui/proc-macro/issue-78675-captured-inner-attrs.stdout @@ -46,7 +46,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [ stream: TokenStream [ Punct { ch: '#', - spacing: Alone, + spacing: Joint, span: $DIR/issue-78675-captured-inner-attrs.rs:28:9: 28:16 (#0), }, Punct { diff --git a/tests/ui/proc-macro/macro-rules-derive-cfg.stdout b/tests/ui/proc-macro/macro-rules-derive-cfg.stdout index aee0f966d0fce..60beef1e33a40 100644 --- a/tests/ui/proc-macro/macro-rules-derive-cfg.stdout +++ b/tests/ui/proc-macro/macro-rules-derive-cfg.stdout @@ -1,4 +1,13 @@ PRINT-DERIVE INPUT (DISPLAY): struct Foo +{ + val : + [bool ; + { + let a = #[rustc_dummy(first)] #[rustc_dummy(second)] + { #![allow(unused)] 30 } ; 0 + }] +} +PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): struct Foo { val : [bool ; @@ -111,7 +120,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [ stream: TokenStream [ Punct { ch: '#', - spacing: Alone, + spacing: Joint, span: $DIR/macro-rules-derive-cfg.rs:27:5: 27:6 (#0), }, Punct {