Auto merge of #57367 - petrochenkov:unrestab, r=Centril

Stabilize `unrestricted_attribute_tokens`

In accordance with a plan described in https://internals.rust-lang.org/t/unrestricted-attribute-tokens-feature-status/8561/3.

Delimited non-macro non-builtin attributes now support the same syntax as macro attributes:
```
PATH
PATH `(` TOKEN_STREAM `)`
PATH `[` TOKEN_STREAM `]`
PATH `{` TOKEN_STREAM `}`
```
Such attributes mostly serve as inert proc macro helpers or tool attributes.
To some extent these attributes are de-facto stable due to a hole in feature gate checking (feature gating is done too late - after macro expansion.)
So if macro *removes* such helper attributes during expansion (and it must remove them, unless it's a derive macro), then the code will work on stable.

Key-value non-macro non-builtin attributes are now restricted to bare minimum required to support what we support on stable - unsuffixed literals (#34981).
```
PATH `=` LITERAL
```
(Key-value macro attributes are not supported at all right now.)
Crater run in #57321 found no regressions for this change.
There are multiple possible ways to extend key-value attributes (#57321 (comment)), but I'd expect an RFC for that and it's not a pressing enough issue to block stabilization of delimited attributes.

Built-in attributes are still restricted to the "classic" meta-item syntax, nothing changes here.
#57321 goes further and adds some additional restrictions (more consistent input checking) to built-in attributes.

Closes #55208

src/libcore/lib.rs

@@ -123,7 +123,6 @@
 #![feature(abi_unadjusted)]
 #![feature(adx_target_feature)]
 #![feature(maybe_uninit, maybe_uninit_slice, maybe_uninit_array)]
-#![feature(unrestricted_attribute_tokens)]
 #![feature(external_doc)]
 
 #[prelude_import]

src/libsyntax/feature_gate.rs

@@ -21,8 +21,9 @@ use crate::early_buffered_lints::BufferedEarlyLintId;
 use crate::source_map::Spanned;
 use crate::edition::{ALL_EDITIONS, Edition};
 use crate::visit::{self, FnKind, Visitor};
-use crate::parse::ParseSess;
+use crate::parse::{token, ParseSess};
 use crate::symbol::Symbol;
+use crate::tokenstream::TokenTree;
 
 use errors::{DiagnosticBuilder, Handler};
 use rustc_data_structures::fx::FxHashMap;
@@ -431,9 +432,6 @@ declare_features! (
     // Added for testing E0705; perma-unstable.
     (active, test_2018_feature, "1.31.0", Some(0), Some(Edition::Edition2018)),
 
-    // support for arbitrary delimited token streams in non-macro attributes
-    (active, unrestricted_attribute_tokens, "1.30.0", Some(55208), None),
-
     // Allows unsized rvalues at arguments and parameters.
     (active, unsized_locals, "1.30.0", Some(48055), None),
 
@@ -700,6 +698,8 @@ declare_features! (
     (accepted, cfg_target_vendor, "1.33.0", Some(29718), None),
     // `extern crate self as foo;` puts local crate root into extern prelude under name `foo`.
     (accepted, extern_crate_self, "1.34.0", Some(56409), None),
+    // support for arbitrary delimited token streams in non-macro attributes
+    (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208), None),
 );
 
 // If you change this, please modify `src/doc/unstable-book` as well. You must
@@ -1660,13 +1660,9 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
         match BUILTIN_ATTRIBUTES.iter().find(|(name, ..)| attr.path == name) {
             Some(&(name, _, template, _)) => self.check_builtin_attribute(attr, name, template),
-            None => if !self.context.features.unrestricted_attribute_tokens {
-                // Unfortunately, `parse_meta` cannot be called speculatively
-                // because it can report errors by itself, so we have to call it
-                // only if the feature is disabled.
-                if let Err(mut err) = attr.parse_meta(self.context.parse_sess) {
-                    err.help("try enabling `#![feature(unrestricted_attribute_tokens)]`").emit()
-                }
+            None => if let Some(TokenTree::Token(_, token::Eq)) = attr.tokens.trees().next() {
+                // All key-value attributes are restricted to meta-item syntax.
+                attr.parse_meta(self.context.parse_sess).map_err(|mut err| err.emit()).ok();
             }
         }
     }

src/libsyntax/parse/attr.rs

@@ -158,11 +158,21 @@ impl<'a> Parser<'a> {
                    self.parse_token_tree().into()
             } else if self.eat(&token::Eq) {
                 let eq = TokenTree::Token(self.prev_span, token::Eq);
-                let tree = match self.token {
-                    token::CloseDelim(_) | token::Eof => self.unexpected()?,
-                    _ => self.parse_token_tree(),
+                let mut is_interpolated_expr = false;
+                if let token::Interpolated(nt) = &self.token {
+                    if let token::NtExpr(..) = **nt {
+                        is_interpolated_expr = true;
+                    }
+                }
+                let tokens = if is_interpolated_expr {
+                    // We need to accept arbitrary interpolated expressions to continue
+                    // supporting things like `doc = $expr` that work on stable.
+                    // Non-literal interpolated expressions are rejected after expansion.
+                    self.parse_token_tree().into()
+                } else {
+                    self.parse_unsuffixed_lit()?.tokens()
                 };
-                TokenStream::new(vec![eq.into(), tree.into()])
+                TokenStream::from_streams(vec![eq.into(), tokens])
             } else {
                 TokenStream::empty()
             };

src/libsyntax/tokenstream.rs

@@ -254,7 +254,7 @@ impl TokenStream {
         }
     }
 
-    fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream {
+    pub(crate) fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
             1 => streams.pop().unwrap(),

src/test/run-pass/proc-macro/derive-b.rs

@@ -1,7 +1,5 @@
 // aux-build:derive-b.rs
 
-#![feature(unrestricted_attribute_tokens)]
-
 extern crate derive_b;
 
 #[derive(Debug, PartialEq, derive_b::B, Eq, Copy, Clone)]

src/test/ui/attr-eq-token-tree.rs

@@ -1,6 +1,4 @@
-// compile-pass
+#![feature(custom_attribute)]
 
-#![feature(custom_attribute, unrestricted_attribute_tokens)]
-
-#[my_attr = !] // OK under feature gate
+#[my_attr = !] //~ ERROR unexpected token: `!`
 fn main() {}

src/test/ui/attr-eq-token-tree.stderr

@@ -0,0 +1,8 @@
+error: unexpected token: `!`
+  --> $DIR/attr-eq-token-tree.rs:3:11
+   |
+LL | #[my_attr = !] //~ ERROR unexpected token: `!`
+   |           ^
+
+error: aborting due to previous error
+

src/test/ui/macros/macro-attribute.rs

@@ -1,4 +1,2 @@
-#![feature(unrestricted_attribute_tokens)]
-
-#[doc = $not_there] //~ ERROR expected `]`, found `not_there`
+#[doc = $not_there] //~ ERROR unexpected token: `$`
 fn main() { }

src/test/ui/macros/macro-attribute.stderr

@@ -1,8 +1,8 @@
-error: expected `]`, found `not_there`
-  --> $DIR/macro-attribute.rs:3:10
+error: unexpected token: `$`
+  --> $DIR/macro-attribute.rs:1:7
    |
-LL | #[doc = $not_there] //~ ERROR expected `]`, found `not_there`
-   |          ^^^^^^^^^ expected `]`
+LL | #[doc = $not_there] //~ ERROR unexpected token: `$`
+   |       ^
 
 error: aborting due to previous error
 

src/test/ui/malformed/malformed-interpolated.rs

@@ -0,0 +1,18 @@
+#![feature(custom_attribute)]
+
+macro_rules! check {
+    ($expr: expr) => (
+        #[my_attr = $expr] //~ ERROR suffixed literals are not allowed in attributes
+                           //~| ERROR unexpected token: `-0`
+                           //~| ERROR unexpected token: `0 + 0`
+        use main as _;
+    );
+}
+
+check!("0"); // OK
+check!(0); // OK
+check!(0u8); // ERROR, see above
+check!(-0); // ERROR, see above
+check!(0 + 0); // ERROR, see above
+
+fn main() {}

src/test/ui/malformed/malformed-interpolated.stderr

@@ -0,0 +1,31 @@
+error: suffixed literals are not allowed in attributes
+  --> $DIR/malformed-interpolated.rs:5:21
+   |
+LL |         #[my_attr = $expr] //~ ERROR suffixed literals are not allowed in attributes
+   |                     ^^^^^
+...
+LL | check!(0u8); // ERROR, see above
+   | ------------ in this macro invocation
+   |
+   = help: instead of using a suffixed literal (1u8, 1.0f32, etc.), use an unsuffixed version (1, 1.0, etc.).
+
+error: unexpected token: `-0`
+  --> $DIR/malformed-interpolated.rs:5:19
+   |
+LL |         #[my_attr = $expr] //~ ERROR suffixed literals are not allowed in attributes
+   |                   ^
+...
+LL | check!(-0); // ERROR, see above
+   | ----------- in this macro invocation
+
+error: unexpected token: `0 + 0`
+  --> $DIR/malformed-interpolated.rs:5:19
+   |
+LL |         #[my_attr = $expr] //~ ERROR suffixed literals are not allowed in attributes
+   |                   ^
+...
+LL | check!(0 + 0); // ERROR, see above
+   | -------------- in this macro invocation
+
+error: aborting due to 3 previous errors
+

src/test/ui/marker_trait_attr/marker-attribute-with-values.rs

@@ -1,5 +1,4 @@
 #![feature(marker_trait_attr)]
-#![feature(unrestricted_attribute_tokens)]
 
 #[marker(always)]
 trait Marker1 {}
@@ -9,8 +8,8 @@ trait Marker1 {}
 trait Marker2 {}
 //~^^ ERROR attribute must be of the form
 
-#[marker(key = value)]
+#[marker(key = "value")]
 trait Marker3 {}
-//~^^ ERROR expected unsuffixed literal or identifier, found value
+//~^^ ERROR attribute must be of the form `#[marker]`
 
 fn main() {}

src/test/ui/marker_trait_attr/marker-attribute-with-values.stderr

@@ -1,20 +1,20 @@
 error: attribute must be of the form `#[marker]`
-  --> $DIR/marker-attribute-with-values.rs:4:1
+  --> $DIR/marker-attribute-with-values.rs:3:1
    |
 LL | #[marker(always)]
    | ^^^^^^^^^^^^^^^^^
 
 error: attribute must be of the form `#[marker]`
-  --> $DIR/marker-attribute-with-values.rs:8:1
+  --> $DIR/marker-attribute-with-values.rs:7:1
    |
 LL | #[marker("never")]
    | ^^^^^^^^^^^^^^^^^^
 
-error: expected unsuffixed literal or identifier, found value
-  --> $DIR/marker-attribute-with-values.rs:12:10
+error: attribute must be of the form `#[marker]`
+  --> $DIR/marker-attribute-with-values.rs:11:1
    |
-LL | #[marker(key = value)]
-   |          ^^^
+LL | #[marker(key = "value")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: aborting due to 3 previous errors
 

src/test/ui/parser/attr-bad-meta-2.stderr

@@ -1,8 +1,8 @@
 error: unexpected token: `]`
-  --> $DIR/attr-bad-meta-2.rs:1:9
+  --> $DIR/attr-bad-meta-2.rs:1:8
    |
 LL | #[path =] //~ ERROR unexpected token: `]`
-   |         ^ unexpected token after this
+   |        ^
 
 error: aborting due to previous error
 

src/test/ui/parser/attr-bad-meta.rs

@@ -1,4 +1,2 @@
-#![feature(unrestricted_attribute_tokens)]
-
 #[path*] //~ ERROR expected one of `(`, `::`, `=`, `[`, `]`, or `{`, found `*`
 mod m {}

src/test/ui/parser/attr-bad-meta.stderr

@@ -1,5 +1,5 @@
 error: expected one of `(`, `::`, `=`, `[`, `]`, or `{`, found `*`
-  --> $DIR/attr-bad-meta.rs:3:7
+  --> $DIR/attr-bad-meta.rs:1:7
    |
 LL | #[path*] //~ ERROR expected one of `(`, `::`, `=`, `[`, `]`, or `{`, found `*`
    |       ^ expected one of `(`, `::`, `=`, `[`, `]`, or `{` here

src/test/ui/proc-macro/proc-macro-attributes.rs

@@ -8,7 +8,6 @@ extern crate derive_b;
 #[B(D)] //~ ERROR `B` is ambiguous
 #[B(E = "foo")] //~ ERROR `B` is ambiguous
 #[B(arbitrary tokens)] //~ ERROR `B` is ambiguous
-                       //~^ ERROR expected one of `(`, `)`, `,`, `::`, or `=`, found `tokens`
 #[derive(B)]
 struct B;
 

src/test/ui/proc-macro/proc-macro-attributes.stderr

@@ -13,7 +13,7 @@ LL | #[B] //~ ERROR `B` is ambiguous
    |   ^ ambiguous name
    |
 note: `B` could refer to the derive helper attribute defined here
-  --> $DIR/proc-macro-attributes.rs:12:10
+  --> $DIR/proc-macro-attributes.rs:11:10
    |
 LL | #[derive(B)]
    |          ^
@@ -30,7 +30,7 @@ LL | #[B(D)] //~ ERROR `B` is ambiguous
    |   ^ ambiguous name
    |
 note: `B` could refer to the derive helper attribute defined here
-  --> $DIR/proc-macro-attributes.rs:12:10
+  --> $DIR/proc-macro-attributes.rs:11:10
    |
 LL | #[derive(B)]
    |          ^
@@ -47,7 +47,7 @@ LL | #[B(E = "foo")] //~ ERROR `B` is ambiguous
    |   ^ ambiguous name
    |
 note: `B` could refer to the derive helper attribute defined here
-  --> $DIR/proc-macro-attributes.rs:12:10
+  --> $DIR/proc-macro-attributes.rs:11:10
    |
 LL | #[derive(B)]
    |          ^
@@ -64,7 +64,7 @@ LL | #[B(arbitrary tokens)] //~ ERROR `B` is ambiguous
    |   ^ ambiguous name
    |
 note: `B` could refer to the derive helper attribute defined here
-  --> $DIR/proc-macro-attributes.rs:12:10
+  --> $DIR/proc-macro-attributes.rs:11:10
    |
 LL | #[derive(B)]
    |          ^
@@ -74,13 +74,7 @@ note: `B` could also refer to the derive macro imported here
 LL | #[macro_use]
    | ^^^^^^^^^^^^
 
-error: expected one of `(`, `)`, `,`, `::`, or `=`, found `tokens`
-  --> $DIR/proc-macro-attributes.rs:10:15
-   |
-LL | #[B(arbitrary tokens)] //~ ERROR `B` is ambiguous
-   |               ^^^^^^ expected one of `(`, `)`, `,`, `::`, or `=` here
-
-error: aborting due to 6 previous errors
+error: aborting due to 5 previous errors
 
 Some errors occurred: E0658, E0659.
 For more information about an error, try `rustc --explain E0658`.

src/test/ui/proc-macro/proc-macro-gates.rs

@@ -19,7 +19,7 @@ mod _test2_inner {
           //~| ERROR: non-builtin inner attributes are unstable
 }
 
-#[a = y] //~ ERROR: must only be followed by a delimiter token
+#[a = "y"] //~ ERROR: must only be followed by a delimiter token
 fn _test3() {}
 
 fn attrs() {

src/test/ui/proc-macro/proc-macro-gates.stderr

@@ -33,8 +33,8 @@ LL |     #![a] //~ ERROR: custom attributes cannot be applied to modules
 error: custom attribute invocations must be of the form #[foo] or #[foo(..)], the macro name must only be followed by a delimiter token
   --> $DIR/proc-macro-gates.rs:22:1
    |
-LL | #[a = y] //~ ERROR: must only be followed by a delimiter token
-   | ^^^^^^^^
+LL | #[a = "y"] //~ ERROR: must only be followed by a delimiter token
+   | ^^^^^^^^^^
 
 error[E0658]: custom attributes cannot be applied to statements (see issue #54727)
   --> $DIR/proc-macro-gates.rs:31:5

src/test/ui/unrestricted-attribute-tokens.rs

@@ -1,6 +1,8 @@
 // compile-pass
 
-#![feature(custom_attribute, unrestricted_attribute_tokens)]
+#![feature(custom_attribute)]
 
 #[my_attr(a b c d)]
+#[my_attr[a b c d]]
+#[my_attr{a b c d}]
 fn main() {}