Accept arbitrary expressions in key-value attributes at parse time

compiler/rustc_ast/src/ast.rs

@@ -24,7 +24,7 @@ pub use UnsafeSource::*;
 
 use crate::ptr::P;
 use crate::token::{self, CommentKind, DelimToken};
-use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree};
+use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -39,7 +39,6 @@ use rustc_span::{Span, DUMMY_SP};
 use std::cmp::Ordering;
 use std::convert::TryFrom;
 use std::fmt;
-use std::iter;
 
 #[cfg(test)]
 mod tests;
@@ -1514,20 +1513,6 @@ impl MacArgs {
         }
     }
 
-    /// Tokens together with the delimiters or `=`.
-    /// Use of this method generally means that something suboptimal or hacky is happening.
-    pub fn outer_tokens(&self) -> TokenStream {
-        match *self {
-            MacArgs::Empty => TokenStream::default(),
-            MacArgs::Delimited(dspan, delim, ref tokens) => {
-                TokenTree::Delimited(dspan, delim.to_token(), tokens.clone()).into()
-            }
-            MacArgs::Eq(eq_span, ref tokens) => {
-                iter::once(TokenTree::token(token::Eq, eq_span)).chain(tokens.trees()).collect()
-            }
-        }
-    }
-
     /// Whether a macro with these arguments needs a semicolon
     /// when used as a standalone item or statement.
     pub fn need_semicolon(&self) -> bool {

compiler/rustc_ast/src/mut_visit.rs

@@ -371,20 +371,15 @@ pub fn visit_mac_args<T: MutVisitor>(args: &mut MacArgs, vis: &mut T) {
             // The value in `#[key = VALUE]` must be visited as an expression for backward
             // compatibility, so that macros can be expanded in that position.
             if !vis.token_visiting_enabled() {
-                if let Some(TokenTree::Token(token)) = tokens.trees_ref().next() {
-                    if let token::Interpolated(..) = token.kind {
-                        // ^^ Do not `make_mut` unless we have to.
-                        match Lrc::make_mut(&mut tokens.0).get_mut(0) {
-                            Some((TokenTree::Token(token), _spacing)) => match &mut token.kind {
-                                token::Interpolated(nt) => match Lrc::make_mut(nt) {
-                                    token::NtExpr(expr) => vis.visit_expr(expr),
-                                    t => panic!("unexpected token in key-value attribute: {:?}", t),
-                                },
-                                t => panic!("unexpected token in key-value attribute: {:?}", t),
-                            },
+                match Lrc::make_mut(&mut tokens.0).get_mut(0) {
+                    Some((TokenTree::Token(token), _spacing)) => match &mut token.kind {
+                        token::Interpolated(nt) => match Lrc::make_mut(nt) {
+                            token::NtExpr(expr) => vis.visit_expr(expr),
                             t => panic!("unexpected token in key-value attribute: {:?}", t),
-                        }
-                    }
+                        },
+                        t => panic!("unexpected token in key-value attribute: {:?}", t),
+                    },
+                    t => panic!("unexpected token in key-value attribute: {:?}", t),
                 }
             }
         }

compiler/rustc_ast/src/visit.rs

@@ -906,7 +906,6 @@ pub fn walk_mac_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a MacArgs) {
                     token::NtExpr(expr) => visitor.visit_expr(expr),
                     t => panic!("unexpected token in key-value attribute: {:?}", t),
                 },
-                token::Literal(..) | token::Ident(..) => {}
                 t => panic!("unexpected token in key-value attribute: {:?}", t),
             },
             t => panic!("unexpected token in key-value attribute: {:?}", t),

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -630,6 +630,10 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
     gate_all!(const_trait_impl, "const trait impls are experimental");
     gate_all!(half_open_range_patterns, "half-open range patterns are unstable");
     gate_all!(inline_const, "inline-const is experimental");
+    gate_all!(
+        extended_key_value_attributes,
+        "arbitrary expressions in key-value attributes are unstable"
+    );
     if sess.parse_sess.span_diagnostic.err_count() == 0 {
         // Errors for `destructuring_assignment` can get quite noisy, especially where `_` is
         // involved, so we only emit errors where there are no other parsing errors.

compiler/rustc_feature/src/active.rs

@@ -620,6 +620,9 @@ declare_features! (
     /// Allows capturing disjoint fields in a closure/generator (RFC 2229).
     (active, capture_disjoint_fields, "1.49.0", Some(53488), None),
 
+    /// Allows arbitrary expressions in key-value attributes at parse time.
+    (active, extended_key_value_attributes, "1.50.0", Some(78835), None),
+
     // -------------------------------------------------------------------------
     // feature-group-end: actual feature gates
     // -------------------------------------------------------------------------

compiler/rustc_parse/src/parser/mod.rs

@@ -23,6 +23,7 @@ use rustc_ast::{self as ast, AnonConst, AttrStyle, AttrVec, Const, CrateSugar, E
 use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacDelimiter, Mutability, StrLit};
 use rustc_ast::{Visibility, VisibilityKind};
 use rustc_ast_pretty::pprust;
+use rustc_data_structures::sync::Lrc;
 use rustc_errors::PResult;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError};
 use rustc_session::parse::ParseSess;
@@ -935,16 +936,24 @@ impl<'a> Parser<'a> {
                             is_interpolated_expr = true;
                         }
                     }
-                    let token_tree = if is_interpolated_expr {
-                        // We need to accept arbitrary interpolated expressions to continue
-                        // supporting things like `doc = $expr` that work on stable.
-                        // Non-literal interpolated expressions are rejected after expansion.
-                        self.parse_token_tree()
-                    } else {
-                        self.parse_unsuffixed_lit()?.token_tree()
-                    };
 
-                    MacArgs::Eq(eq_span, token_tree.into())
+                    // The value here is never passed to macros as tokens by itself (not as a part
+                    // of the whole attribute), so we don't collect tokens here. If this changes,
+                    // then token will need to be collected. One catch here is that we are using
+                    // a nonterminal for keeping the expression, but this nonterminal should not
+                    // be wrapped into a group when converting to token stream.
+                    let expr = self.parse_expr()?;
+                    let span = expr.span;
+
+                    match &expr.kind {
+                        // Not gated to supporte things like `doc = $expr` that work on stable.
+                        _ if is_interpolated_expr => {}
+                        ExprKind::Lit(lit) if lit.kind.is_unsuffixed() => {}
+                        _ => self.sess.gated_spans.gate(sym::extended_key_value_attributes, span),
+                    }
+
+                    let token = token::Interpolated(Lrc::new(token::NtExpr(expr)));
+                    MacArgs::Eq(eq_span, TokenTree::token(token, span).into())
                 } else {
                     MacArgs::Empty
                 }

compiler/rustc_span/src/symbol.rs

@@ -496,6 +496,7 @@ symbols! {
         expf64,
         export_name,
         expr,
+        extended_key_value_attributes,
         extern_absolute_paths,
         extern_crate_item_prelude,
         extern_crate_self,

src/test/rustdoc/external-doc.rs

@@ -1,8 +1,24 @@
 #![feature(external_doc)]
+#![feature(extended_key_value_attributes)]
 
 // @has external_doc/struct.CanHasDocs.html
 // @has - '//h1' 'External Docs'
 // @has - '//h2' 'Inline Docs'
 #[doc(include = "auxiliary/external-doc.md")]
 /// ## Inline Docs
 pub struct CanHasDocs;
+
+// @has external_doc/struct.IncludeStrDocs.html
+// @has - '//h1' 'External Docs'
+// @has - '//h2' 'Inline Docs'
+#[doc = include_str!("auxiliary/external-doc.md")]
+/// ## Inline Docs
+pub struct IncludeStrDocs;
+
+macro_rules! dir { () => { "auxiliary" } }
+
+// @has external_doc/struct.EagerExpansion.html
+// @has - '//h1' 'External Docs'
+#[doc = include_str!(concat!(dir!(), "/external-doc.md"))]
+/// ## Inline Docs
+pub struct EagerExpansion;

src/test/ui/ast-json/ast-json-noexpand-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Interpolated","fields":[{"variant":"NtExpr","fields":[{"id":0,"kind":{"variant":"Lit","fields":[{"token":{"kind":"Str","symbol":"lib","suffix":null},"kind":{"variant":"Str","fields":["lib","Cooked"]},"span":{"lo":0,"hi":0}}]},"span":{"lo":0,"hi":0},"attrs":{"0":null},"tokens":null}]}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

src/test/ui/ast-json/ast-json-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Interpolated","fields":[{"variant":"NtExpr","fields":[{"id":0,"kind":{"variant":"Lit","fields":[{"token":{"kind":"Str","symbol":"lib","suffix":null},"kind":{"variant":"Str","fields":["lib","Cooked"]},"span":{"lo":0,"hi":0}}]},"span":{"lo":0,"hi":0},"attrs":{"0":null},"tokens":null}]}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

src/test/ui/attr-eq-token-tree.rs

@@ -1,2 +1,2 @@
-#[my_attr = !] //~ ERROR unexpected token: `!`
+#[my_attr = !] //~ ERROR expected expression, found `]`
 fn main() {}

src/test/ui/attr-eq-token-tree.stderr

@@ -1,8 +1,8 @@
-error: unexpected token: `!`
-  --> $DIR/attr-eq-token-tree.rs:1:13
+error: expected expression, found `]`
+  --> $DIR/attr-eq-token-tree.rs:1:14
    |
 LL | #[my_attr = !]
-   |             ^
+   |              ^ expected expression
 
 error: aborting due to previous error
 

src/test/ui/attributes/key-value-expansion.rs

@@ -12,23 +12,22 @@ extern crate key_value_expansion;
 macro_rules! bug {
     ($expr:expr) => {
         #[rustc_dummy = $expr] // Any key-value attribute, not necessarily `doc`
-        //~^ ERROR unexpected token: `(7u32)`
         struct S;
     };
 }
 
 // Any expressions containing macro call `X` that's more complex than `X` itself.
 // Parentheses will work.
-bug!((column!()));
+bug!((column!())); //~ ERROR unexpected token: `(7u32)`
 
 // Original test case.
 
 macro_rules! bug {
     () => {
-        bug!("bug" + stringify!(found));
+        bug!("bug" + stringify!(found)); //~ ERROR unexpected token: `"bug" + "found"`
     };
     ($test:expr) => {
-        #[doc = $test] //~ ERROR unexpected token: `"bug" + "found"`
+        #[doc = $test]
         struct Test {}
     };
 }
@@ -39,14 +38,15 @@ bug!();
 
 macro_rules! doc_comment {
     ($x:expr) => {
-        #[doc = $x] //~ ERROR unexpected token: `{
+        #[doc = $x]
         extern {}
     };
 }
 
 macro_rules! some_macro {
     ($t1: ty) => {
         doc_comment! {format!("{coor}", coor = stringify!($t1)).as_str()}
+        //~^ ERROR unexpected token: `{
     };
 }
 

src/test/ui/attributes/key-value-expansion.stderr

@@ -1,19 +1,14 @@
 error: unexpected token: `(7u32)`
-  --> $DIR/key-value-expansion.rs:14:25
+  --> $DIR/key-value-expansion.rs:21:6
    |
-LL |         #[rustc_dummy = $expr] // Any key-value attribute, not necessarily `doc`
-   |                         ^^^^^
-...
 LL | bug!((column!()));
-   | ------------------ in this macro invocation
-   |
-   = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
+   |      ^^^^^^^^^^^
 
 error: unexpected token: `"bug" + "found"`
-  --> $DIR/key-value-expansion.rs:31:17
+  --> $DIR/key-value-expansion.rs:27:14
    |
-LL |         #[doc = $test]
-   |                 ^^^^^
+LL |         bug!("bug" + stringify!(found));
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^
 ...
 LL | bug!();
    | ------- in this macro invocation
@@ -30,10 +25,10 @@ error: unexpected token: `{
                                                              }));
     res
 }.as_str()`
-  --> $DIR/key-value-expansion.rs:42:17
+  --> $DIR/key-value-expansion.rs:48:23
    |
-LL |         #[doc = $x]
-   |                 ^^
+LL |         doc_comment! {format!("{coor}", coor = stringify!($t1)).as_str()}
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 ...
 LL | some_macro!(u8);
    | ---------------- in this macro invocation