Auto merge of #53293 - petrochenkov:gramattr2, r=alexcrichton

syntax: Enforce attribute grammar in the parser

Also fix feature-gating for `unrestricted_attribute_tokens` that was introduced in #53270, but was actually broken.

cc #50911

Author: bors

src/libsyntax/attr/mod.rs

@@ -607,7 +607,7 @@ impl NestedMetaItemKind {
 }
 
 impl Lit {
-    fn tokens(&self) -> TokenStream {
+    crate fn tokens(&self) -> TokenStream {
         TokenTree::Token(self.span, self.node.token()).into()
     }
 }
@@ -794,7 +794,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.span;
-        let (path, tokens) = panictry!(parser.parse_path_and_tokens());
+        let (path, tokens) = panictry!(parser.parse_meta_item_unrestricted());
         let end_span = parser.span;
         if parser.token != token::Eof {
             parse_sess.span_diagnostic

src/libsyntax/config.rs

@@ -90,7 +90,7 @@ impl<'a> StripUnconfigured<'a> {
             let cfg = parser.parse_meta_item()?;
             parser.expect(&token::Comma)?;
             let lo = parser.span.lo();
-            let (path, tokens) = parser.parse_path_and_tokens()?;
+            let (path, tokens) = parser.parse_meta_item_unrestricted()?;
             parser.expect(&token::CloseDelim(token::Paren))?;
             Ok((cfg, path, tokens, parser.prev_span.with_lo(lo)))
         }) {

src/libsyntax/feature_gate.rs

@@ -1526,27 +1526,29 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
         }
 
-        match attr.parse_meta(self.context.parse_sess) {
-            Ok(meta) => {
-                // allow attr_literals in #[repr(align(x))] and #[repr(packed(n))]
-                let mut allow_attr_literal = false;
-                if attr.path == "repr" {
-                    if let Some(content) = meta.meta_item_list() {
-                        allow_attr_literal = content.iter().any(
-                            |c| c.check_name("align") || c.check_name("packed"));
+        if !self.context.features.unrestricted_attribute_tokens {
+            // Unfortunately, `parse_meta` cannot be called speculatively because it can report
+            // errors by itself, so we have to call it only if the feature is disabled.
+            match attr.parse_meta(self.context.parse_sess) {
+                Ok(meta) => {
+                    // allow attr_literals in #[repr(align(x))] and #[repr(packed(n))]
+                    let mut allow_attr_literal = false;
+                    if attr.path == "repr" {
+                        if let Some(content) = meta.meta_item_list() {
+                            allow_attr_literal = content.iter().any(
+                                |c| c.check_name("align") || c.check_name("packed"));
+                        }
                     }
-                }
 
-                if !allow_attr_literal && contains_novel_literal(&meta) {
-                    gate_feature_post!(&self, attr_literals, attr.span,
-                                    "non-string literals in attributes, or string \
-                                    literals in top-level positions, are experimental");
+                    if !allow_attr_literal && contains_novel_literal(&meta) {
+                        gate_feature_post!(&self, attr_literals, attr.span,
+                                        "non-string literals in attributes, or string \
+                                        literals in top-level positions, are experimental");
+                    }
+                }
+                Err(mut err) => {
+                    err.help("try enabling `#![feature(unrestricted_attribute_tokens)]`").emit()
                 }
-            }
-            Err(mut err) => {
-                err.cancel();
-                gate_feature_post!(&self, unrestricted_attribute_tokens, attr.span,
-                                    "arbitrary tokens in non-macro attributes are unstable");
             }
         }
     }

src/libsyntax/parse/attr.rs

@@ -12,9 +12,9 @@ use attr;
 use ast;
 use codemap::respan;
 use parse::{SeqSep, PResult};
-use parse::token::{self, Nonterminal};
+use parse::token::{self, Nonterminal, DelimToken};
 use parse::parser::{Parser, TokenType, PathStyle};
-use tokenstream::TokenStream;
+use tokenstream::{TokenStream, TokenTree};
 
 #[derive(Debug)]
 enum InnerAttributeParsePolicy<'a> {
@@ -116,7 +116,7 @@ impl<'a> Parser<'a> {
                 };
 
                 self.expect(&token::OpenDelim(token::Bracket))?;
-                let (path, tokens) = self.parse_path_and_tokens()?;
+                let (path, tokens) = self.parse_meta_item_unrestricted()?;
                 self.expect(&token::CloseDelim(token::Bracket))?;
                 let hi = self.prev_span;
 
@@ -138,7 +138,16 @@ impl<'a> Parser<'a> {
         })
     }
 
-    crate fn parse_path_and_tokens(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
+    /// Parse an inner part of attribute - path and following tokens.
+    /// The tokens must be either a delimited token stream, or empty token stream,
+    /// or the "legacy" key-value form.
+    /// PATH `(` TOKEN_STREAM `)`
+    /// PATH `[` TOKEN_STREAM `]`
+    /// PATH `{` TOKEN_STREAM `}`
+    /// PATH
+    /// PATH `=` TOKEN_TREE
+    /// The delimiters or `=` are still put into the resulting token stream.
+    crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
         let meta = match self.token {
             token::Interpolated(ref nt) => match nt.0 {
                 Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
@@ -150,7 +159,22 @@ impl<'a> Parser<'a> {
             self.bump();
             (meta.ident, meta.node.tokens(meta.span))
         } else {
-            (self.parse_path(PathStyle::Mod)?, self.parse_tokens())
+            let path = self.parse_path(PathStyle::Mod)?;
+            let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+               self.check(&token::OpenDelim(DelimToken::Bracket)) ||
+               self.check(&token::OpenDelim(DelimToken::Brace)) {
+                   self.parse_token_tree().into()
+            } else if self.eat(&token::Eq) {
+                let eq = TokenTree::Token(self.prev_span, token::Eq);
+                let tree = match self.token {
+                    token::CloseDelim(_) | token::Eof => self.unexpected()?,
+                    _ => self.parse_token_tree(),
+                };
+                TokenStream::concat(vec![eq.into(), tree.into()])
+            } else {
+                TokenStream::empty()
+            };
+            (path, tokens)
         })
     }
 

src/libsyntax/parse/parser.rs

@@ -825,7 +825,7 @@ impl<'a> Parser<'a> {
     ///
     /// This method will automatically add `tok` to `expected_tokens` if `tok` is not
     /// encountered.
-    fn check(&mut self, tok: &token::Token) -> bool {
+    crate fn check(&mut self, tok: &token::Token) -> bool {
         let is_present = self.token == *tok;
         if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); }
         is_present

src/test/compile-fail-fulldeps/issue-48941.rs

@@ -17,10 +17,7 @@
 #![feature(plugin)]
 #![plugin(macro_crate_test)]
 
-#[noop_attribute"x"] //~ ERROR expected one of
-fn night() { }
-
-#[noop_attribute("hi"), rank = 2] //~ ERROR unexpected token
+#[noop_attribute("hi", rank = a)] //~ ERROR expected unsuffixed literal or identifier, found a
 fn knight() { }
 
 #[noop_attribute("/user", data= = "<user")] //~ ERROR literal or identifier

src/test/compile-fail-fulldeps/proc-macro/proc-macro-attributes.rs

@@ -21,7 +21,7 @@ extern crate derive_b;
 #[C] //~ ERROR: The attribute `C` is currently unknown to the compiler
 #[B(D)]
 #[B(E = "foo")]
-#[B arbitrary tokens] //~ ERROR arbitrary tokens in non-macro attributes are unstable
+#[B(arbitrary tokens)] //~ ERROR expected one of `(`, `)`, `,`, `::`, or `=`, found `tokens`
 struct B;
 
 fn main() {}

src/test/compile-fail-fulldeps/proc-macro/proc-macro-gates.rs

@@ -41,12 +41,6 @@ mod _test2_inner {
 #[a = y] //~ ERROR: must only be followed by a delimiter token
 fn _test3() {}
 
-#[a = ] //~ ERROR: must only be followed by a delimiter token
-fn _test4() {}
-
-#[a () = ] //~ ERROR: must only be followed by a delimiter token
-fn _test5() {}
-
 fn attrs() {
     // Statement, item
     #[a] // OK

src/test/parse-fail/attr-bad-meta-2.rs

@@ -0,0 +1,12 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[path =] //~ ERROR unexpected token: `]`
+mod m {}

src/test/parse-fail/attr-bad-meta-3.rs

@@ -0,0 +1,12 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[path() token] //~ ERROR expected `]`, found `token`
+mod m {}

src/test/parse-fail/attr-bad-meta.rs

@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-// asterisk is bogus
-#[path*] //~ ERROR arbitrary tokens in non-macro attributes are unstable
+#![feature(unrestricted_attribute_tokens)]
+
+#[path*] //~ ERROR expected one of `(`, `::`, `=`, `[`, `]`, or `{`, found `*`
 mod m {}

src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-b.rs

@@ -19,7 +19,7 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(B, attributes(B, C))]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert!(input.contains("#[B arbitrary tokens]"));
+    assert!(input.contains("#[B [ arbitrary tokens ]]"));
     assert!(input.contains("struct B {"));
     assert!(input.contains("#[C]"));
     "".parse().unwrap()

src/test/run-pass-fulldeps/proc-macro/derive-b.rs

@@ -16,7 +16,7 @@
 extern crate derive_b;
 
 #[derive(Debug, PartialEq, derive_b::B, Eq, Copy, Clone)]
-#[cfg_attr(all(), B arbitrary tokens)]
+#[cfg_attr(all(), B[arbitrary tokens])]
 struct B {
     #[C]
     a: u64

src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs

@@ -17,6 +17,6 @@ extern crate attribute_spans_preserved as foo;
 use foo::foo;
 
 #[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types
-#[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types
+#[ bar { let x: u32 = "y"; } ] //~ ERROR: mismatched types
 fn main() {
 }

src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr

@@ -8,10 +8,10 @@ LL | #[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types
               found type `&'static str`
 
 error[E0308]: mismatched types
-  --> $DIR/attribute-spans-preserved.rs:20:21
+  --> $DIR/attribute-spans-preserved.rs:20:23
    |
-LL | #[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types
-   |                     ^^^ expected u32, found reference
+LL | #[ bar { let x: u32 = "y"; } ] //~ ERROR: mismatched types
+   |                       ^^^ expected u32, found reference
    |
    = note: expected type `u32`
               found type `&'static str`

src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout

@@ -1 +1 @@
-fn main (  ) { let y : u32 = "z" ; let x : u32 = "y" ; }
+fn main (  ) { let y : u32 = "z" ; { let x : u32 = "y" ; } }

src/test/ui/attr-eq-token-tree.rs

@@ -0,0 +1,16 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-pass
+
+#![feature(custom_attribute, unrestricted_attribute_tokens)]
+
+#[my_attr = !] // OK under feature gate
+fn main() {}

src/test/ui/macros/macro-attribute.rs

@@ -8,5 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-#[doc = $not_there] //~ ERROR arbitrary tokens in non-macro attributes are unstable
+#![feature(unrestricted_attribute_tokens)]
+
+#[doc = $not_there] //~ ERROR expected `]`, found `not_there`
 fn main() { }

src/test/ui/macros/macro-attribute.stderr

@@ -1,11 +1,8 @@
-error[E0658]: arbitrary tokens in non-macro attributes are unstable (see issue #44690)
-  --> $DIR/macro-attribute.rs:11:1
+error: expected `]`, found `not_there`
+  --> $DIR/macro-attribute.rs:13:10
    |
-LL | #[doc = $not_there] //~ ERROR arbitrary tokens in non-macro attributes are unstable
-   | ^^^^^^^^^^^^^^^^^^^
-   |
-   = help: add #![feature(unrestricted_attribute_tokens)] to the crate attributes to enable
+LL | #[doc = $not_there] //~ ERROR expected `]`, found `not_there`
+   |          ^^^^^^^^^ expected `]`
 
 error: aborting due to previous error
 
-For more information about this error, try `rustc --explain E0658`.

src/test/ui/unrestricted-attribute-tokens.rs

@@ -0,0 +1,16 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-pass
+
+#![feature(custom_attribute, unrestricted_attribute_tokens)]
+
+#[my_attr(a b c d)]
+fn main() {}