Improve wording and spans for unexpected token

* Point at where the token was expected instead of the last token
  successfuly parsed.
* Only show `unexpected token` if the next char and the unexpected token
  don't have the same span.
* Change some cfail and pfail tests to ui test.
* Don't show all possible tokens in span label if they are more than 6.

Author: estebank

src/libsyntax/parse/parser.rs

@@ -548,19 +548,32 @@ impl<'a> Parser<'a> {
             expected.dedup();
             let expect = tokens_to_string(&expected[..]);
             let actual = self.this_token_to_string();
-            let (msg_exp, label_exp) = if expected.len() > 1 {
+            let (msg_exp, (label_sp, label_exp)) = if expected.len() > 1 {
+                let short_expect = if expected.len() > 6 {
+                    format!("{} possible tokens", expected.len())
+                } else {
+                    expect.clone()
+                };
                 (format!("expected one of {}, found `{}`", expect, actual),
-                 format!("expected one of {} after this", expect))
+                 (self.prev_span.next_point(), format!("expected one of {} here", short_expect)))
             } else if expected.is_empty() {
                 (format!("unexpected token: `{}`", actual),
-                 "unexpected token after this".to_string())
+                 (self.prev_span, "unexpected token after this".to_string()))
             } else {
                 (format!("expected {}, found `{}`", expect, actual),
-                 format!("expected {} after this", expect))
+                 (self.prev_span.next_point(), format!("expected {} here", expect)))
             };
             let mut err = self.fatal(&msg_exp);
-            err.span_label(self.prev_span, &label_exp);
-            err.span_label(self.span, &"unexpected token");
+            let sp = if self.token == token::Token::Eof {
+                // This is EOF, don't want to point at the following char, but rather the last token
+                self.prev_span
+            } else {
+                label_sp
+            };
+            err.span_label(sp, &label_exp);
+            if label_sp != self.span {
+                err.span_label(self.span, &"unexpected token");
+            }
             Err(err)
         }
     }

src/libsyntax_pos/lib.rs

@@ -79,6 +79,12 @@ impl Span {
         Span { lo: BytePos(lo), hi: self.hi, expn_id: self.expn_id}
     }
 
+    /// Returns a new span representing the next character after the end-point of this span
+    pub fn next_point(self) -> Span {
+        let lo = BytePos(cmp::max(self.hi.0, self.lo.0 + 1));
+        Span { lo: lo, hi: lo, expn_id: self.expn_id}
+    }
+
     /// Returns `self` if `self` is not the dummy span, and `other` otherwise.
     pub fn substitute_dummy(self, other: Span) -> Span {
         if self.source_equal(&DUMMY_SP) { other } else { self }

src/test/parse-fail/match-refactor-to-expr.rs

@@ -14,7 +14,7 @@ fn main() {
     let foo =
         match //~ NOTE did you mean to remove this `match` keyword?
         Some(4).unwrap_or_else(5)
-        //~^ NOTE expected one of `.`, `?`, `{`, or an operator after this
+        //~^ NOTE expected one of `.`, `?`, `{`, or an operator here
         ; //~ NOTE unexpected token
         //~^ ERROR expected one of `.`, `?`, `{`, or an operator, found `;`
 

src/test/ui/resolve/token-error-correct-3.stderr

@@ -14,15 +14,13 @@ error: expected one of `,`, `.`, `?`, or an operator, found `;`
   --> $DIR/token-error-correct-3.rs:23:35
    |
 23 |             callback(path.as_ref();  //~ NOTE: unclosed delimiter
-   |                                  -^ unexpected token
-   |                                  |
-   |                                  expected one of `,`, `.`, `?`, or an operator after this
+   |                                   ^ expected one of `,`, `.`, `?`, or an operator here
 
 error: expected one of `.`, `;`, `?`, `}`, or an operator, found `)`
   --> $DIR/token-error-correct-3.rs:29:9
    |
 25 |             fs::create_dir_all(path.as_ref()).map(|()| true) //~ ERROR: mismatched types
-   |                                                            - expected one of `.`, `;`, `?`, `}`, or an operator after this
+   |                                                             - expected one of `.`, `;`, `?`, `}`, or an operator here
 ...
 29 |         } else { //~ ERROR: incorrect close delimiter: `}`
    |         ^ unexpected token

src/test/ui/resolve/token-error-correct.stderr

@@ -32,9 +32,7 @@ error: expected one of `)`, `,`, `.`, `<`, `?`, `break`, `continue`, `false`, `f
   --> $DIR/token-error-correct.rs:14:13
    |
 14 |     foo(bar(;
-   |            -^ unexpected token
-   |            |
-   |            expected one of `)`, `,`, `.`, `<`, `?`, `break`, `continue`, `false`, `for`, `if`, `loop`, `match`, `move`, `return`, `true`, `unsafe`, `while`, or an operator after this
+   |             ^ expected one of 18 possible tokens here
 
 error: expected expression, found `)`
   --> $DIR/token-error-correct.rs:23:1

src/test/parse-fail/bounds-obj-parens.rs renamed to src/test/ui/token/bounds-obj-parens.rs

@@ -0,0 +1,7 @@
+error: expected one of `!` or `::`, found `<eof>`
+  --> $DIR/bounds-obj-parens.rs:15:1
+   |
+15 | FAIL
+   | ^^^^ expected one of `!` or `::` here
+
+error: aborting due to previous error

src/test/ui/token/bounds-obj-parens.stderr

@@ -0,0 +1,27 @@
+error: incorrect close delimiter: `}`
+  --> $DIR/issue-10636-2.rs:19:1
+   |
+19 | } //~ ERROR: incorrect close delimiter
+   | ^
+   |
+note: unclosed delimiter
+  --> $DIR/issue-10636-2.rs:15:15
+   |
+15 |     option.map(|some| 42; //~ NOTE: unclosed delimiter
+   |               ^
+
+error: expected one of `,`, `.`, `?`, or an operator, found `;`
+  --> $DIR/issue-10636-2.rs:15:25
+   |
+15 |     option.map(|some| 42; //~ NOTE: unclosed delimiter
+   |                         ^ expected one of `,`, `.`, `?`, or an operator here
+
+error: expected expression, found `)`
+  --> $DIR/issue-10636-2.rs:19:1
+   |
+19 | } //~ ERROR: incorrect close delimiter
+   | ^
+
+error: main function not found
+
+error: aborting due to 4 previous errors

src/test/compile-fail/issue-10636-2.rs renamed to src/test/ui/token/issue-10636-2.rs

@@ -20,7 +20,7 @@ macro_rules! ignored_item {
 
 macro_rules! ignored_expr {
     () => ( 1,  //~ ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `,`
-                //~^ NOTE expected one of `.`, `;`, `?`, `}`, or an operator after this
+                //~^ NOTE expected one of `.`, `;`, `?`, `}`, or an operator here
                 //~| NOTE unexpected token
             2 )
 }

src/test/ui/token/issue-10636-2.stderr

@@ -0,0 +1,31 @@
+error: macro expansion ignores token `,` and any following
+  --> $DIR/macro-incomplete-parse.rs:17:9
+   |
+17 |         , //~ ERROR macro expansion ignores token `,`
+   |         ^
+   |
+note: caused by the macro expansion here; the usage of `ignored_item!` is likely invalid in item context
+  --> $DIR/macro-incomplete-parse.rs:32:1
+   |
+32 | ignored_item!(); //~ NOTE caused by the macro expansion here
+   | ^^^^^^^^^^^^^^^^
+
+error: expected one of `.`, `;`, `?`, `}`, or an operator, found `,`
+  --> $DIR/macro-incomplete-parse.rs:22:14
+   |
+22 |     () => ( 1,  //~ ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `,`
+   |              ^ expected one of `.`, `;`, `?`, `}`, or an operator here
+
+error: macro expansion ignores token `,` and any following
+  --> $DIR/macro-incomplete-parse.rs:29:14
+   |
+29 |     () => ( 1, 2 ) //~ ERROR macro expansion ignores token `,`
+   |              ^
+   |
+note: caused by the macro expansion here; the usage of `ignored_pat!` is likely invalid in pattern context
+  --> $DIR/macro-incomplete-parse.rs:37:9
+   |
+37 |         ignored_pat!() => (), //~ NOTE caused by the macro expansion here
+   |         ^^^^^^^^^^^^^^
+
+error: aborting due to 3 previous errors

src/test/compile-fail/macro-incomplete-parse.rs renamed to src/test/ui/token/macro-incomplete-parse.rs

@@ -0,0 +1,7 @@
+error: expected one of `!` or `::`, found `<eof>`
+  --> ../../src/test/ui/token/trailing-plus-in-bounds.rs:19:1
+   |
+19 | FAIL
+   | ^^^^ expected one of `!` or `::` here
+
+error: aborting due to previous error