Ignore non-semantic tokens for 'probably_eq' streams.

Author: SergioBenitez

src/libsyntax/parse/token.rs

@@ -570,8 +570,9 @@ impl Token {
         //
         // Instead the "probably equal" check here is "does each token
         // recursively have the same discriminant?" We basically don't look at
-        // the token values here and assume that such fine grained modifications
-        // of token streams doesn't happen.
+        // the token values here and assume that such fine grained token stream
+        // modifications, including adding/removing typically non-semantic
+        // tokens such as extra braces and commas, don't happen.
         if let Some(tokens) = tokens {
             if tokens.probably_equal_for_proc_macro(&tokens_for_real) {
                 return tokens

src/libsyntax/tokenstream.rs

@@ -26,7 +26,7 @@ use syntax_pos::{BytePos, Mark, Span, DUMMY_SP};
 use ext::base;
 use ext::tt::{macro_parser, quoted};
 use parse::Directory;
-use parse::token::{self, Token};
+use parse::token::{self, DelimToken, Token};
 use print::pprust;
 use serialize::{Decoder, Decodable, Encoder, Encodable};
 use util::RcVec;
@@ -38,7 +38,7 @@ use std::{fmt, iter, mem};
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
 pub struct Delimited {
     /// The type of delimiter
-    pub delim: token::DelimToken,
+    pub delim: DelimToken,
     /// The delimited sequence of token trees
     pub tts: ThinTokenStream,
 }
@@ -368,8 +368,30 @@ impl TokenStream {
     // This is otherwise the same as `eq_unspanned`, only recursing with a
     // different method.
     pub fn probably_equal_for_proc_macro(&self, other: &TokenStream) -> bool {
-        let mut t1 = self.trees();
-        let mut t2 = other.trees();
+        // When checking for `probably_eq`, we ignore certain tokens that aren't
+        // preserved in the AST. Because they are not preserved, the pretty
+        // printer arbitrarily adds or removes them when printing as token
+        // streams, making a comparison between a token stream generated from an
+        // AST and a token stream which was parsed into an AST more reliable.
+        fn semantic_tree(tree: &TokenTree) -> bool {
+            match tree {
+                // The pretty printer tends to add trailing commas to
+                // everything, and in particular, after struct fields.
+                | TokenTree::Token(_, Token::Comma)
+                // The pretty printer emits `NoDelim` as whitespace.
+                | TokenTree::Token(_, Token::OpenDelim(DelimToken::NoDelim))
+                | TokenTree::Token(_, Token::CloseDelim(DelimToken::NoDelim))
+                // The pretty printer collapses many semicolons into one.
+                | TokenTree::Token(_, Token::Semi)
+                // The pretty printer collapses whitespace arbitrarily and can
+                // introduce whitespace from `NoDelim`.
+                | TokenTree::Token(_, Token::Whitespace) => false,
+                _ => true
+            }
+        }
+
+        let mut t1 = self.trees().filter(semantic_tree);
+        let mut t2 = other.trees().filter(semantic_tree);
         for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
             if !t1.probably_equal_for_proc_macro(&t2) {
                 return false;

src/test/ui-fulldeps/proc-macro/auxiliary/span-preservation.rs

@@ -0,0 +1,12 @@
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn foo(_: TokenStream, input: TokenStream) -> TokenStream {
+    input.into_iter().collect()
+}

src/test/ui-fulldeps/proc-macro/span-preservation.rs

@@ -0,0 +1,51 @@
+// aux-build:span-preservation.rs
+
+// For each of these, we should get the appropriate type mismatch error message,
+// and the function should be echoed.
+
+extern crate span_preservation as foo;
+
+use foo::foo;
+
+#[foo]
+fn a() {
+    let x: usize = "hello";;;;;
+}
+
+#[foo]
+fn b(x: Option<isize>) -> usize {
+    match x {
+        Some(x) => { return x },
+        None => 10
+    }
+}
+
+#[foo]
+fn c() {
+    struct Foo {
+        a: usize
+    }
+
+    struct Bar {
+        a: usize,
+        b: usize
+    }
+
+    let x = Foo { a: 10isize };
+    let y = Foo { a: 10, b: 10isize };
+}
+
+// FIXME: This doesn't work at the moment. See the one below. The pretty-printer
+// injects a "C" between `extern` and `fn` which causes a "probably_eq"
+// `TokenStream` mismatch. The lack of `"C"` should be preserved in the AST.
+#[foo]
+extern fn bar() {
+    0
+}
+
+#[foo]
+extern "C" fn baz() {
+    0
+}
+
+fn main() {}

src/test/ui-fulldeps/proc-macro/span-preservation.stderr

@@ -0,0 +1,49 @@
+error[E0308]: mismatched types
+   |
+   = note: expected type `()`
+              found type `{integer}`
+
+error[E0308]: mismatched types
+  --> $DIR/span-preservation.rs:12:20
+   |
+LL |     let x: usize = "hello";;;;;
+   |                    ^^^^^^^ expected usize, found reference
+   |
+   = note: expected type `usize`
+              found type `&'static str`
+
+error[E0308]: mismatched types
+  --> $DIR/span-preservation.rs:18:29
+   |
+LL |         Some(x) => { return x },
+   |                             ^ expected usize, found isize
+
+error[E0308]: mismatched types
+  --> $DIR/span-preservation.rs:34:22
+   |
+LL |     let x = Foo { a: 10isize };
+   |                      ^^^^^^^ expected usize, found isize
+
+error[E0560]: struct `c::Foo` has no field named `b`
+  --> $DIR/span-preservation.rs:35:26
+   |
+LL |     let y = Foo { a: 10, b: 10isize };
+   |                          ^ `c::Foo` does not have this field
+   |
+   = note: available fields are: `a`
+
+error[E0308]: mismatched types
+  --> $DIR/span-preservation.rs:48:5
+   |
+LL | extern "C" fn baz() {
+   |                     - possibly return type missing here?
+LL |     0
+   |     ^ expected (), found integral variable
+   |
+   = note: expected type `()`
+              found type `{integer}`
+
+error: aborting due to 6 previous errors
+
+Some errors occurred: E0308, E0560.
+For more information about an error, try `rustc --explain E0308`.