Rollup merge of #33041 - petrochenkov:path, r=nrc,Manishearth

 Paths are mostly parsed without taking whitespaces into account, e.g. `std :: vec :: Vec :: new ()` parses successfully, however, there are some special cases involving keywords `super`, `self` and `Self`. For example, `self::` is considered a path start only if there are no spaces between `self` and `::`. These restrictions probably made sense when `self` and friends weren't keywords, but now they are unnecessary.

The first two commits remove this special treatment of whitespaces by removing `token::IdentStyle` entirely and therefore fix #14109.
This change also affects naked `self` and `super` (which are not tightly followed by `::`, obviously) they can now be parsed as paths, however they are still not resolved correctly in imports (cc @jseyfried, see `compile-fail/use-keyword.rs`), so #29036 is not completely fixed.

The third commit also makes `super`, `self`, `Self` and `static` keywords nominally (before this they acted as keywords for all purposes) and removes most of remaining \"special idents\".

The last commit (before tests) contains some small improvements - some qualified paths with type parameters are parsed correctly, `parse_path` is not used for parsing single identifiers, imports are sanity checked for absence of type parameters - such type parameters can be generated by syntax extensions or by macros when #10415 is fixed (~~soon!~~already!).

This patch changes some pretty basic things in `libsyntax`, like `token::Token` and the keyword list, so it's a plugin-[breaking-change].

r? @eddyb

Author: Manishearth

src/etc/generate-keyword-tests.py

@@ -34,15 +34,17 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// compile-flags: -Z parse-only
+
 // This file was auto-generated using 'src/etc/generate-keyword-tests.py %s'
 
 fn main() {
-    let %s = "foo"; //~ error: ident
+    let %s = "foo"; //~ error: expected pattern, found keyword `%s`
 }
 """
 
 test_dir = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), '../test/compile-fail')
+    os.path.join(os.path.dirname(__file__), '../test/parse-fail')
 )
 
 for kw in sys.argv[1:]:
@@ -53,7 +55,7 @@
         os.chmod(test_file, stat.S_IWUSR)
 
     with open(test_file, 'wt') as f:
-        f.write(template % (datetime.datetime.now().year, kw, kw))
+        f.write(template % (datetime.datetime.now().year, kw, kw, kw))
 
     # mark file read-only
     os.chmod(test_file, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)

src/librustc/hir/fold.rs

@@ -18,7 +18,7 @@ use syntax::attr::ThinAttributesExt;
 use hir;
 use syntax::codemap::{respan, Span, Spanned};
 use syntax::ptr::P;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 use syntax::util::move_map::MoveMap;
 
 pub trait Folder : Sized {
@@ -867,7 +867,7 @@ pub fn noop_fold_crate<T: Folder>(Crate { module, attrs, config, span,
     let config = folder.fold_meta_items(config);
 
     let crate_mod = folder.fold_item(hir::Item {
-        name: token::special_idents::invalid.name,
+        name: keywords::Invalid.name(),
         attrs: attrs,
         id: DUMMY_NODE_ID,
         vis: hir::Public,

src/librustc/hir/print.rs

@@ -14,9 +14,8 @@ use syntax::abi::Abi;
 use syntax::ast;
 use syntax::codemap::{self, CodeMap, BytePos, Spanned};
 use syntax::errors;
-use syntax::parse::token::{self, BinOpToken};
+use syntax::parse::token::{self, keywords, BinOpToken};
 use syntax::parse::lexer::comments;
-use syntax::parse;
 use syntax::print::pp::{self, break_offset, word, space, hardbreak};
 use syntax::print::pp::{Breaks, eof};
 use syntax::print::pp::Breaks::{Consistent, Inconsistent};
@@ -2209,9 +2208,8 @@ impl<'a> State<'a> {
             hir::TyInfer if is_closure => self.print_pat(&input.pat)?,
             _ => {
                 match input.pat.node {
-                    PatKind::Ident(_, ref path1, _) if
-                        path1.node.name ==
-                            parse::token::special_idents::invalid.name => {
+                    PatKind::Ident(_, ref path1, _)
+                            if path1.node.name == keywords::Invalid.name() => {
                         // Do nothing.
                     }
                     _ => {

src/librustc/middle/liveness.rs

@@ -125,7 +125,7 @@ use std::io;
 use std::rc::Rc;
 use syntax::ast::{self, NodeId};
 use syntax::codemap::{BytePos, original_sp, Span};
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use syntax::ptr::P;
 
 use hir::Expr;
@@ -1578,7 +1578,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 let var = self.variable(p_id, sp);
                 // Ignore unused self.
                 let name = path1.node;
-                if name != special_idents::self_.name {
+                if name != keywords::SelfValue.name() {
                     if !self.warn_about_unused(sp, p_id, entry_ln, var) {
                         if self.live_on_entry(entry_ln, var).is_none() {
                             self.report_dead_assign(p_id, sp, var, true);

src/librustc/middle/resolve_lifetime.rs

@@ -29,7 +29,7 @@ use std::fmt;
 use std::mem::replace;
 use syntax::ast;
 use syntax::codemap::Span;
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use util::nodemap::NodeMap;
 
 use hir;
@@ -245,7 +245,7 @@ impl<'a, 'v> Visitor<'v> for LifetimeContext<'a> {
     }
 
     fn visit_lifetime(&mut self, lifetime_ref: &hir::Lifetime) {
-        if lifetime_ref.name == special_idents::static_lifetime.name {
+        if lifetime_ref.name == keywords::StaticLifetime.name() {
             self.insert_lifetime(lifetime_ref, DefStaticRegion);
             return;
         }
@@ -672,9 +672,8 @@ impl<'a> LifetimeContext<'a> {
         for i in 0..lifetimes.len() {
             let lifetime_i = &lifetimes[i];
 
-            let special_idents = [special_idents::static_lifetime];
             for lifetime in lifetimes {
-                if special_idents.iter().any(|&i| i.name == lifetime.lifetime.name) {
+                if lifetime.lifetime.name == keywords::StaticLifetime.name() {
                     span_err!(self.sess, lifetime.lifetime.span, E0262,
                         "invalid lifetime parameter name: `{}`", lifetime.lifetime.name);
                 }

src/librustc/mir/repr.rs

@@ -200,7 +200,7 @@ pub struct ArgDecl<'tcx> {
     /// and has to be collected from multiple actual arguments.
     pub spread: bool,
 
-    /// Either special_idents::invalid or the name of a single-binding
+    /// Either keywords::Invalid or the name of a single-binding
     /// pattern associated with this argument. Useful for debuginfo.
     pub debug_name: Name
 }

src/librustc/ty/context.rs

@@ -44,7 +44,7 @@ use std::hash::{Hash, Hasher};
 use std::rc::Rc;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
-use syntax::parse::token::{self, special_idents};
+use syntax::parse::token::{self, keywords};
 
 use hir;
 
@@ -1069,7 +1069,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn mk_self_type(&self) -> Ty<'tcx> {
-        self.mk_param(subst::SelfSpace, 0, special_idents::type_self.name)
+        self.mk_param(subst::SelfSpace, 0, keywords::SelfType.name())
     }
 
     pub fn mk_param_from_def(&self, def: &ty::TypeParameterDef) -> Ty<'tcx> {

src/librustc/ty/sty.rs

@@ -24,7 +24,7 @@ use std::ops;
 use std::mem;
 use syntax::abi;
 use syntax::ast::{self, Name};
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 
 use serialize::{Decodable, Decoder};
 
@@ -533,7 +533,7 @@ impl ParamTy {
     }
 
     pub fn for_self() -> ParamTy {
-        ParamTy::new(subst::SelfSpace, 0, special_idents::type_self.name)
+        ParamTy::new(subst::SelfSpace, 0, keywords::SelfType.name())
     }
 
     pub fn for_def(def: &ty::TypeParameterDef) -> ParamTy {

src/librustc_mir/build/mod.rs

@@ -18,7 +18,7 @@ use rustc::hir::pat_util::pat_is_binding;
 use std::ops::{Index, IndexMut};
 use syntax::ast;
 use syntax::codemap::Span;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 
 pub struct Builder<'a, 'tcx: 'a> {
     hir: Cx<'a, 'tcx>,
@@ -238,7 +238,7 @@ pub fn construct<'a,'tcx>(hir: Cx<'a,'tcx>,
                 ty::UpvarCapture::ByRef(..) => true
             });
             let mut decl = UpvarDecl {
-                debug_name: token::special_idents::invalid.name,
+                debug_name: keywords::Invalid.name(),
                 by_ref: by_ref
             };
             if let Some(hir::map::NodeLocal(pat)) = tcx.map.find(fv.def.var_id()) {
@@ -296,7 +296,7 @@ impl<'a,'tcx> Builder<'a,'tcx> {
                 self.schedule_drop(pattern.as_ref().map_or(ast_block.span, |pat| pat.span),
                                    argument_extent, &lvalue, ty);
 
-                let mut name = token::special_idents::invalid.name;
+                let mut name = keywords::Invalid.name();
                 if let Some(pat) = pattern {
                     if let hir::PatKind::Ident(_, ref ident, _) = pat.node {
                         if pat_is_binding(&self.hir.tcx().def_map.borrow(), pat) {

src/librustc_resolve/build_reduced_graph.rs

@@ -112,15 +112,14 @@ impl<'b, 'tcx:'b> Resolver<'b, 'tcx> {
             !segment.parameters.bindings().is_empty()
         });
         if found_param {
-            self.session.span_err(path.span,
-                                  "type or lifetime parameter is found in import path");
+            self.session.span_err(path.span, "type or lifetime parameters in import path");
         }
 
         // Checking for special identifiers in path
         // prevent `self` or `super` at beginning of global path
         if path.global && path.segments.len() > 0 {
             let first = path.segments[0].identifier.name;
-            if first == keywords::Super.to_name() || first == keywords::SelfValue.to_name() {
+            if first == keywords::Super.name() || first == keywords::SelfValue.name() {
                 self.session.add_lint(
                     lint::builtin::SUPER_OR_SELF_IN_GLOBAL_PATH, id, path.span,
                     format!("expected identifier, found keyword `{}`", first)

src/librustc_resolve/lib.rs

@@ -62,7 +62,7 @@ use syntax::ast::{CRATE_NODE_ID, Name, NodeId, CrateNum, IntTy, UintTy};
 use syntax::attr::AttrMetaMethods;
 use syntax::codemap::{self, Span, Pos};
 use syntax::errors::DiagnosticBuilder;
-use syntax::parse::token::{self, special_names, special_idents};
+use syntax::parse::token::{self, keywords};
 use syntax::util::lev_distance::find_best_match_for_name;
 
 use rustc::hir::intravisit::{self, FnKind, Visitor};
@@ -1954,8 +1954,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         let mut self_type_rib = Rib::new(NormalRibKind);
 
         // plain insert (no renaming, types are not currently hygienic....)
-        let name = special_names::type_self;
-        self_type_rib.bindings.insert(name, self_def);
+        self_type_rib.bindings.insert(keywords::SelfType.name(), self_def);
         self.type_ribs.push(self_type_rib);
         f(self);
         if !self.resolved {
@@ -2195,11 +2194,10 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                             "type name"
                         };
 
-                        let self_type_name = special_idents::type_self.name;
                         let is_invalid_self_type_name = path.segments.len() > 0 &&
                                                         maybe_qself.is_none() &&
                                                         path.segments[0].identifier.name ==
-                                                        self_type_name;
+                                                        keywords::SelfType.name();
                         if is_invalid_self_type_name {
                             resolve_error(self,
                                           ty.span,
@@ -2643,7 +2641,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                           namespace: Namespace,
                           record_used: bool)
                           -> Option<LocalDef> {
-        if identifier.name == special_idents::invalid.name {
+        if identifier.unhygienic_name == keywords::Invalid.name() {
             return Some(LocalDef::from_def(Def::Err));
         }
 
@@ -3074,7 +3072,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                                 false // Stop advancing
                             });
 
-                            if method_scope && special_names::self_.as_str() == &path_name[..] {
+                            if method_scope &&
+                                    &path_name[..] == keywords::SelfValue.name().as_str() {
                                 resolve_error(self,
                                               expr.span,
                                               ResolutionError::SelfNotAvailableInStaticMethod);
@@ -3612,7 +3611,7 @@ fn module_to_string(module: Module) -> String {
             }
             BlockParentLink(ref module, _) => {
                 // danger, shouldn't be ident?
-                names.push(special_idents::opaque.name);
+                names.push(token::intern("<opaque>"));
                 collect_mod(names, module);
             }
         }

src/librustc_save_analysis/dump_visitor.rs

@@ -1011,7 +1011,7 @@ impl<'v, 'l, 'tcx: 'l, 'll, D: Dump +'ll> Visitor<'v> for DumpVisitor<'l, 'tcx,
                                 span: sub_span.expect("No span found for use"),
                                 id: item.id,
                                 mod_id: mod_id,
-                                name: ident.name.to_string(),
+                                name: ident.to_string(),
                                 scope: self.cur_scope
                             }.normalize(&self.tcx));
                         }
@@ -1075,7 +1075,7 @@ impl<'v, 'l, 'tcx: 'l, 'll, D: Dump +'ll> Visitor<'v> for DumpVisitor<'l, 'tcx,
                 if !self.span.filter_generated(alias_span, item.span) {
                     self.dumper.extern_crate(item.span, ExternCrateData {
                         id: item.id,
-                        name: item.ident.name.to_string(),
+                        name: item.ident.to_string(),
                         crate_num: cnum,
                         location: location,
                         span: alias_span.expect("No span found for extern crate"),

src/librustc_trans/mir/mod.rs

@@ -22,7 +22,7 @@ use machine;
 use type_of;
 
 use syntax::codemap::DUMMY_SP;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 
 use std::ops::Deref;
 use std::rc::Rc;
@@ -286,7 +286,7 @@ fn arg_value_refs<'bcx, 'tcx>(bcx: &BlockAndBuilder<'bcx, 'tcx>,
                         alloca: lltemp,
                         address_operations: &ops
                     };
-                    declare_local(bcx, token::special_idents::invalid.name,
+                    declare_local(bcx, keywords::Invalid.name(),
                                   tupled_arg_ty, scope, variable_access,
                                   VariableKind::ArgumentVariable(arg_index + i + 1),
                                   bcx.fcx().span.unwrap_or(DUMMY_SP));

src/librustc_typeck/astconv.rs

@@ -73,7 +73,7 @@ use syntax::{abi, ast};
 use syntax::codemap::{Span, Pos};
 use syntax::errors::DiagnosticBuilder;
 use syntax::feature_gate::{GateIssue, emit_feature_err};
-use syntax::parse::token;
+use syntax::parse::token::{self, keywords};
 
 use rustc::hir::print as pprust;
 use rustc::hir;
@@ -1313,7 +1313,7 @@ fn associated_path_def_to_ty<'tcx>(this: &AstConv<'tcx>,
             let trait_node_id = tcx.map.as_local_node_id(trait_did).unwrap();
             match find_bound_for_assoc_item(this,
                                             trait_node_id,
-                                            token::special_idents::type_self.name,
+                                            keywords::SelfType.name(),
                                             assoc_name,
                                             span) {
                 Ok(bound) => bound,

src/librustc_typeck/check/mod.rs

@@ -120,7 +120,7 @@ use syntax::attr;
 use syntax::attr::AttrMetaMethods;
 use syntax::codemap::{self, Span, Spanned};
 use syntax::errors::DiagnosticBuilder;
-use syntax::parse::token::{self, InternedString, special_idents};
+use syntax::parse::token::{self, InternedString, keywords};
 use syntax::ptr::P;
 use syntax::util::lev_distance::find_best_match_for_name;
 
@@ -2851,7 +2851,7 @@ fn check_expr_with_expectation_and_lvalue_pref<'a, 'tcx>(fcx: &FnCtxt<'a, 'tcx>,
                 method_ty
             }
             Err(error) => {
-                if method_name.node != special_idents::invalid.name {
+                if method_name.node != keywords::Invalid.name() {
                     method::report_error(fcx, method_name.span, expr_t,
                                          method_name.node, Some(rcvr), error);
                 }
@@ -2990,7 +2990,7 @@ fn check_expr_with_expectation_and_lvalue_pref<'a, 'tcx>(fcx: &FnCtxt<'a, 'tcx>,
             let msg = format!("field `{}` of struct `{}` is private", field.node, struct_path);
             fcx.tcx().sess.span_err(expr.span, &msg);
             fcx.write_ty(expr.id, field_ty);
-        } else if field.node == special_idents::invalid.name {
+        } else if field.node == keywords::Invalid.name() {
             fcx.write_error(expr.id);
         } else if method::exists(fcx, field.span, field.node, expr_t, expr.id) {
             fcx.type_error_struct(field.span,
@@ -3780,7 +3780,7 @@ pub fn resolve_ty_and_def_ufcs<'a, 'b, 'tcx>(fcx: &FnCtxt<'b, 'tcx>,
                     method::MethodError::PrivateMatch(def) => Some(def),
                     _ => None,
                 };
-                if item_name != special_idents::invalid.name {
+                if item_name != keywords::Invalid.name() {
                     method::report_error(fcx, span, ty, item_name, None, error);
                 }
                 def

src/librustc_typeck/check/wfcheck.rs

@@ -24,7 +24,7 @@ use std::collections::HashSet;
 use syntax::ast;
 use syntax::codemap::{Span};
 use syntax::errors::DiagnosticBuilder;
-use syntax::parse::token::{special_idents};
+use syntax::parse::token::keywords;
 use rustc::hir::intravisit::{self, Visitor};
 use rustc::hir;
 
@@ -472,7 +472,7 @@ impl<'ccx, 'tcx> CheckTypeWellFormedVisitor<'ccx, 'tcx> {
     {
         let name = match space {
             TypeSpace => ast_generics.ty_params[index].name,
-            SelfSpace => special_idents::type_self.name,
+            SelfSpace => keywords::SelfType.name(),
             FnSpace => bug!("Fn space occupied?"),
         };
 

src/librustc_typeck/collect.rs

@@ -93,7 +93,7 @@ use syntax::abi;
 use syntax::ast;
 use syntax::attr;
 use syntax::codemap::Span;
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use syntax::ptr::P;
 use rustc::hir::{self, PatKind};
 use rustc::hir::intravisit;
@@ -1655,7 +1655,7 @@ fn ty_generics_for_trait<'a, 'tcx>(ccx: &CrateCtxt<'a, 'tcx>,
     let def = ty::TypeParameterDef {
         space: SelfSpace,
         index: 0,
-        name: special_idents::type_self.name,
+        name: keywords::SelfType.name(),
         def_id: ccx.tcx.map.local_def_id(param_id),
         default_def_id: ccx.tcx.map.local_def_id(parent),
         default: None,