Auto merge of #50307 - petrochenkov:keyhyg2, r=nikomatsakis

Implement edition hygiene for keywords

Determine "keywordness" of an identifier in its hygienic context.
cc #49611

I've resurrected `proc` as an Edition-2015-only keyword for testing purposes, but it should probably be buried again. EDIT: `proc` is removed again.

Author: bors

src/libproc_macro/lib.rs

@@ -818,7 +818,7 @@ impl Ident {
     pub fn new_raw(string: &str, span: Span) -> Ident {
         let mut ident = Ident::new(string, span);
         if ident.sym == keywords::Underscore.name() ||
-           token::is_path_segment_keyword(ast::Ident::with_empty_ctxt(ident.sym)) {
+           ast::Ident::with_empty_ctxt(ident.sym).is_path_segment_keyword() {
             panic!("`{:?}` is not a valid raw identifier", string)
         }
         ident.is_raw = true;

src/librustc/hir/lowering.rs

@@ -593,6 +593,7 @@ impl<'a> LoweringContext<'a> {
                 span: Some(span),
                 allow_internal_unstable: true,
                 allow_internal_unsafe: false,
+                edition: codemap::hygiene::default_edition(),
             },
         });
         span.with_ctxt(SyntaxContext::empty().apply_mark(mark))

src/librustc/hir/print.rs

@@ -13,7 +13,7 @@ pub use self::AnnNode::*;
 use rustc_target::spec::abi::Abi;
 use syntax::ast;
 use syntax::codemap::{CodeMap, Spanned};
-use syntax::parse::{token, ParseSess};
+use syntax::parse::ParseSess;
 use syntax::parse::lexer::comments;
 use syntax::print::pp::{self, Breaks};
 use syntax::print::pp::Breaks::{Consistent, Inconsistent};
@@ -1559,7 +1559,7 @@ impl<'a> State<'a> {
     }
 
     pub fn print_name(&mut self, name: ast::Name) -> io::Result<()> {
-        if token::is_raw_guess(ast::Ident::with_empty_ctxt(name)) {
+        if name.to_ident().is_raw_guess() {
             self.s.word(&format!("r#{}", name))?;
         } else {
             self.s.word(&name.as_str())?;

src/librustc/ich/impls_syntax.rs

@@ -131,6 +131,15 @@ impl_stable_hash_for!(struct ::syntax::attr::Stability {
     rustc_const_unstable
 });
 
+impl<'a> HashStable<StableHashingContext<'a>>
+for ::syntax::edition::Edition {
+    fn hash_stable<W: StableHasherResult>(&self,
+                                          hcx: &mut StableHashingContext<'a>,
+                                          hasher: &mut StableHasher<W>) {
+        mem::discriminant(self).hash_stable(hcx, hasher);
+    }
+}
+
 impl<'a> HashStable<StableHashingContext<'a>>
 for ::syntax::attr::StabilityLevel {
     fn hash_stable<W: StableHasherResult>(&self,
@@ -389,6 +398,7 @@ impl_stable_hash_for!(struct ::syntax_pos::hygiene::NameAndSpan {
     format,
     allow_internal_unstable,
     allow_internal_unsafe,
+    edition,
     span
 });
 

src/librustc/middle/cstore.rs

@@ -34,6 +34,7 @@ use session::search_paths::PathKind;
 use std::any::Any;
 use std::path::{Path, PathBuf};
 use syntax::ast;
+use syntax::edition::Edition;
 use syntax::ext::base::SyntaxExtension;
 use syntax::symbol::Symbol;
 use syntax_pos::Span;
@@ -235,6 +236,7 @@ pub trait CrateStore {
     fn crate_name_untracked(&self, cnum: CrateNum) -> Symbol;
     fn crate_disambiguator_untracked(&self, cnum: CrateNum) -> CrateDisambiguator;
     fn crate_hash_untracked(&self, cnum: CrateNum) -> Svh;
+    fn crate_edition_untracked(&self, cnum: CrateNum) -> Edition;
     fn struct_field_names_untracked(&self, def: DefId) -> Vec<ast::Name>;
     fn item_children_untracked(&self, did: DefId, sess: &Session) -> Vec<def::Export>;
     fn load_macro_untracked(&self, did: DefId, sess: &Session) -> LoadedMacro;
@@ -309,6 +311,7 @@ impl CrateStore for DummyCrateStore {
         bug!("crate_disambiguator")
     }
     fn crate_hash_untracked(&self, cnum: CrateNum) -> Svh { bug!("crate_hash") }
+    fn crate_edition_untracked(&self, cnum: CrateNum) -> Edition { bug!("crate_edition_untracked") }
 
     // resolve
     fn def_key(&self, def: DefId) -> DefKey { bug!("def_key") }

src/librustc_allocator/expand.rs

@@ -21,7 +21,7 @@ use syntax::ext::base::ExtCtxt;
 use syntax::ext::base::Resolver;
 use syntax::ext::build::AstBuilder;
 use syntax::ext::expand::ExpansionConfig;
-use syntax::ext::hygiene::{Mark, SyntaxContext};
+use syntax::ext::hygiene::{self, Mark, SyntaxContext};
 use syntax::fold::{self, Folder};
 use syntax::parse::ParseSess;
 use syntax::ptr::P;
@@ -86,6 +86,7 @@ impl<'a> Folder for ExpandAllocatorDirectives<'a> {
                 span: None,
                 allow_internal_unstable: true,
                 allow_internal_unsafe: false,
+                edition: hygiene::default_edition(),
             },
         });
         let span = item.span.with_ctxt(SyntaxContext::empty().apply_mark(mark));

src/librustc_driver/lib.rs

@@ -108,7 +108,7 @@ use syntax::ast;
 use syntax::codemap::{CodeMap, FileLoader, RealFileLoader};
 use syntax::feature_gate::{GatedCfg, UnstableFeatures};
 use syntax::parse::{self, PResult};
-use syntax_pos::{DUMMY_SP, MultiSpan, FileName};
+use syntax_pos::{hygiene, DUMMY_SP, MultiSpan, FileName};
 
 #[cfg(test)]
 mod test;
@@ -466,6 +466,7 @@ pub fn run_compiler<'a>(args: &[String],
         };
 
         let (sopts, cfg) = config::build_session_options_and_crate_config(&matches);
+        hygiene::set_default_edition(sopts.edition);
 
         driver::spawn_thread_pool(sopts, |sopts| {
             run_compiler_with_pool(matches, sopts, cfg, callbacks, file_loader, emitter_dest)

src/librustc_metadata/creader.rs

@@ -35,6 +35,7 @@ use std::{cmp, fs};
 
 use syntax::ast;
 use syntax::attr;
+use syntax::edition::Edition;
 use syntax::ext::base::SyntaxExtension;
 use syntax::symbol::Symbol;
 use syntax::visit;
@@ -535,7 +536,10 @@ impl<'a> CrateLoader<'a> {
             mem::transmute::<*mut u8, fn(&mut Registry)>(sym)
         };
 
-        struct MyRegistrar(Vec<(ast::Name, Lrc<SyntaxExtension>)>);
+        struct MyRegistrar {
+            extensions: Vec<(ast::Name, Lrc<SyntaxExtension>)>,
+            edition: Edition,
+        }
 
         impl Registry for MyRegistrar {
             fn register_custom_derive(&mut self,
@@ -544,36 +548,38 @@ impl<'a> CrateLoader<'a> {
                                       attributes: &[&'static str]) {
                 let attrs = attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
                 let derive = ProcMacroDerive::new(expand, attrs.clone());
-                let derive = SyntaxExtension::ProcMacroDerive(Box::new(derive), attrs);
-                self.0.push((Symbol::intern(trait_name), Lrc::new(derive)));
+                let derive = SyntaxExtension::ProcMacroDerive(
+                    Box::new(derive), attrs, self.edition
+                );
+                self.extensions.push((Symbol::intern(trait_name), Lrc::new(derive)));
             }
 
             fn register_attr_proc_macro(&mut self,
                                         name: &str,
                                         expand: fn(TokenStream, TokenStream) -> TokenStream) {
                 let expand = SyntaxExtension::AttrProcMacro(
-                    Box::new(AttrProcMacro { inner: expand })
+                    Box::new(AttrProcMacro { inner: expand }), self.edition
                 );
-                self.0.push((Symbol::intern(name), Lrc::new(expand)));
+                self.extensions.push((Symbol::intern(name), Lrc::new(expand)));
             }
 
             fn register_bang_proc_macro(&mut self,
                                         name: &str,
                                         expand: fn(TokenStream) -> TokenStream) {
                 let expand = SyntaxExtension::ProcMacro(
-                    Box::new(BangProcMacro { inner: expand })
+                    Box::new(BangProcMacro { inner: expand }), self.edition
                 );
-                self.0.push((Symbol::intern(name), Lrc::new(expand)));
+                self.extensions.push((Symbol::intern(name), Lrc::new(expand)));
             }
         }
 
-        let mut my_registrar = MyRegistrar(Vec::new());
+        let mut my_registrar = MyRegistrar { extensions: Vec::new(), edition: root.edition };
         registrar(&mut my_registrar);
 
         // Intentionally leak the dynamic library. We can't ever unload it
         // since the library can make things that will live arbitrarily long.
         mem::forget(lib);
-        my_registrar.0
+        my_registrar.extensions
     }
 
     /// Look for a plugin registrar. Returns library path, crate

src/librustc_metadata/cstore.rs

@@ -24,6 +24,7 @@ use rustc::util::nodemap::{FxHashMap, NodeMap};
 
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 use syntax::{ast, attr};
+use syntax::edition::Edition;
 use syntax::ext::base::SyntaxExtension;
 use syntax::symbol::Symbol;
 use syntax_pos;
@@ -234,4 +235,8 @@ impl CrateMetadata {
     pub fn panic_strategy(&self) -> PanicStrategy {
         self.root.panic_strategy.clone()
     }
+
+    pub fn edition(&self) -> Edition {
+        self.root.edition
+    }
 }

src/librustc_metadata/cstore_impl.rs

@@ -38,6 +38,7 @@ use std::sync::Arc;
 use syntax::ast;
 use syntax::attr;
 use syntax::codemap;
+use syntax::edition::Edition;
 use syntax::ext::base::SyntaxExtension;
 use syntax::parse::filemap_to_stream;
 use syntax::symbol::Symbol;
@@ -464,6 +465,11 @@ impl CrateStore for cstore::CStore {
         self.get_crate_data(cnum).hash()
     }
 
+    fn crate_edition_untracked(&self, cnum: CrateNum) -> Edition
+    {
+        self.get_crate_data(cnum).edition()
+    }
+
     /// Returns the `DefKey` for a given `DefId`. This indicates the
     /// parent `DefId` as well as some idea of what kind of data the
     /// `DefId` refers to.
@@ -512,7 +518,8 @@ impl CrateStore for cstore::CStore {
             return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
         } else if data.name == "proc_macro" &&
                   self.get_crate_data(id.krate).item_name(id.index) == "quote" {
-            let ext = SyntaxExtension::ProcMacro(Box::new(::proc_macro::__internal::Quoter));
+            let ext = SyntaxExtension::ProcMacro(Box::new(::proc_macro::__internal::Quoter),
+                                                 data.edition());
             return LoadedMacro::ProcMacro(Lrc::new(ext));
         }
 

src/librustc_metadata/encoder.rs

@@ -44,7 +44,7 @@ use syntax::ast::{self, CRATE_NODE_ID};
 use syntax::codemap::Spanned;
 use syntax::attr;
 use syntax::symbol::Symbol;
-use syntax_pos::{self, FileName, FileMap, Span, DUMMY_SP};
+use syntax_pos::{self, hygiene, FileName, FileMap, Span, DUMMY_SP};
 
 use rustc::hir::{self, PatKind};
 use rustc::hir::itemlikevisit::ItemLikeVisitor;
@@ -496,6 +496,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             hash: link_meta.crate_hash,
             disambiguator: tcx.sess.local_crate_disambiguator(),
             panic_strategy: tcx.sess.panic_strategy(),
+            edition: hygiene::default_edition(),
             has_global_allocator: has_global_allocator,
             has_default_lib_allocator: has_default_lib_allocator,
             plugin_registrar_fn: tcx.sess

src/librustc_metadata/schema.rs

@@ -23,6 +23,7 @@ use rustc_target::spec::{PanicStrategy, TargetTriple};
 
 use rustc_serialize as serialize;
 use syntax::{ast, attr};
+use syntax::edition::Edition;
 use syntax::symbol::Symbol;
 use syntax_pos::{self, Span};
 
@@ -189,6 +190,7 @@ pub struct CrateRoot {
     pub hash: hir::svh::Svh,
     pub disambiguator: CrateDisambiguator,
     pub panic_strategy: PanicStrategy,
+    pub edition: Edition,
     pub has_global_allocator: bool,
     pub has_default_lib_allocator: bool,
     pub plugin_registrar_fn: Option<DefIndex>,

src/librustc_passes/ast_validation.rs

@@ -21,7 +21,6 @@ use rustc::session::Session;
 use syntax::ast::*;
 use syntax::attr;
 use syntax::codemap::Spanned;
-use syntax::parse::token;
 use syntax::symbol::keywords;
 use syntax::visit::{self, Visitor};
 use syntax_pos::Span;
@@ -40,14 +39,13 @@ impl<'a> AstValidator<'a> {
         let valid_names = [keywords::UnderscoreLifetime.name(),
                            keywords::StaticLifetime.name(),
                            keywords::Invalid.name()];
-        if !valid_names.contains(&ident.name) &&
-            token::is_reserved_ident(ident.without_first_quote()) {
+        if !valid_names.contains(&ident.name) && ident.without_first_quote().is_reserved() {
             self.err_handler().span_err(ident.span, "lifetimes cannot use keyword names");
         }
     }
 
     fn check_label(&self, ident: Ident) {
-        if token::is_reserved_ident(ident.without_first_quote()) {
+        if ident.without_first_quote().is_reserved() {
             self.err_handler()
                 .span_err(ident.span, &format!("invalid label name `{}`", ident.name));
         }

src/librustc_plugin/registry.rs

@@ -15,6 +15,7 @@ use rustc::session::Session;
 
 use syntax::ext::base::{SyntaxExtension, NamedSyntaxExtension, NormalTT, IdentTT};
 use syntax::ext::base::MacroExpanderFn;
+use syntax::ext::hygiene;
 use syntax::symbol::Symbol;
 use syntax::ast;
 use syntax::feature_gate::AttributeType;
@@ -107,15 +108,17 @@ impl<'a> Registry<'a> {
                 def_info: _,
                 allow_internal_unstable,
                 allow_internal_unsafe,
-                unstable_feature
+                unstable_feature,
+                edition,
             } => {
                 let nid = ast::CRATE_NODE_ID;
                 NormalTT {
                     expander,
                     def_info: Some((nid, self.krate_span)),
                     allow_internal_unstable,
                     allow_internal_unsafe,
-                    unstable_feature
+                    unstable_feature,
+                    edition,
                 }
             }
             IdentTT(ext, _, allow_internal_unstable) => {
@@ -150,6 +153,7 @@ impl<'a> Registry<'a> {
             allow_internal_unstable: false,
             allow_internal_unsafe: false,
             unstable_feature: None,
+            edition: hygiene::default_edition(),
         });
     }
 

src/librustc_resolve/build_reduced_graph.rs

@@ -588,7 +588,8 @@ impl<'a> Resolver<'a> {
 
         let ext = Lrc::new(macro_rules::compile(&self.session.parse_sess,
                                                &self.session.features_untracked(),
-                                               &macro_def));
+                                               &macro_def,
+                                               self.cstore.crate_edition_untracked(def_id.krate)));
         self.macro_map.insert(def_id, ext.clone());
         ext
     }

src/librustc_resolve/lib.rs

@@ -58,7 +58,6 @@ use syntax::ast::{Item, ItemKind, ImplItem, ImplItemKind};
 use syntax::ast::{Label, Local, Mutability, Pat, PatKind, Path};
 use syntax::ast::{QSelf, TraitItemKind, TraitRef, Ty, TyKind};
 use syntax::feature_gate::{feature_err, GateIssue};
-use syntax::parse::token;
 use syntax::ptr::P;
 
 use syntax_pos::{Span, DUMMY_SP, MultiSpan};
@@ -3274,7 +3273,7 @@ impl<'a> Resolver<'a> {
                     // `$crate::a::b`
                     module = Some(self.resolve_crate_root(ident.span.ctxt(), true));
                     continue
-                } else if i == 1 && !token::is_path_segment_keyword(ident) {
+                } else if i == 1 && !ident.is_path_segment_keyword() {
                     let prev_name = path[0].name;
                     if prev_name == keywords::Extern.name() ||
                        prev_name == keywords::CrateRoot.name() &&