diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs index 220118ae5ccb8..6b7d49a2d8b9c 100644 --- a/compiler/rustc_middle/src/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/query/on_disk_cache.rs @@ -1,16 +1,14 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::stable_hasher::Hash64; -use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, RwLock}; +use rustc_data_structures::sync::{Lock, Lrc, RwLock}; use rustc_data_structures::unhash::UnhashMap; -use rustc_data_structures::unord::UnordSet; -use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE}; -use rustc_hir::definitions::DefPathHash; -use rustc_index::{Idx, IndexVec}; +use rustc_hir::def_id::{CrateNum, DefId, DefIndex, StableCrateId, LOCAL_CRATE}; +use rustc_index::Idx; use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; -use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState}; -use rustc_middle::mir::{self, interpret}; -use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder}; +use rustc_middle::mir::interpret; +use rustc_middle::mir::interpret::AllocDecodingState; +use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_query_system::query::QuerySideEffects; use rustc_serialize::{ @@ -18,31 +16,28 @@ use rustc_serialize::{ Decodable, Decoder, Encodable, Encoder, }; use rustc_session::Session; -use rustc_span::hygiene::{ - ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData, -}; +use rustc_span::hygiene::{ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext}; use rustc_span::source_map::{SourceMap, StableSourceFileId}; -use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span}; use rustc_span::{CachingSourceMapView, Symbol}; +use rustc_span::{ExpnHash, Pos, SourceFile, Span}; use std::collections::hash_map::Entry; use std::io; -use std::mem; const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE; // A normal span encoded with both location information and a `SyntaxContext` -const TAG_FULL_SPAN: u8 = 0; +pub const TAG_FULL_SPAN: u8 = 0; // A partial span with no location information, encoded only with a `SyntaxContext` -const TAG_PARTIAL_SPAN: u8 = 1; -const TAG_RELATIVE_SPAN: u8 = 2; +pub const TAG_PARTIAL_SPAN: u8 = 1; +pub const TAG_RELATIVE_SPAN: u8 = 2; -const TAG_SYNTAX_CONTEXT: u8 = 0; -const TAG_EXPN_DATA: u8 = 1; +pub const TAG_SYNTAX_CONTEXT: u8 = 0; +pub const TAG_EXPN_DATA: u8 = 1; // Tags for encoding Symbol's -const SYMBOL_STR: u8 = 0; -const SYMBOL_OFFSET: u8 = 1; -const SYMBOL_PREINTERNED: u8 = 2; +pub const SYMBOL_STR: u8 = 0; +pub const SYMBOL_OFFSET: u8 = 1; +pub const SYMBOL_PREINTERNED: u8 = 2; /// Provides an interface to incremental compilation data cached from the /// previous compilation session. This data will eventually include the results @@ -50,34 +45,34 @@ const SYMBOL_PREINTERNED: u8 = 2; /// any side effects that have been emitted during a query. pub struct OnDiskCache<'sess> { // The complete cache data in serialized form. - serialized_data: RwLock>, + pub serialized_data: RwLock>, // Collects all `QuerySideEffects` created during the current compilation // session. current_side_effects: Lock>, - source_map: &'sess SourceMap, - file_index_to_stable_id: FxHashMap, + pub source_map: &'sess SourceMap, + pub file_index_to_stable_id: FxHashMap, // Caches that are populated lazily during decoding. - file_index_to_file: Lock>>, + pub file_index_to_file: Lock>>, // A map from dep-node to the position of the cached query result in // `serialized_data`. - query_result_index: FxHashMap, + pub query_result_index: FxHashMap, // A map from dep-node to the position of any associated `QuerySideEffects` in // `serialized_data`. - prev_side_effects_index: FxHashMap, + pub prev_side_effects_index: FxHashMap, - alloc_decoding_state: AllocDecodingState, + pub alloc_decoding_state: AllocDecodingState, // A map from syntax context ids to the position of their associated // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext` // to represent the fact that we are storing *encoded* ids. When we decode // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`, // which will almost certainly be different than the serialized id. - syntax_contexts: FxHashMap, + pub syntax_contexts: FxHashMap, // A map from the `DefPathHash` of an `ExpnId` to the position // of their associated `ExpnData`. Ideally, we would store a `DefId`, // but we need to decode this before we've constructed a `TyCtxt` (which @@ -87,14 +82,14 @@ pub struct OnDiskCache<'sess> { // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively, // we could look up the `ExpnData` from the metadata of foreign crates, // but it seemed easier to have `OnDiskCache` be independent of the `CStore`. - expn_data: UnhashMap, + pub expn_data: UnhashMap, // Additional information used when decoding hygiene data. - hygiene_context: HygieneDecodeContext, + pub hygiene_context: HygieneDecodeContext, // Maps `ExpnHash`es to their raw value from the *previous* // compilation session. This is used as an initial 'guess' when // we try to map an `ExpnHash` to its value in the current // compilation session. - foreign_expn_data: UnhashMap, + pub foreign_expn_data: UnhashMap, } // This type is used only for serialization and deserialization. @@ -115,7 +110,7 @@ struct Footer { pub type EncodedDepNodeIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)] -struct SourceFileIndex(u32); +pub struct SourceFileIndex(u32); #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)] pub struct AbsoluteBytePos(u64); @@ -127,7 +122,7 @@ impl AbsoluteBytePos { } #[inline] - fn to_usize(self) -> usize { + pub fn to_usize(self) -> usize { self.0 as usize } } @@ -138,14 +133,14 @@ impl AbsoluteBytePos { /// without any additional context, i.e. with a simple `opaque::Decoder` (which /// is the only thing available when decoding the cache's [Footer]. #[derive(Encodable, Decodable, Clone, Debug)] -struct EncodedSourceFileId { +pub struct EncodedSourceFileId { file_name_hash: Hash64, stable_crate_id: StableCrateId, } impl EncodedSourceFileId { #[inline] - fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId { + pub fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId { let cnum = tcx.stable_crate_id_to_crate_num(self.stable_crate_id); StableSourceFileId { file_name_hash: self.file_name_hash, cnum } } @@ -361,18 +356,6 @@ impl<'sess> OnDiskCache<'sess> { }) } - /// Loads a `QuerySideEffects` created during the previous compilation session. - pub fn load_side_effects( - &self, - tcx: TyCtxt<'_>, - dep_node_index: SerializedDepNodeIndex, - ) -> QuerySideEffects { - let side_effects: Option = - self.load_indexed(tcx, dep_node_index, &self.prev_side_effects_index); - - side_effects.unwrap_or_default() - } - /// Stores a `QuerySideEffects` emitted during the current compilation session. /// Anything stored like this will be available via `load_side_effects` in /// the next compilation session. @@ -389,21 +372,6 @@ impl<'sess> OnDiskCache<'sess> { // with_decoder is infallible, so we can stop here } - /// Returns the cached query result if there is something in the cache for - /// the given `SerializedDepNodeIndex`; otherwise returns `None`. - pub fn try_load_query_result<'tcx, T>( - &self, - tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, - ) -> Option - where - T: for<'a> Decodable>, - { - let opt_value = self.load_indexed(tcx, dep_node_index, &self.query_result_index); - debug_assert_eq!(opt_value.is_some(), self.loadable_from_disk(dep_node_index)); - opt_value - } - /// Stores side effect emitted during computation of an anonymous query. /// Since many anonymous queries can share the same `DepNode`, we aggregate /// them -- as opposed to regular queries where we assume that there is a @@ -418,106 +386,11 @@ impl<'sess> OnDiskCache<'sess> { let x = current_side_effects.entry(dep_node_index).or_default(); x.append(side_effects); } - - fn load_indexed<'tcx, T>( - &self, - tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, - index: &FxHashMap, - ) -> Option - where - T: for<'a> Decodable>, - { - let pos = index.get(&dep_node_index).cloned()?; - let value = self.with_decoder(tcx, pos, |decoder| decode_tagged(decoder, dep_node_index)); - Some(value) - } - - fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>( - &'sess self, - tcx: TyCtxt<'tcx>, - pos: AbsoluteBytePos, - f: F, - ) -> T - where - T: Decodable>, - { - let serialized_data = self.serialized_data.read(); - let mut decoder = CacheDecoder { - tcx, - opaque: MemDecoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()), - source_map: self.source_map, - file_index_to_file: &self.file_index_to_file, - file_index_to_stable_id: &self.file_index_to_stable_id, - alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(), - syntax_contexts: &self.syntax_contexts, - expn_data: &self.expn_data, - foreign_expn_data: &self.foreign_expn_data, - hygiene_context: &self.hygiene_context, - }; - f(&mut decoder) - } -} - -//- DECODING ------------------------------------------------------------------- - -/// A decoder that can read from the incremental compilation cache. It is similar to the one -/// we use for crate metadata decoding in that it can rebase spans and eventually -/// will also handle things that contain `Ty` instances. -pub struct CacheDecoder<'a, 'tcx> { - tcx: TyCtxt<'tcx>, - opaque: MemDecoder<'a>, - source_map: &'a SourceMap, - file_index_to_file: &'a Lock>>, - file_index_to_stable_id: &'a FxHashMap, - alloc_decoding_session: AllocDecodingSession<'a>, - syntax_contexts: &'a FxHashMap, - expn_data: &'a UnhashMap, - foreign_expn_data: &'a UnhashMap, - hygiene_context: &'a HygieneDecodeContext, -} - -impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { - #[inline] - fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc { - let CacheDecoder { - tcx, - ref file_index_to_file, - ref file_index_to_stable_id, - ref source_map, - .. - } = *self; - - file_index_to_file - .borrow_mut() - .entry(index) - .or_insert_with(|| { - let stable_id = file_index_to_stable_id[&index].translate(tcx); - - // If this `SourceFile` is from a foreign crate, then make sure - // that we've imported all of the source files from that crate. - // This has usually already been done during macro invocation. - // However, when encoding query results like `TypeckResults`, - // we might encode an `AdtDef` for a foreign type (because it - // was referenced in the body of the function). There is no guarantee - // that we will load the source files from that crate during macro - // expansion, so we use `import_source_files` to ensure that the foreign - // source files are actually imported before we call `source_file_by_stable_id`. - if stable_id.cnum != LOCAL_CRATE { - self.tcx.cstore_untracked().import_source_files(self.tcx.sess, stable_id.cnum); - } - - source_map - .source_file_by_stable_id(stable_id) - .expect("failed to lookup `SourceFile` in new context") - }) - .clone() - } } // Decodes something that was encoded with `encode_tagged()` and verify that the // tag matches and the correct amount of bytes was read. -fn decode_tagged(decoder: &mut D, expected_tag: T) -> V +pub fn decode_tagged(decoder: &mut D, expected_tag: T) -> V where T: Decodable + Eq + std::fmt::Debug, V: Decodable, @@ -536,304 +409,6 @@ where value } -impl<'a, 'tcx> TyDecoder for CacheDecoder<'a, 'tcx> { - type I = TyCtxt<'tcx>; - const CLEAR_CROSS_CRATE: bool = false; - - #[inline] - fn interner(&self) -> TyCtxt<'tcx> { - self.tcx - } - - fn cached_ty_for_shorthand(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx> - where - F: FnOnce(&mut Self) -> Ty<'tcx>, - { - let tcx = self.tcx; - - let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand }; - - if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) { - return ty; - } - - let ty = or_insert_with(self); - // This may overwrite the entry, but it should overwrite with the same value. - tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty); - ty - } - - fn with_position(&mut self, pos: usize, f: F) -> R - where - F: FnOnce(&mut Self) -> R, - { - debug_assert!(pos < self.opaque.len()); - - let new_opaque = MemDecoder::new(self.opaque.data(), pos); - let old_opaque = mem::replace(&mut self.opaque, new_opaque); - let r = f(self); - self.opaque = old_opaque; - r - } - - fn decode_alloc_id(&mut self) -> interpret::AllocId { - let alloc_decoding_session = self.alloc_decoding_session; - alloc_decoding_session.decode_alloc_id(self) - } -} - -rustc_middle::implement_ty_decoder!(CacheDecoder<'a, 'tcx>); - -// This ensures that the `Decodable::decode` specialization for `Vec` is used -// when a `CacheDecoder` is passed to `Decodable::decode`. Unfortunately, we have to manually opt -// into specializations this way, given how `CacheDecoder` and the decoding traits currently work. -impl<'a, 'tcx> Decodable> for Vec { - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - Decodable::decode(&mut d.opaque) - } -} - -impl<'a, 'tcx> Decodable> for SyntaxContext { - fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { - let syntax_contexts = decoder.syntax_contexts; - rustc_span::hygiene::decode_syntax_context(decoder, decoder.hygiene_context, |this, id| { - // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing. - // We look up the position of the associated `SyntaxData` and decode it. - let pos = syntax_contexts.get(&id).unwrap(); - this.with_position(pos.to_usize(), |decoder| { - let data: SyntaxContextData = decode_tagged(decoder, TAG_SYNTAX_CONTEXT); - data - }) - }) - } -} - -impl<'a, 'tcx> Decodable> for ExpnId { - fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { - let hash = ExpnHash::decode(decoder); - if hash.is_root() { - return ExpnId::root(); - } - - if let Some(expn_id) = ExpnId::from_hash(hash) { - return expn_id; - } - - let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id()); - - let expn_id = if krate == LOCAL_CRATE { - // We look up the position of the associated `ExpnData` and decode it. - let pos = decoder - .expn_data - .get(&hash) - .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data)); - - let data: ExpnData = decoder - .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA)); - let expn_id = rustc_span::hygiene::register_local_expn_id(data, hash); - - #[cfg(debug_assertions)] - { - use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; - let local_hash = decoder.tcx.with_stable_hashing_context(|mut hcx| { - let mut hasher = StableHasher::new(); - expn_id.expn_data().hash_stable(&mut hcx, &mut hasher); - hasher.finish() - }); - debug_assert_eq!(hash.local_hash(), local_hash); - } - - expn_id - } else { - let index_guess = decoder.foreign_expn_data[&hash]; - decoder.tcx.cstore_untracked().expn_hash_to_expn_id( - decoder.tcx.sess, - krate, - index_guess, - hash, - ) - }; - - debug_assert_eq!(expn_id.krate, krate); - expn_id - } -} - -impl<'a, 'tcx> Decodable> for Span { - fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { - let ctxt = SyntaxContext::decode(decoder); - let parent = Option::::decode(decoder); - let tag: u8 = Decodable::decode(decoder); - - if tag == TAG_PARTIAL_SPAN { - return Span::new(BytePos(0), BytePos(0), ctxt, parent); - } else if tag == TAG_RELATIVE_SPAN { - let dlo = u32::decode(decoder); - let dto = u32::decode(decoder); - - let enclosing = decoder.tcx.source_span_untracked(parent.unwrap()).data_untracked(); - let span = Span::new( - enclosing.lo + BytePos::from_u32(dlo), - enclosing.lo + BytePos::from_u32(dto), - ctxt, - parent, - ); - - return span; - } else { - debug_assert_eq!(tag, TAG_FULL_SPAN); - } - - let file_lo_index = SourceFileIndex::decode(decoder); - let line_lo = usize::decode(decoder); - let col_lo = BytePos::decode(decoder); - let len = BytePos::decode(decoder); - - let file_lo = decoder.file_index_to_file(file_lo_index); - let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo); - let hi = lo + len; - - Span::new(lo, hi, ctxt, parent) - } -} - -// copy&paste impl from rustc_metadata -impl<'a, 'tcx> Decodable> for Symbol { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - let tag = d.read_u8(); - - match tag { - SYMBOL_STR => { - let s = d.read_str(); - Symbol::intern(s) - } - SYMBOL_OFFSET => { - // read str offset - let pos = d.read_usize(); - - // move to str offset and read - d.opaque.with_position(pos, |d| { - let s = d.read_str(); - Symbol::intern(s) - }) - } - SYMBOL_PREINTERNED => { - let symbol_index = d.read_u32(); - Symbol::new_from_decoded(symbol_index) - } - _ => unreachable!(), - } - } -} - -impl<'a, 'tcx> Decodable> for CrateNum { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - let stable_id = StableCrateId::decode(d); - let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id); - cnum - } -} - -// This impl makes sure that we get a runtime error when we try decode a -// `DefIndex` that is not contained in a `DefId`. Such a case would be problematic -// because we would not know how to transform the `DefIndex` to the current -// context. -impl<'a, 'tcx> Decodable> for DefIndex { - fn decode(_d: &mut CacheDecoder<'a, 'tcx>) -> DefIndex { - panic!("trying to decode `DefIndex` outside the context of a `DefId`") - } -} - -// Both the `CrateNum` and the `DefIndex` of a `DefId` can change in between two -// compilation sessions. We use the `DefPathHash`, which is stable across -// sessions, to map the old `DefId` to the new one. -impl<'a, 'tcx> Decodable> for DefId { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - // Load the `DefPathHash` which is was we encoded the `DefId` as. - let def_path_hash = DefPathHash::decode(d); - - // Using the `DefPathHash`, we can lookup the new `DefId`. - // Subtle: We only encode a `DefId` as part of a query result. - // If we get to this point, then all of the query inputs were green, - // which means that the definition with this hash is guaranteed to - // still exist in the current compilation session. - d.tcx.def_path_hash_to_def_id(def_path_hash, &mut || { - panic!("Failed to convert DefPathHash {def_path_hash:?}") - }) - } -} - -impl<'a, 'tcx> Decodable> for &'tcx UnordSet { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - RefDecodable::decode(d) - } -} - -impl<'a, 'tcx> Decodable> - for &'tcx FxHashMap>> -{ - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - RefDecodable::decode(d) - } -} - -impl<'a, 'tcx> Decodable> - for &'tcx IndexVec> -{ - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - RefDecodable::decode(d) - } -} - -impl<'a, 'tcx> Decodable> for &'tcx [(ty::Predicate<'tcx>, Span)] { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - RefDecodable::decode(d) - } -} - -impl<'a, 'tcx> Decodable> for &'tcx [(ty::Clause<'tcx>, Span)] { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - RefDecodable::decode(d) - } -} - -impl<'a, 'tcx> Decodable> for &'tcx [rustc_ast::InlineAsmTemplatePiece] { - #[inline] - fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { - RefDecodable::decode(d) - } -} - -macro_rules! impl_ref_decoder { - (<$tcx:tt> $($ty:ty,)*) => { - $(impl<'a, $tcx> Decodable> for &$tcx [$ty] { - #[inline] - fn decode(d: &mut CacheDecoder<'a, $tcx>) -> Self { - RefDecodable::decode(d) - } - })* - }; -} - -impl_ref_decoder! {<'tcx> - Span, - rustc_ast::Attribute, - rustc_span::symbol::Ident, - ty::Variance, - rustc_span::def_id::DefId, - rustc_span::def_id::LocalDefId, - (rustc_middle::middle::exported_symbols::ExportedSymbol<'tcx>, rustc_middle::middle::exported_symbols::SymbolExportInfo), - ty::DeducedParamAttrs, -} - //- ENCODING ------------------------------------------------------------------- /// An encoder that can write to the incremental compilation cache. diff --git a/compiler/rustc_query_impl/src/cache_decoder.rs b/compiler/rustc_query_impl/src/cache_decoder.rs new file mode 100644 index 0000000000000..7c6fe96fc3881 --- /dev/null +++ b/compiler/rustc_query_impl/src/cache_decoder.rs @@ -0,0 +1,446 @@ +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::{HashMapExt, Lock, Lrc}; +use rustc_data_structures::unhash::UnhashMap; +use rustc_data_structures::unord::UnordSet; +use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE}; +use rustc_hir::definitions::DefPathHash; +use rustc_index::IndexVec; +use rustc_middle::dep_graph::SerializedDepNodeIndex; +use rustc_middle::mir::interpret::AllocDecodingSession; +use rustc_middle::mir::{self, interpret}; +use rustc_middle::query::on_disk_cache::{ + decode_tagged, AbsoluteBytePos, EncodedSourceFileId, OnDiskCache, SourceFileIndex, + SYMBOL_OFFSET, SYMBOL_PREINTERNED, SYMBOL_STR, TAG_EXPN_DATA, TAG_FULL_SPAN, TAG_PARTIAL_SPAN, + TAG_RELATIVE_SPAN, TAG_SYNTAX_CONTEXT, +}; +use rustc_middle::ty::codec::{RefDecodable, TyDecoder}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_query_system::query::QuerySideEffects; +use rustc_serialize::{opaque::MemDecoder, Decodable, Decoder}; +use rustc_span::hygiene::{ExpnId, HygieneDecodeContext, SyntaxContext, SyntaxContextData}; +use rustc_span::source_map::SourceMap; +use rustc_span::Symbol; +use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span}; +use std::mem; + +/// Loads a `QuerySideEffects` created during the previous compilation session. +pub fn load_side_effects( + on_disk_cache: &OnDiskCache<'_>, + tcx: TyCtxt<'_>, + dep_node_index: SerializedDepNodeIndex, +) -> QuerySideEffects { + let side_effects: Option = + load_indexed(on_disk_cache, tcx, dep_node_index, &on_disk_cache.prev_side_effects_index); + + side_effects.unwrap_or_default() +} + +/// Returns the cached query result if there is something in the cache for +/// the given `SerializedDepNodeIndex`; otherwise returns `None`. +pub fn try_load_query_result<'tcx, T>( + on_disk_cache: &OnDiskCache<'_>, + tcx: TyCtxt<'tcx>, + dep_node_index: SerializedDepNodeIndex, +) -> Option +where + T: for<'a> Decodable>, +{ + let opt_value = + load_indexed(on_disk_cache, tcx, dep_node_index, &on_disk_cache.query_result_index); + debug_assert_eq!(opt_value.is_some(), on_disk_cache.loadable_from_disk(dep_node_index)); + opt_value +} + +fn load_indexed<'tcx, T>( + on_disk_cache: &OnDiskCache<'_>, + tcx: TyCtxt<'tcx>, + dep_node_index: SerializedDepNodeIndex, + index: &FxHashMap, +) -> Option +where + T: for<'a> Decodable>, +{ + let pos = index.get(&dep_node_index).cloned()?; + let value = + with_decoder(on_disk_cache, tcx, pos, |decoder| decode_tagged(decoder, dep_node_index)); + Some(value) +} + +fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>( + on_disk_cache: &OnDiskCache<'_>, + tcx: TyCtxt<'tcx>, + pos: AbsoluteBytePos, + f: F, +) -> T +where + T: Decodable>, +{ + let serialized_data = on_disk_cache.serialized_data.read(); + let mut decoder = CacheDecoder { + tcx, + opaque: MemDecoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()), + source_map: on_disk_cache.source_map, + file_index_to_file: &on_disk_cache.file_index_to_file, + file_index_to_stable_id: &on_disk_cache.file_index_to_stable_id, + alloc_decoding_session: on_disk_cache.alloc_decoding_state.new_decoding_session(), + syntax_contexts: &on_disk_cache.syntax_contexts, + expn_data: &on_disk_cache.expn_data, + foreign_expn_data: &on_disk_cache.foreign_expn_data, + hygiene_context: &on_disk_cache.hygiene_context, + }; + f(&mut decoder) +} + +//- DECODING ------------------------------------------------------------------- + +/// A decoder that can read from the incremental compilation cache. It is similar to the one +/// we use for crate metadata decoding in that it can rebase spans and eventually +/// will also handle things that contain `Ty` instances. +pub struct CacheDecoder<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + opaque: MemDecoder<'a>, + source_map: &'a SourceMap, + file_index_to_file: &'a Lock>>, + file_index_to_stable_id: &'a FxHashMap, + alloc_decoding_session: AllocDecodingSession<'a>, + syntax_contexts: &'a FxHashMap, + expn_data: &'a UnhashMap, + foreign_expn_data: &'a UnhashMap, + hygiene_context: &'a HygieneDecodeContext, +} + +impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { + #[inline] + fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc { + let CacheDecoder { + tcx, + ref file_index_to_file, + ref file_index_to_stable_id, + ref source_map, + .. + } = *self; + + file_index_to_file + .borrow_mut() + .entry(index) + .or_insert_with(|| { + let stable_id = file_index_to_stable_id[&index].translate(tcx); + + // If this `SourceFile` is from a foreign crate, then make sure + // that we've imported all of the source files from that crate. + // This has usually already been done during macro invocation. + // However, when encoding query results like `TypeckResults`, + // we might encode an `AdtDef` for a foreign type (because it + // was referenced in the body of the function). There is no guarantee + // that we will load the source files from that crate during macro + // expansion, so we use `import_source_files` to ensure that the foreign + // source files are actually imported before we call `source_file_by_stable_id`. + if stable_id.cnum != LOCAL_CRATE { + self.tcx.cstore_untracked().import_source_files(self.tcx.sess, stable_id.cnum); + } + + source_map + .source_file_by_stable_id(stable_id) + .expect("failed to lookup `SourceFile` in new context") + }) + .clone() + } +} + +impl<'a, 'tcx> TyDecoder for CacheDecoder<'a, 'tcx> { + type I = TyCtxt<'tcx>; + const CLEAR_CROSS_CRATE: bool = false; + + #[inline] + fn interner(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn cached_ty_for_shorthand(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx> + where + F: FnOnce(&mut Self) -> Ty<'tcx>, + { + let tcx = self.tcx; + + let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand }; + + if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) { + return ty; + } + + let ty = or_insert_with(self); + // This may overwrite the entry, but it should overwrite with the same value. + tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty); + ty + } + + fn with_position(&mut self, pos: usize, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + debug_assert!(pos < self.opaque.len()); + + let new_opaque = MemDecoder::new(self.opaque.data(), pos); + let old_opaque = mem::replace(&mut self.opaque, new_opaque); + let r = f(self); + self.opaque = old_opaque; + r + } + + fn decode_alloc_id(&mut self) -> interpret::AllocId { + let alloc_decoding_session = self.alloc_decoding_session; + alloc_decoding_session.decode_alloc_id(self) + } +} + +rustc_middle::implement_ty_decoder!(CacheDecoder<'a, 'tcx>); + +// This ensures that the `Decodable::decode` specialization for `Vec` is used +// when a `CacheDecoder` is passed to `Decodable::decode`. Unfortunately, we have to manually opt +// into specializations this way, given how `CacheDecoder` and the decoding traits currently work. +impl<'a, 'tcx> Decodable> for Vec { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + Decodable::decode(&mut d.opaque) + } +} + +impl<'a, 'tcx> Decodable> for SyntaxContext { + fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { + let syntax_contexts = decoder.syntax_contexts; + rustc_span::hygiene::decode_syntax_context(decoder, decoder.hygiene_context, |this, id| { + // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing. + // We look up the position of the associated `SyntaxData` and decode it. + let pos = syntax_contexts.get(&id).unwrap(); + this.with_position(pos.to_usize(), |decoder| { + let data: SyntaxContextData = decode_tagged(decoder, TAG_SYNTAX_CONTEXT); + data + }) + }) + } +} + +impl<'a, 'tcx> Decodable> for ExpnId { + fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { + let hash = ExpnHash::decode(decoder); + if hash.is_root() { + return ExpnId::root(); + } + + if let Some(expn_id) = ExpnId::from_hash(hash) { + return expn_id; + } + + let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id()); + + let expn_id = if krate == LOCAL_CRATE { + // We look up the position of the associated `ExpnData` and decode it. + let pos = decoder + .expn_data + .get(&hash) + .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data)); + + let data: ExpnData = decoder + .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA)); + let expn_id = rustc_span::hygiene::register_local_expn_id(data, hash); + + #[cfg(debug_assertions)] + { + use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; + let local_hash = decoder.tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + expn_id.expn_data().hash_stable(&mut hcx, &mut hasher); + hasher.finish() + }); + debug_assert_eq!(hash.local_hash(), local_hash); + } + + expn_id + } else { + let index_guess = decoder.foreign_expn_data[&hash]; + decoder.tcx.cstore_untracked().expn_hash_to_expn_id( + decoder.tcx.sess, + krate, + index_guess, + hash, + ) + }; + + debug_assert_eq!(expn_id.krate, krate); + expn_id + } +} + +impl<'a, 'tcx> Decodable> for Span { + fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { + let ctxt = SyntaxContext::decode(decoder); + let parent = Option::::decode(decoder); + let tag: u8 = Decodable::decode(decoder); + + if tag == TAG_PARTIAL_SPAN { + return Span::new(BytePos(0), BytePos(0), ctxt, parent); + } else if tag == TAG_RELATIVE_SPAN { + let dlo = u32::decode(decoder); + let dto = u32::decode(decoder); + + let enclosing = decoder.tcx.source_span_untracked(parent.unwrap()).data_untracked(); + let span = Span::new( + enclosing.lo + BytePos::from_u32(dlo), + enclosing.lo + BytePos::from_u32(dto), + ctxt, + parent, + ); + + return span; + } else { + debug_assert_eq!(tag, TAG_FULL_SPAN); + } + + let file_lo_index = SourceFileIndex::decode(decoder); + let line_lo = usize::decode(decoder); + let col_lo = BytePos::decode(decoder); + let len = BytePos::decode(decoder); + + let file_lo = decoder.file_index_to_file(file_lo_index); + let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo); + let hi = lo + len; + + Span::new(lo, hi, ctxt, parent) + } +} + +// copy&paste impl from rustc_metadata +impl<'a, 'tcx> Decodable> for Symbol { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + let tag = d.read_u8(); + + match tag { + SYMBOL_STR => { + let s = d.read_str(); + Symbol::intern(s) + } + SYMBOL_OFFSET => { + // read str offset + let pos = d.read_usize(); + + // move to str offset and read + d.opaque.with_position(pos, |d| { + let s = d.read_str(); + Symbol::intern(s) + }) + } + SYMBOL_PREINTERNED => { + let symbol_index = d.read_u32(); + Symbol::new_from_decoded(symbol_index) + } + _ => unreachable!(), + } + } +} + +impl<'a, 'tcx> Decodable> for CrateNum { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + let stable_id = StableCrateId::decode(d); + let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id); + cnum + } +} + +// This impl makes sure that we get a runtime error when we try decode a +// `DefIndex` that is not contained in a `DefId`. Such a case would be problematic +// because we would not know how to transform the `DefIndex` to the current +// context. +impl<'a, 'tcx> Decodable> for DefIndex { + fn decode(_d: &mut CacheDecoder<'a, 'tcx>) -> DefIndex { + panic!("trying to decode `DefIndex` outside the context of a `DefId`") + } +} + +// Both the `CrateNum` and the `DefIndex` of a `DefId` can change in between two +// compilation sessions. We use the `DefPathHash`, which is stable across +// sessions, to map the old `DefId` to the new one. +impl<'a, 'tcx> Decodable> for DefId { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + // Load the `DefPathHash` which is was we encoded the `DefId` as. + let def_path_hash = DefPathHash::decode(d); + + // Using the `DefPathHash`, we can lookup the new `DefId`. + // Subtle: We only encode a `DefId` as part of a query result. + // If we get to this point, then all of the query inputs were green, + // which means that the definition with this hash is guaranteed to + // still exist in the current compilation session. + d.tcx.def_path_hash_to_def_id(def_path_hash, &mut || { + panic!("Failed to convert DefPathHash {def_path_hash:?}") + }) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx UnordSet { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> + for &'tcx FxHashMap>> +{ + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> + for &'tcx IndexVec> +{ + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx [(ty::Predicate<'tcx>, Span)] { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx [(ty::Clause<'tcx>, Span)] { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx [rustc_ast::InlineAsmTemplatePiece] { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +macro_rules! impl_ref_decoder { + (<$tcx:tt> $($ty:ty,)*) => { + $(impl<'a, $tcx> Decodable> for &$tcx [$ty] { + #[inline] + fn decode(d: &mut CacheDecoder<'a, $tcx>) -> Self { + RefDecodable::decode(d) + } + })* + }; +} + +impl_ref_decoder! {<'tcx> + Span, + rustc_ast::Attribute, + rustc_span::symbol::Ident, + ty::Variance, + rustc_span::def_id::DefId, + rustc_span::def_id::LocalDefId, + (rustc_middle::middle::exported_symbols::ExportedSymbol<'tcx>, rustc_middle::middle::exported_symbols::SymbolExportInfo), + ty::DeducedParamAttrs, +} diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 4cf0f1305a709..b10d77ab376c7 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -46,6 +46,8 @@ use rustc_span::Span; mod plumbing; pub use crate::plumbing::QueryCtxt; +mod cache_decoder; + mod profiling_support; pub use self::profiling_support::alloc_self_profile_query_strings; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 244f0e84b43d9..23042c755650f 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -2,6 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. +use crate::cache_decoder::{load_side_effects, try_load_query_result, CacheDecoder}; use crate::rustc_middle::dep_graph::DepContext; use crate::rustc_middle::ty::TyEncoder; use crate::QueryConfigRestored; @@ -13,7 +14,7 @@ use rustc_middle::dep_graph::{ self, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex, }; use rustc_middle::query::on_disk_cache::AbsoluteBytePos; -use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder, EncodedDepNodeIndex}; +use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex}; use rustc_middle::query::Key; use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::ty::{self, TyCtxt}; @@ -93,7 +94,7 @@ impl QueryContext for QueryCtxt<'_> { self.query_system .on_disk_cache .as_ref() - .map(|c| c.load_side_effects(self.tcx, prev_dep_node_index)) + .map(|c| load_side_effects(c, self.tcx, prev_dep_node_index)) .unwrap_or_default() } @@ -406,7 +407,7 @@ where // details. let value = tcx .dep_graph - .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index)); + .with_query_deserialization(|| try_load_query_result(on_disk_cache, tcx, prev_index)); prof_timer.finish_with_query_invocation_id(index.into());