Skip to content

Commit de56c29

Browse files
borsbors
committed
Auto merge of rust-lang#95867 - cjgillot:fixed-size, r=oli-obk
Skip `Lazy` for some metadata tables Some metadata tables encode their entries indirectly, through the Lazy construct. This is useful when dealing with variable length encoding, but incurs the extra cost of one u32. Meanwhile, some fields can be encoded in a single u8, or can use a short fixed-length encoding. This PR proposes to do so, and avoid the overhead.
2 parents 327caac + b4cf2cd commit de56c29

File tree

5 files changed

+322
-142
lines changed

5 files changed

+322
-142
lines changed

‎compiler/rustc_metadata/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
#![feature(nll)]
66
#![feature(once_cell)]
77
#![feature(proc_macro_internals)]
8+
#![feature(macro_metavar_expr)]
89
#![feature(min_specialization)]
10+
#![feature(slice_as_chunks)]
911
#![feature(try_blocks)]
1012
#![feature(never_type)]
1113
#![recursion_limit = "256"]

‎compiler/rustc_metadata/src/rmeta/decoder.rs

+42-16
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,18 @@ trait LazyQueryDecodable<'a, 'tcx, T> {
292292
) -> T;
293293
}
294294

295+
impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for T {
296+
fn decode_query(self, _: CrateMetadataRef<'a>, _: TyCtxt<'tcx>, _: impl FnOnce() -> !) -> T {
297+
self
298+
}
299+
}
300+
301+
impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for Option<T> {
302+
fn decode_query(self, _: CrateMetadataRef<'a>, _: TyCtxt<'tcx>, err: impl FnOnce() -> !) -> T {
303+
if let Some(l) = self { l } else { err() }
304+
}
305+
}
306+
295307
impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for Option<Lazy<T>>
296308
where
297309
T: Decodable<DecodeContext<'a, 'tcx>>,
@@ -376,6 +388,17 @@ impl<'a, 'tcx> LazyQueryDecodable<'a, 'tcx, Option<DeprecationEntry>>
376388
}
377389
}
378390

391+
impl<'a, 'tcx> LazyQueryDecodable<'a, 'tcx, Option<DefId>> for Option<RawDefId> {
392+
fn decode_query(
393+
self,
394+
cdata: CrateMetadataRef<'a>,
395+
_: TyCtxt<'tcx>,
396+
_: impl FnOnce() -> !,
397+
) -> Option<DefId> {
398+
self.map(|raw_def_id| raw_def_id.decode(cdata))
399+
}
400+
}
401+
379402
impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
380403
#[inline]
381404
fn tcx(&self) -> TyCtxt<'tcx> {
@@ -394,8 +417,9 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
394417
self.cdata.unwrap()
395418
}
396419

420+
#[inline]
397421
fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
398-
if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] }
422+
self.cdata().map_encoded_cnum_to_current(cnum)
399423
}
400424

401425
fn read_lazy_with_meta<T: ?Sized + LazyMeta>(&mut self, meta: T::Meta) -> Lazy<T> {
@@ -706,8 +730,7 @@ impl<'a, 'tcx, T: Decodable<DecodeContext<'a, 'tcx>>> Decodable<DecodeContext<'a
706730
}
707731
}
708732

709-
impl<'a, 'tcx, I: Idx, T: Decodable<DecodeContext<'a, 'tcx>>> Decodable<DecodeContext<'a, 'tcx>>
710-
for Lazy<Table<I, T>>
733+
impl<'a, 'tcx, I: Idx, T> Decodable<DecodeContext<'a, 'tcx>> for Lazy<Table<I, T>>
711734
where
712735
Option<T>: FixedSizeEncoding,
713736
{
@@ -844,6 +867,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
844867
self.root.tables.kind.get(self, item_id).map(|k| k.decode(self))
845868
}
846869

870+
#[inline]
871+
pub(super) fn map_encoded_cnum_to_current(self, cnum: CrateNum) -> CrateNum {
872+
if cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[cnum] }
873+
}
874+
847875
fn kind(self, item_id: DefIndex) -> EntryKind {
848876
self.maybe_kind(item_id).unwrap_or_else(|| {
849877
bug!(
@@ -856,16 +884,14 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
856884
}
857885

858886
fn def_kind(self, item_id: DefIndex) -> DefKind {
859-
self.root.tables.opt_def_kind.get(self, item_id).map(|k| k.decode(self)).unwrap_or_else(
860-
|| {
861-
bug!(
862-
"CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}",
863-
item_id,
864-
self.root.name,
865-
self.cnum,
866-
)
867-
},
868-
)
887+
self.root.tables.opt_def_kind.get(self, item_id).unwrap_or_else(|| {
888+
bug!(
889+
"CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}",
890+
item_id,
891+
self.root.name,
892+
self.cnum,
893+
)
894+
})
869895
}
870896

871897
fn get_span(self, index: DefIndex, sess: &Session) -> Span {
@@ -1449,9 +1475,9 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
14491475
index: DefIndex,
14501476
def_path_hashes: &mut FxHashMap<DefIndex, DefPathHash>,
14511477
) -> DefPathHash {
1452-
*def_path_hashes.entry(index).or_insert_with(|| {
1453-
self.root.tables.def_path_hashes.get(self, index).unwrap().decode(self)
1454-
})
1478+
*def_path_hashes
1479+
.entry(index)
1480+
.or_insert_with(|| self.root.tables.def_path_hashes.get(self, index).unwrap())
14551481
}
14561482

14571483
#[inline]

‎compiler/rustc_metadata/src/rmeta/encoder.rs

+23-25
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,7 @@ impl<'a, 'tcx, T: Encodable<EncodeContext<'a, 'tcx>>> Encodable<EncodeContext<'a
147147
}
148148
}
149149

150-
impl<'a, 'tcx, I: Idx, T: Encodable<EncodeContext<'a, 'tcx>>> Encodable<EncodeContext<'a, 'tcx>>
151-
for Lazy<Table<I, T>>
150+
impl<'a, 'tcx, I: Idx, T> Encodable<EncodeContext<'a, 'tcx>> for Lazy<Table<I, T>>
152151
where
153152
Option<T>: FixedSizeEncoding,
154153
{
@@ -461,16 +460,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
461460
.chain(self.tcx.resolutions(()).proc_macros.iter().map(|p| p.local_def_index))
462461
{
463462
let def_key = self.lazy(table.def_key(def_index));
464-
let def_path_hash = self.lazy(table.def_path_hash(def_index));
463+
let def_path_hash = table.def_path_hash(def_index);
465464
self.tables.def_keys.set(def_index, def_key);
466465
self.tables.def_path_hashes.set(def_index, def_path_hash);
467466
}
468467
} else {
469468
for (def_index, def_key, def_path_hash) in table.enumerated_keys_and_path_hashes() {
470469
let def_key = self.lazy(def_key);
471-
let def_path_hash = self.lazy(def_path_hash);
472470
self.tables.def_keys.set(def_index, def_key);
473-
self.tables.def_path_hashes.set(def_index, def_path_hash);
471+
self.tables.def_path_hashes.set(def_index, *def_path_hash);
474472
}
475473
}
476474
}
@@ -988,7 +986,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
988986
let def_id = local_id.to_def_id();
989987
let def_kind = tcx.opt_def_kind(local_id);
990988
let Some(def_kind) = def_kind else { continue };
991-
record!(self.tables.opt_def_kind[def_id] <- def_kind);
989+
self.tables.opt_def_kind.set(def_id.index, def_kind);
992990
record!(self.tables.def_span[def_id] <- tcx.def_span(def_id));
993991
record!(self.tables.attributes[def_id] <- tcx.get_attrs(def_id));
994992
record!(self.tables.expn_that_defined[def_id] <- self.tcx.expn_that_defined(def_id));
@@ -1048,7 +1046,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
10481046
};
10491047

10501048
record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data)));
1051-
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
1049+
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
10521050
record!(self.tables.children[def_id] <- variant.fields.iter().map(|f| {
10531051
assert!(f.did.is_local());
10541052
f.did.index
@@ -1078,7 +1076,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
10781076
};
10791077

10801078
record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data)));
1081-
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
1079+
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
10821080
self.encode_item_type(def_id);
10831081
if variant.ctor_kind == CtorKind::Fn {
10841082
record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
@@ -1157,7 +1155,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
11571155
};
11581156

11591157
record!(self.tables.repr_options[def_id] <- adt_def.repr());
1160-
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
1158+
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
11611159
record!(self.tables.kind[def_id] <- EntryKind::Struct(self.lazy(data)));
11621160
self.encode_item_type(def_id);
11631161
if variant.ctor_kind == CtorKind::Fn {
@@ -1207,8 +1205,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
12071205
record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body))
12081206
}
12091207
};
1210-
record!(self.tables.asyncness[def_id] <- m_sig.header.asyncness);
1211-
record!(self.tables.impl_constness[def_id] <- hir::Constness::NotConst);
1208+
self.tables.asyncness.set(def_id.index, m_sig.header.asyncness);
1209+
self.tables.impl_constness.set(def_id.index, hir::Constness::NotConst);
12121210
record!(self.tables.kind[def_id] <- EntryKind::AssocFn(self.lazy(AssocFnData {
12131211
container,
12141212
has_self: trait_item.fn_has_self_parameter,
@@ -1265,15 +1263,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
12651263
}
12661264
ty::AssocKind::Fn => {
12671265
let hir::ImplItemKind::Fn(ref sig, body) = ast_item.kind else { bug!() };
1268-
record!(self.tables.asyncness[def_id] <- sig.header.asyncness);
1266+
self.tables.asyncness.set(def_id.index, sig.header.asyncness);
12691267
record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
12701268
// Can be inside `impl const Trait`, so using sig.header.constness is not reliable
12711269
let constness = if self.tcx.is_const_fn_raw(def_id) {
12721270
hir::Constness::Const
12731271
} else {
12741272
hir::Constness::NotConst
12751273
};
1276-
record!(self.tables.impl_constness[def_id] <- constness);
1274+
self.tables.impl_constness.set(def_id.index, constness);
12771275
record!(self.tables.kind[def_id] <- EntryKind::AssocFn(self.lazy(AssocFnData {
12781276
container,
12791277
has_self: impl_item.fn_has_self_parameter,
@@ -1286,7 +1284,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
12861284
self.encode_ident_span(def_id, impl_item.ident(self.tcx));
12871285
self.encode_item_type(def_id);
12881286
if let Some(trait_item_def_id) = impl_item.trait_item_def_id {
1289-
record!(self.tables.trait_item_def_id[def_id] <- trait_item_def_id);
1287+
self.tables.trait_item_def_id.set(def_id.index, trait_item_def_id.into());
12901288
}
12911289
if impl_item.kind == ty::AssocKind::Fn {
12921290
record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
@@ -1394,9 +1392,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
13941392
EntryKind::Const
13951393
}
13961394
hir::ItemKind::Fn(ref sig, .., body) => {
1397-
record!(self.tables.asyncness[def_id] <- sig.header.asyncness);
1395+
self.tables.asyncness.set(def_id.index, sig.header.asyncness);
13981396
record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
1399-
record!(self.tables.impl_constness[def_id] <- sig.header.constness);
1397+
self.tables.impl_constness.set(def_id.index, sig.header.constness);
14001398
EntryKind::Fn
14011399
}
14021400
hir::ItemKind::Macro(ref macro_def, _) => {
@@ -1420,7 +1418,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
14201418
hir::ItemKind::Struct(ref struct_def, _) => {
14211419
let adt_def = self.tcx.adt_def(def_id);
14221420
record!(self.tables.repr_options[def_id] <- adt_def.repr());
1423-
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
1421+
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
14241422

14251423
// Encode def_ids for each field and method
14261424
// for methods, write all the stuff get_trait_method
@@ -1450,15 +1448,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
14501448
}))
14511449
}
14521450
hir::ItemKind::Impl(hir::Impl { defaultness, constness, .. }) => {
1453-
record!(self.tables.impl_defaultness[def_id] <- defaultness);
1454-
record!(self.tables.impl_constness[def_id] <- constness);
1451+
self.tables.impl_defaultness.set(def_id.index, defaultness);
1452+
self.tables.impl_constness.set(def_id.index, constness);
14551453

14561454
let trait_ref = self.tcx.impl_trait_ref(def_id);
14571455
if let Some(trait_ref) = trait_ref {
14581456
let trait_def = self.tcx.trait_def(trait_ref.def_id);
14591457
if let Some(mut an) = trait_def.ancestors(self.tcx, def_id).ok() {
14601458
if let Some(specialization_graph::Node::Impl(parent)) = an.nth(1) {
1461-
record!(self.tables.impl_parent[def_id] <- parent);
1459+
self.tables.impl_parent.set(def_id.index, parent.into());
14621460
}
14631461
}
14641462

@@ -1472,7 +1470,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
14721470
}
14731471

14741472
let polarity = self.tcx.impl_polarity(def_id);
1475-
record!(self.tables.impl_polarity[def_id] <- polarity);
1473+
self.tables.impl_polarity.set(def_id.index, polarity);
14761474

14771475
EntryKind::Impl
14781476
}
@@ -1644,7 +1642,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
16441642
self.tables.proc_macro_quoted_spans.set(i, span);
16451643
}
16461644

1647-
record!(self.tables.opt_def_kind[LOCAL_CRATE.as_def_id()] <- DefKind::Mod);
1645+
self.tables.opt_def_kind.set(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
16481646
record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id()));
16491647
record!(self.tables.attributes[LOCAL_CRATE.as_def_id()] <- tcx.get_attrs(LOCAL_CRATE.as_def_id()));
16501648
record!(self.tables.visibility[LOCAL_CRATE.as_def_id()] <- tcx.visibility(LOCAL_CRATE.as_def_id()));
@@ -1685,7 +1683,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
16851683
def_key.disambiguated_data.data = DefPathData::MacroNs(name);
16861684

16871685
let def_id = id.to_def_id();
1688-
record!(self.tables.opt_def_kind[def_id] <- DefKind::Macro(macro_kind));
1686+
self.tables.opt_def_kind.set(def_id.index, DefKind::Macro(macro_kind));
16891687
record!(self.tables.kind[def_id] <- EntryKind::ProcMacro(macro_kind));
16901688
record!(self.tables.attributes[def_id] <- attrs);
16911689
record!(self.tables.def_keys[def_id] <- def_key);
@@ -1886,14 +1884,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
18861884

18871885
match nitem.kind {
18881886
hir::ForeignItemKind::Fn(_, ref names, _) => {
1889-
record!(self.tables.asyncness[def_id] <- hir::IsAsync::NotAsync);
1887+
self.tables.asyncness.set(def_id.index, hir::IsAsync::NotAsync);
18901888
record!(self.tables.fn_arg_names[def_id] <- *names);
18911889
let constness = if self.tcx.is_const_fn_raw(def_id) {
18921890
hir::Constness::Const
18931891
} else {
18941892
hir::Constness::NotConst
18951893
};
1896-
record!(self.tables.impl_constness[def_id] <- constness);
1894+
self.tables.impl_constness.set(def_id.index, constness);
18971895
record!(self.tables.kind[def_id] <- EntryKind::ForeignFn);
18981896
}
18991897
hir::ForeignItemKind::Static(..) => {

‎compiler/rustc_metadata/src/rmeta/mod.rs

+33-9
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::creader::CrateMetadataRef;
12
use decoder::Metadata;
23
use def_path_hash_map::DefPathHashMapRef;
34
use table::{Table, TableBuilder};
@@ -8,7 +9,7 @@ use rustc_data_structures::svh::Svh;
89
use rustc_data_structures::sync::MetadataRef;
910
use rustc_hir as hir;
1011
use rustc_hir::def::{CtorKind, DefKind};
11-
use rustc_hir::def_id::{DefId, DefIndex, DefPathHash, StableCrateId};
12+
use rustc_hir::def_id::{CrateNum, DefId, DefIndex, DefPathHash, StableCrateId};
1213
use rustc_hir::definitions::DefKey;
1314
use rustc_hir::lang_items;
1415
use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec};
@@ -237,6 +238,29 @@ crate struct CrateRoot<'tcx> {
237238
symbol_mangling_version: SymbolManglingVersion,
238239
}
239240

241+
/// On-disk representation of `DefId`.
242+
/// This creates a type-safe way to enforce that we remap the CrateNum between the on-disk
243+
/// representation and the compilation session.
244+
#[derive(Copy, Clone)]
245+
crate struct RawDefId {
246+
krate: u32,
247+
index: u32,
248+
}
249+
250+
impl Into<RawDefId> for DefId {
251+
fn into(self) -> RawDefId {
252+
RawDefId { krate: self.krate.as_u32(), index: self.index.as_u32() }
253+
}
254+
}
255+
256+
impl RawDefId {
257+
fn decode(self, cdata: CrateMetadataRef<'_>) -> DefId {
258+
let krate = CrateNum::from_u32(self.krate);
259+
let krate = cdata.map_encoded_cnum_to_current(krate);
260+
DefId { krate, index: DefIndex::from_u32(self.index) }
261+
}
262+
}
263+
240264
#[derive(Encodable, Decodable)]
241265
crate struct CrateDep {
242266
pub name: Symbol,
@@ -286,7 +310,7 @@ define_tables! {
286310
attributes: Table<DefIndex, Lazy<[ast::Attribute]>>,
287311
children: Table<DefIndex, Lazy<[DefIndex]>>,
288312

289-
opt_def_kind: Table<DefIndex, Lazy<DefKind>>,
313+
opt_def_kind: Table<DefIndex, DefKind>,
290314
visibility: Table<DefIndex, Lazy<ty::Visibility>>,
291315
def_span: Table<DefIndex, Lazy<Span>>,
292316
def_ident_span: Table<DefIndex, Lazy<Span>>,
@@ -309,20 +333,20 @@ define_tables! {
309333
mir_for_ctfe: Table<DefIndex, Lazy!(mir::Body<'tcx>)>,
310334
promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>,
311335
thir_abstract_const: Table<DefIndex, Lazy!(&'tcx [thir::abstract_const::Node<'tcx>])>,
312-
impl_parent: Table<DefIndex, Lazy!(DefId)>,
313-
impl_polarity: Table<DefIndex, Lazy!(ty::ImplPolarity)>,
314-
impl_constness: Table<DefIndex, Lazy!(hir::Constness)>,
315-
impl_defaultness: Table<DefIndex, Lazy!(hir::Defaultness)>,
336+
impl_parent: Table<DefIndex, RawDefId>,
337+
impl_polarity: Table<DefIndex, ty::ImplPolarity>,
338+
impl_constness: Table<DefIndex, hir::Constness>,
339+
impl_defaultness: Table<DefIndex, hir::Defaultness>,
316340
// FIXME(eddyb) perhaps compute this on the fly if cheap enough?
317341
coerce_unsized_info: Table<DefIndex, Lazy!(ty::adjustment::CoerceUnsizedInfo)>,
318342
mir_const_qualif: Table<DefIndex, Lazy!(mir::ConstQualifs)>,
319343
rendered_const: Table<DefIndex, Lazy!(String)>,
320-
asyncness: Table<DefIndex, Lazy!(hir::IsAsync)>,
344+
asyncness: Table<DefIndex, hir::IsAsync>,
321345
fn_arg_names: Table<DefIndex, Lazy!([Ident])>,
322346
generator_kind: Table<DefIndex, Lazy!(hir::GeneratorKind)>,
323347
trait_def: Table<DefIndex, Lazy!(ty::TraitDef)>,
324348

325-
trait_item_def_id: Table<DefIndex, Lazy<DefId>>,
349+
trait_item_def_id: Table<DefIndex, RawDefId>,
326350
inherent_impls: Table<DefIndex, Lazy<[DefIndex]>>,
327351
expn_that_defined: Table<DefIndex, Lazy<ExpnId>>,
328352
unused_generic_params: Table<DefIndex, Lazy<FiniteBitSet<u32>>>,
@@ -332,7 +356,7 @@ define_tables! {
332356
// `DefPathTable` up front, since we may only ever use a few
333357
// definitions from any given crate.
334358
def_keys: Table<DefIndex, Lazy<DefKey>>,
335-
def_path_hashes: Table<DefIndex, Lazy<DefPathHash>>,
359+
def_path_hashes: Table<DefIndex, DefPathHash>,
336360
proc_macro_quoted_spans: Table<usize, Lazy<Span>>,
337361
}
338362

0 commit comments

Comments
 (0)