Skip to content

Commit f00c139

Browse files
committed
Auto merge of #110050 - saethlin:better-u32-encoding, r=nnethercote
Use a specialized varint + bitpacking scheme for DepGraph encoding The previous scheme here uses leb128 to encode the edge tables that represent the incr comp dependency graph. The problem with that scheme is that leb128 has overhead for larger values, and generally relies on the distribution of encoded values being heavily skewed towards smaller values. That is definitely not the case for a dep node index, since they are handed out sequentially and the whole range is covered, the distribution is actually biased in the opposite direction: Most dep nodes are large. This PR implements a different varint encoding scheme. Instead of applying varint encoding to individual dep node indices (which is extremely branchy) we now apply it per node. While being built, each node now stores its edges in a `SmallVec` with a bit of extra logic to track the max value of each edge. Then we varint encode the whole batch. This is a gamble: We save on space by only claiming 2 bits per node instead of ~3 bits per edge which is a nice savings but needs to balance out with the space overhead that a single large index in a node with a lot of edges will encode unnecessary bytes in each of that node's edge indices. Then, to keep the runtime overhead of this encoding scheme down we deserialize our indices by loading 4 bytes for each then masking off the bytes that are't ours. This is much less code and branches than leb128, but relies on having some readable bytes past the end of each edge list. We explicitly add such padding to the in-memory data during decoding. And we also do this decoding lazily, turning a dense on-disk encoding into a peak memory reduction. Then we apply a bit-packing scheme; since in #115391 we now have unused bits on `DepKind`, we use those unused bits (currently there are 7!) to store the 2 bits that we need for the byte width of the edges in each node, then use the remaining bits to store the length of the edge list, if it fits. r? `@nnethercote`
2 parents 4e5b31c + 469dc8f commit f00c139

File tree

8 files changed

+399
-45
lines changed

8 files changed

+399
-45
lines changed

compiler/rustc_middle/src/dep_graph/dep_node.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ macro_rules! define_dep_nodes {
9797
// discriminants of the variants have been assigned consecutively from 0
9898
// so that just the one comparison suffices to check that the u16 can be
9999
// transmuted to a DepKind.
100-
const VARIANTS: u16 = {
100+
pub const VARIANTS: u16 = {
101101
let deps: &[DepKind] = &[$(DepKind::$variant,)*];
102102
let mut i = 0;
103103
while i < deps.len() {

compiler/rustc_middle/src/dep_graph/mod.rs

+16
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub type DepKindStruct<'tcx> = rustc_query_system::dep_graph::DepKindStruct<TyCt
2626
impl rustc_query_system::dep_graph::DepKind for DepKind {
2727
const NULL: Self = DepKind::Null;
2828
const RED: Self = DepKind::Red;
29+
const MAX: u16 = DepKind::VARIANTS - 1;
2930

3031
fn debug_node(node: &DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3132
write!(f, "{:?}(", node.kind)?;
@@ -68,6 +69,21 @@ impl rustc_query_system::dep_graph::DepKind for DepKind {
6869
op(icx.task_deps)
6970
})
7071
}
72+
73+
#[track_caller]
74+
#[inline]
75+
fn from_u16(u: u16) -> Self {
76+
if u > Self::MAX {
77+
panic!("Invalid DepKind {u}");
78+
}
79+
// SAFETY: See comment on DepKind::VARIANTS
80+
unsafe { std::mem::transmute(u) }
81+
}
82+
83+
#[inline]
84+
fn to_u16(self) -> u16 {
85+
self as u16
86+
}
7187
}
7288

7389
impl<'tcx> DepContext for TyCtxt<'tcx> {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
use crate::dep_graph::DepNodeIndex;
2+
use smallvec::SmallVec;
3+
use std::hash::{Hash, Hasher};
4+
use std::iter::Extend;
5+
use std::ops::Deref;
6+
7+
#[derive(Default, Debug)]
8+
pub struct EdgesVec {
9+
max: u32,
10+
edges: SmallVec<[DepNodeIndex; EdgesVec::INLINE_CAPACITY]>,
11+
}
12+
13+
impl Hash for EdgesVec {
14+
#[inline]
15+
fn hash<H: Hasher>(&self, hasher: &mut H) {
16+
Hash::hash(&self.edges, hasher)
17+
}
18+
}
19+
20+
impl EdgesVec {
21+
pub const INLINE_CAPACITY: usize = 8;
22+
23+
#[inline]
24+
pub fn new() -> Self {
25+
Self::default()
26+
}
27+
28+
#[inline]
29+
pub fn push(&mut self, edge: DepNodeIndex) {
30+
self.max = self.max.max(edge.as_u32());
31+
self.edges.push(edge);
32+
}
33+
34+
#[inline]
35+
pub fn max_index(&self) -> u32 {
36+
self.max
37+
}
38+
}
39+
40+
impl Deref for EdgesVec {
41+
type Target = [DepNodeIndex];
42+
43+
#[inline]
44+
fn deref(&self) -> &Self::Target {
45+
self.edges.as_slice()
46+
}
47+
}
48+
49+
impl FromIterator<DepNodeIndex> for EdgesVec {
50+
#[inline]
51+
fn from_iter<T>(iter: T) -> Self
52+
where
53+
T: IntoIterator<Item = DepNodeIndex>,
54+
{
55+
let mut vec = EdgesVec::new();
56+
for index in iter {
57+
vec.push(index)
58+
}
59+
vec
60+
}
61+
}
62+
63+
impl Extend<DepNodeIndex> for EdgesVec {
64+
#[inline]
65+
fn extend<T>(&mut self, iter: T)
66+
where
67+
T: IntoIterator<Item = DepNodeIndex>,
68+
{
69+
for elem in iter {
70+
self.push(elem);
71+
}
72+
}
73+
}

compiler/rustc_query_system/src/dep_graph/graph.rs

+10-15
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering};
88
use rustc_data_structures::unord::UnordMap;
99
use rustc_index::IndexVec;
1010
use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
11-
use smallvec::{smallvec, SmallVec};
1211
use std::assert_matches::assert_matches;
1312
use std::collections::hash_map::Entry;
1413
use std::fmt::Debug;
@@ -19,6 +18,7 @@ use std::sync::atomic::Ordering::Relaxed;
1918
use super::query::DepGraphQuery;
2019
use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
2120
use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId};
21+
use crate::dep_graph::EdgesVec;
2222
use crate::ich::StableHashingContext;
2323
use crate::query::{QueryContext, QuerySideEffects};
2424

@@ -137,7 +137,7 @@ impl<K: DepKind> DepGraph<K> {
137137
let _green_node_index = current.intern_new_node(
138138
profiler,
139139
DepNode { kind: DepKind::NULL, hash: current.anon_id_seed.into() },
140-
smallvec![],
140+
EdgesVec::new(),
141141
Fingerprint::ZERO,
142142
);
143143
assert_eq!(_green_node_index, DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE);
@@ -147,7 +147,7 @@ impl<K: DepKind> DepGraph<K> {
147147
profiler,
148148
&prev_graph,
149149
DepNode { kind: DepKind::RED, hash: Fingerprint::ZERO.into() },
150-
smallvec![],
150+
EdgesVec::new(),
151151
None,
152152
false,
153153
);
@@ -356,12 +356,12 @@ impl<K: DepKind> DepGraphData<K> {
356356

357357
let with_deps = |task_deps| K::with_deps(task_deps, || task(cx, arg));
358358
let (result, edges) = if cx.dep_context().is_eval_always(key.kind) {
359-
(with_deps(TaskDepsRef::EvalAlways), smallvec![])
359+
(with_deps(TaskDepsRef::EvalAlways), EdgesVec::new())
360360
} else {
361361
let task_deps = Lock::new(TaskDeps {
362362
#[cfg(debug_assertions)]
363363
node: Some(key),
364-
reads: SmallVec::new(),
364+
reads: EdgesVec::new(),
365365
read_set: Default::default(),
366366
phantom_data: PhantomData,
367367
});
@@ -486,14 +486,14 @@ impl<K: DepKind> DepGraph<K> {
486486

487487
// As long as we only have a low number of reads we can avoid doing a hash
488488
// insert and potentially allocating/reallocating the hashmap
489-
let new_read = if task_deps.reads.len() < TASK_DEPS_READS_CAP {
489+
let new_read = if task_deps.reads.len() < EdgesVec::INLINE_CAPACITY {
490490
task_deps.reads.iter().all(|other| *other != dep_node_index)
491491
} else {
492492
task_deps.read_set.insert(dep_node_index)
493493
};
494494
if new_read {
495495
task_deps.reads.push(dep_node_index);
496-
if task_deps.reads.len() == TASK_DEPS_READS_CAP {
496+
if task_deps.reads.len() == EdgesVec::INLINE_CAPACITY {
497497
// Fill `read_set` with what we have so far so we can use the hashset
498498
// next time
499499
task_deps.read_set.extend(task_deps.reads.iter().copied());
@@ -572,7 +572,7 @@ impl<K: DepKind> DepGraph<K> {
572572
}
573573
}
574574

575-
let mut edges = SmallVec::new();
575+
let mut edges = EdgesVec::new();
576576
K::read_deps(|task_deps| match task_deps {
577577
TaskDepsRef::Allow(deps) => edges.extend(deps.lock().reads.iter().copied()),
578578
TaskDepsRef::EvalAlways => {
@@ -872,7 +872,7 @@ impl<K: DepKind> DepGraphData<K> {
872872

873873
let prev_deps = self.previous.edge_targets_from(prev_dep_node_index);
874874

875-
for &dep_dep_node_index in prev_deps {
875+
for dep_dep_node_index in prev_deps {
876876
self.try_mark_parent_green(qcx, dep_dep_node_index, dep_node, Some(&frame))?;
877877
}
878878

@@ -1308,8 +1308,7 @@ impl<K: DepKind> CurrentDepGraph<K> {
13081308
let key = prev_graph.index_to_node(prev_index);
13091309
let edges = prev_graph
13101310
.edge_targets_from(prev_index)
1311-
.iter()
1312-
.map(|i| prev_index_to_index[*i].unwrap())
1311+
.map(|i| prev_index_to_index[i].unwrap())
13131312
.collect();
13141313
let fingerprint = prev_graph.fingerprint_by_index(prev_index);
13151314
let dep_node_index = self.encoder.borrow().send(profiler, key, fingerprint, edges);
@@ -1335,10 +1334,6 @@ impl<K: DepKind> CurrentDepGraph<K> {
13351334
}
13361335
}
13371336

1338-
/// The capacity of the `reads` field `SmallVec`
1339-
const TASK_DEPS_READS_CAP: usize = 8;
1340-
type EdgesVec = SmallVec<[DepNodeIndex; TASK_DEPS_READS_CAP]>;
1341-
13421337
#[derive(Debug, Clone, Copy)]
13431338
pub enum TaskDepsRef<'a, K: DepKind> {
13441339
/// New dependencies can be added to the

compiler/rustc_query_system/src/dep_graph/mod.rs

+8
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
pub mod debug;
22
mod dep_node;
3+
mod edges;
34
mod graph;
45
mod query;
56
mod serialized;
67

78
pub use dep_node::{DepKindStruct, DepNode, DepNodeParams, WorkProductId};
9+
pub use edges::EdgesVec;
810
pub use graph::{
911
hash_result, DepGraph, DepGraphData, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef,
1012
WorkProduct, WorkProductMap,
@@ -157,4 +159,10 @@ pub trait DepKind: Copy + fmt::Debug + Eq + Hash + Send + Encodable<FileEncoder>
157159
fn read_deps<OP>(op: OP)
158160
where
159161
OP: for<'a> FnOnce(TaskDepsRef<'a, Self>);
162+
163+
fn from_u16(u: u16) -> Self;
164+
165+
fn to_u16(self) -> u16;
166+
167+
const MAX: u16;
160168
}

0 commit comments

Comments
 (0)