Skip to content

Commit d563814

Browse files
borsbors
committed
Auto merge of #68988 - Zoxc:query-caches, r=eddyb
Add an abstraction for custom query caches r? @eddyb
2 parents 7760cd0 + d924a25 commit d563814

File tree

18 files changed

+691
-416
lines changed

18 files changed

+691
-416
lines changed

‎src/librustc/dep_graph/dep_node.rs

+42-97
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,6 @@ macro_rules! erase {
7676
($x:tt) => {{}};
7777
}
7878

79-
macro_rules! replace {
80-
($x:tt with $($y:tt)*) => ($($y)*)
81-
}
82-
8379
macro_rules! is_anon_attr {
8480
(anon) => {
8581
true
@@ -99,19 +95,18 @@ macro_rules! is_eval_always_attr {
9995
}
10096

10197
macro_rules! contains_anon_attr {
102-
($($attr:ident),*) => ({$(is_anon_attr!($attr) | )* false});
98+
($($attr:ident $(($($attr_args:tt)*))* ),*) => ({$(is_anon_attr!($attr) | )* false});
10399
}
104100

105101
macro_rules! contains_eval_always_attr {
106-
($($attr:ident),*) => ({$(is_eval_always_attr!($attr) | )* false});
102+
($($attr:ident $(($($attr_args:tt)*))* ),*) => ({$(is_eval_always_attr!($attr) | )* false});
107103
}
108104

109105
macro_rules! define_dep_nodes {
110106
(<$tcx:tt>
111107
$(
112-
[$($attr:ident),* ]
108+
[$($attrs:tt)*]
113109
$variant:ident $(( $tuple_arg_ty:ty $(,)? ))*
114-
$({ $($struct_arg_name:ident : $struct_arg_ty:ty),* })*
115110
,)*
116111
) => (
117112
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash,
@@ -126,7 +121,7 @@ macro_rules! define_dep_nodes {
126121
match *self {
127122
$(
128123
DepKind :: $variant => {
129-
if contains_anon_attr!($($attr),*) {
124+
if contains_anon_attr!($($attrs)*) {
130125
return false;
131126
}
132127

@@ -136,13 +131,6 @@ macro_rules! define_dep_nodes {
136131
::CAN_RECONSTRUCT_QUERY_KEY;
137132
})*
138133

139-
// struct args
140-
$({
141-
142-
return <( $($struct_arg_ty,)* ) as DepNodeParams>
143-
::CAN_RECONSTRUCT_QUERY_KEY;
144-
})*
145-
146134
true
147135
}
148136
)*
@@ -152,15 +140,15 @@ macro_rules! define_dep_nodes {
152140
pub fn is_anon(&self) -> bool {
153141
match *self {
154142
$(
155-
DepKind :: $variant => { contains_anon_attr!($($attr),*) }
143+
DepKind :: $variant => { contains_anon_attr!($($attrs)*) }
156144
)*
157145
}
158146
}
159147

160148
pub fn is_eval_always(&self) -> bool {
161149
match *self {
162150
$(
163-
DepKind :: $variant => { contains_eval_always_attr!($($attr), *) }
151+
DepKind :: $variant => { contains_eval_always_attr!($($attrs)*) }
164152
)*
165153
}
166154
}
@@ -176,24 +164,50 @@ macro_rules! define_dep_nodes {
176164
return true;
177165
})*
178166

179-
// struct args
180-
$({
181-
$(erase!($struct_arg_name);)*
182-
return true;
183-
})*
184-
185167
false
186168
}
187169
)*
188170
}
189171
}
190172
}
191173

192-
pub enum DepConstructor<$tcx> {
174+
pub struct DepConstructor;
175+
176+
impl DepConstructor {
193177
$(
194-
$variant $(( $tuple_arg_ty ))*
195-
$({ $($struct_arg_name : $struct_arg_ty),* })*
196-
),*
178+
#[inline(always)]
179+
#[allow(unreachable_code, non_snake_case)]
180+
pub fn $variant<'tcx>(_tcx: TyCtxt<'tcx>, $(arg: $tuple_arg_ty)*) -> DepNode {
181+
// tuple args
182+
$({
183+
erase!($tuple_arg_ty);
184+
let hash = DepNodeParams::to_fingerprint(&arg, _tcx);
185+
let dep_node = DepNode {
186+
kind: DepKind::$variant,
187+
hash
188+
};
189+
190+
#[cfg(debug_assertions)]
191+
{
192+
if !dep_node.kind.can_reconstruct_query_key() &&
193+
(_tcx.sess.opts.debugging_opts.incremental_info ||
194+
_tcx.sess.opts.debugging_opts.query_dep_graph)
195+
{
196+
_tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
197+
arg.to_debug_str(_tcx)
198+
});
199+
}
200+
}
201+
202+
return dep_node;
203+
})*
204+
205+
DepNode {
206+
kind: DepKind::$variant,
207+
hash: Fingerprint::ZERO,
208+
}
209+
}
210+
)*
197211
}
198212

199213
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash,
@@ -204,75 +218,6 @@ macro_rules! define_dep_nodes {
204218
}
205219

206220
impl DepNode {
207-
#[allow(unreachable_code, non_snake_case)]
208-
pub fn new<'tcx>(tcx: TyCtxt<'tcx>,
209-
dep: DepConstructor<'tcx>)
210-
-> DepNode
211-
{
212-
match dep {
213-
$(
214-
DepConstructor :: $variant $(( replace!(($tuple_arg_ty) with arg) ))*
215-
$({ $($struct_arg_name),* })*
216-
=>
217-
{
218-
// tuple args
219-
$({
220-
erase!($tuple_arg_ty);
221-
let hash = DepNodeParams::to_fingerprint(&arg, tcx);
222-
let dep_node = DepNode {
223-
kind: DepKind::$variant,
224-
hash
225-
};
226-
227-
#[cfg(debug_assertions)]
228-
{
229-
if !dep_node.kind.can_reconstruct_query_key() &&
230-
(tcx.sess.opts.debugging_opts.incremental_info ||
231-
tcx.sess.opts.debugging_opts.query_dep_graph)
232-
{
233-
tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
234-
arg.to_debug_str(tcx)
235-
});
236-
}
237-
}
238-
239-
return dep_node;
240-
})*
241-
242-
// struct args
243-
$({
244-
let tupled_args = ( $($struct_arg_name,)* );
245-
let hash = DepNodeParams::to_fingerprint(&tupled_args,
246-
tcx);
247-
let dep_node = DepNode {
248-
kind: DepKind::$variant,
249-
hash
250-
};
251-
252-
#[cfg(debug_assertions)]
253-
{
254-
if !dep_node.kind.can_reconstruct_query_key() &&
255-
(tcx.sess.opts.debugging_opts.incremental_info ||
256-
tcx.sess.opts.debugging_opts.query_dep_graph)
257-
{
258-
tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
259-
tupled_args.to_debug_str(tcx)
260-
});
261-
}
262-
}
263-
264-
return dep_node;
265-
})*
266-
267-
DepNode {
268-
kind: DepKind::$variant,
269-
hash: Fingerprint::ZERO,
270-
}
271-
}
272-
)*
273-
}
274-
}
275-
276221
/// Construct a DepNode from the given DepKind and DefPathHash. This
277222
/// method will assert that the given DepKind actually requires a
278223
/// single DefId/DefPathHash parameter.

‎src/librustc/dep_graph/graph.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1122,6 +1122,7 @@ impl CurrentDepGraph {
11221122
}
11231123

11241124
impl DepGraphData {
1125+
#[inline(never)]
11251126
fn read_index(&self, source: DepNodeIndex) {
11261127
ty::tls::with_context_opt(|icx| {
11271128
let icx = if let Some(icx) = icx { icx } else { return };

‎src/librustc/mir/mono.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ impl<'tcx> CodegenUnit<'tcx> {
362362
}
363363

364364
pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode {
365-
DepNode::new(tcx, DepConstructor::CompileCodegenUnit(self.name()))
365+
DepConstructor::CompileCodegenUnit(tcx, self.name())
366366
}
367367
}
368368

‎src/librustc/ty/context.rs

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
33
use crate::arena::Arena;
44
use crate::dep_graph::DepGraph;
5-
use crate::dep_graph::{self, DepConstructor, DepNode};
5+
use crate::dep_graph::{self, DepConstructor};
66
use crate::hir::exports::Export;
77
use crate::hir::map as hir_map;
88
use crate::hir::map::DefPathHash;
@@ -1347,7 +1347,7 @@ impl<'tcx> TyCtxt<'tcx> {
13471347
// We cannot use the query versions of crates() and crate_hash(), since
13481348
// those would need the DepNodes that we are allocating here.
13491349
for cnum in self.cstore.crates_untracked() {
1350-
let dep_node = DepNode::new(self, DepConstructor::CrateMetadata(cnum));
1350+
let dep_node = DepConstructor::CrateMetadata(self, cnum);
13511351
let crate_hash = self.cstore.crate_hash_untracked(cnum);
13521352
self.dep_graph.with_task(
13531353
dep_node,
@@ -1688,6 +1688,7 @@ pub mod tls {
16881688

16891689
/// Gets the pointer to the current `ImplicitCtxt`.
16901690
#[cfg(not(parallel_compiler))]
1691+
#[inline]
16911692
fn get_tlv() -> usize {
16921693
TLV.with(|tlv| tlv.get())
16931694
}

‎src/librustc/ty/query/caches.rs

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
use crate::dep_graph::DepNodeIndex;
2+
use crate::ty::query::config::QueryAccessors;
3+
use crate::ty::query::plumbing::{QueryLookup, QueryState, QueryStateShard};
4+
use crate::ty::TyCtxt;
5+
6+
use rustc_data_structures::fx::FxHashMap;
7+
use rustc_data_structures::sharded::Sharded;
8+
use std::default::Default;
9+
use std::hash::Hash;
10+
11+
pub(crate) trait CacheSelector<K, V> {
12+
type Cache: QueryCache<K, V>;
13+
}
14+
15+
pub(crate) trait QueryCache<K, V>: Default {
16+
type Sharded: Default;
17+
18+
/// Checks if the query is already computed and in the cache.
19+
/// It returns the shard index and a lock guard to the shard,
20+
/// which will be used if the query is not in the cache and we need
21+
/// to compute it.
22+
fn lookup<'tcx, R, GetCache, OnHit, OnMiss, Q>(
23+
&self,
24+
state: &'tcx QueryState<'tcx, Q>,
25+
get_cache: GetCache,
26+
key: K,
27+
// `on_hit` can be called while holding a lock to the query state shard.
28+
on_hit: OnHit,
29+
on_miss: OnMiss,
30+
) -> R
31+
where
32+
Q: QueryAccessors<'tcx>,
33+
GetCache: for<'a> Fn(&'a mut QueryStateShard<'tcx, Q>) -> &'a mut Self::Sharded,
34+
OnHit: FnOnce(&V, DepNodeIndex) -> R,
35+
OnMiss: FnOnce(K, QueryLookup<'tcx, Q>) -> R;
36+
37+
fn complete(
38+
&self,
39+
tcx: TyCtxt<'tcx>,
40+
lock_sharded_storage: &mut Self::Sharded,
41+
key: K,
42+
value: V,
43+
index: DepNodeIndex,
44+
);
45+
46+
fn iter<R, L>(
47+
&self,
48+
shards: &Sharded<L>,
49+
get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
50+
f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
51+
) -> R;
52+
}
53+
54+
pub struct DefaultCacheSelector;
55+
56+
impl<K: Eq + Hash, V: Clone> CacheSelector<K, V> for DefaultCacheSelector {
57+
type Cache = DefaultCache;
58+
}
59+
60+
#[derive(Default)]
61+
pub struct DefaultCache;
62+
63+
impl<K: Eq + Hash, V: Clone> QueryCache<K, V> for DefaultCache {
64+
type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
65+
66+
#[inline(always)]
67+
fn lookup<'tcx, R, GetCache, OnHit, OnMiss, Q>(
68+
&self,
69+
state: &'tcx QueryState<'tcx, Q>,
70+
get_cache: GetCache,
71+
key: K,
72+
on_hit: OnHit,
73+
on_miss: OnMiss,
74+
) -> R
75+
where
76+
Q: QueryAccessors<'tcx>,
77+
GetCache: for<'a> Fn(&'a mut QueryStateShard<'tcx, Q>) -> &'a mut Self::Sharded,
78+
OnHit: FnOnce(&V, DepNodeIndex) -> R,
79+
OnMiss: FnOnce(K, QueryLookup<'tcx, Q>) -> R,
80+
{
81+
let mut lookup = state.get_lookup(&key);
82+
let lock = &mut *lookup.lock;
83+
84+
let result = get_cache(lock).raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
85+
86+
if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
87+
}
88+
89+
#[inline]
90+
fn complete(
91+
&self,
92+
_: TyCtxt<'tcx>,
93+
lock_sharded_storage: &mut Self::Sharded,
94+
key: K,
95+
value: V,
96+
index: DepNodeIndex,
97+
) {
98+
lock_sharded_storage.insert(key, (value, index));
99+
}
100+
101+
fn iter<R, L>(
102+
&self,
103+
shards: &Sharded<L>,
104+
get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
105+
f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
106+
) -> R {
107+
let mut shards = shards.lock_shards();
108+
let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
109+
let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
110+
f(Box::new(results))
111+
}
112+
}

0 commit comments

Comments
 (0)