Skip to content

Commit bd5c107

Browse files
cjgillotcjgillot
committed
Build jump table at runtime.
1 parent 602d3cb commit bd5c107

File tree

10 files changed

+111
-130
lines changed

10 files changed

+111
-130
lines changed

‎compiler/rustc_interface/src/passes.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -838,7 +838,7 @@ pub fn create_global_ctxt<'tcx>(
838838
dep_graph,
839839
queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
840840
queries.as_dyn(),
841-
&rustc_query_impl::QUERY_CALLBACKS,
841+
rustc_query_impl::query_callbacks(arena),
842842
crate_name,
843843
outputs,
844844
)

‎compiler/rustc_middle/src/arena.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ macro_rules! arena_types {
100100
// This is used to decode the &'tcx [Span] for InlineAsm's line_spans.
101101
[decode] span: rustc_span::Span,
102102
[decode] used_trait_imports: rustc_data_structures::fx::FxHashSet<rustc_hir::def_id::LocalDefId>,
103+
104+
[] dep_kind: rustc_middle::dep_graph::DepKindStruct,
103105
], $tcx);
104106
)
105107
}

‎compiler/rustc_middle/src/dep_graph/dep_node.rs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ pub struct DepKindStruct {
8787

8888
/// Whether the query key can be recovered from the hashed fingerprint.
8989
/// See [DepNodeParams] trait for the behaviour of each key type.
90-
pub fingerprint_style: fn() -> FingerprintStyle,
90+
pub fingerprint_style: FingerprintStyle,
9191

9292
/// The red/green evaluation system will try to mark a specific DepNode in the
9393
/// dependency graph as green by recursively trying to mark the dependencies of
@@ -131,10 +131,10 @@ pub struct DepKindStruct {
131131
/// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
132132
/// add it to the "We don't have enough information to reconstruct..." group in
133133
/// the match below.
134-
pub force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
134+
pub force_from_dep_node: Option<fn(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool>,
135135

136136
/// Invoke a query to put the on-disk cached value in memory.
137-
pub try_load_from_on_disk_cache: fn(TyCtxt<'_>, &DepNode),
137+
pub try_load_from_on_disk_cache: Option<fn(TyCtxt<'_>, DepNode)>,
138138
}
139139

140140
impl DepKind {
@@ -145,8 +145,7 @@ impl DepKind {
145145
if data.is_anon {
146146
return FingerprintStyle::Opaque;
147147
}
148-
149-
(data.fingerprint_style)()
148+
data.fingerprint_style
150149
}
151150
}
152151

@@ -159,7 +158,7 @@ macro_rules! define_dep_nodes {
159158
) => (
160159
#[macro_export]
161160
macro_rules! make_dep_kind_array {
162-
($mod:ident) => {[ $(($mod::$variant),)* ]};
161+
($mod:ident) => {[ $($mod::$variant()),* ]};
163162
}
164163

165164
/// This enum serves as an index into arrays built by `make_dep_kind_array`.

‎compiler/rustc_middle/src/dep_graph/mod.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
100100
self.query_kind(kind).is_eval_always
101101
}
102102

103-
fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
103+
fn try_force_from_dep_node(&self, dep_node: DepNode) -> bool {
104104
debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node);
105105

106106
// We must avoid ever having to call `force_from_dep_node()` for a
@@ -122,11 +122,18 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
122122
);
123123

124124
let cb = self.query_kind(dep_node.kind);
125-
(cb.force_from_dep_node)(*self, dep_node)
125+
if let Some(f) = cb.force_from_dep_node {
126+
f(*self, dep_node);
127+
true
128+
} else {
129+
false
130+
}
126131
}
127132

128-
fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) {
133+
fn try_load_from_on_disk_cache(&self, dep_node: DepNode) {
129134
let cb = self.query_kind(dep_node.kind);
130-
(cb.try_load_from_on_disk_cache)(*self, dep_node)
135+
if let Some(f) = cb.try_load_from_on_disk_cache {
136+
f(*self, dep_node)
137+
}
131138
}
132139
}

‎compiler/rustc_middle/src/ty/context.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1186,7 +1186,7 @@ impl<'tcx> TyCtxt<'tcx> {
11861186
}
11871187
}
11881188

1189-
pub fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct {
1189+
crate fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct {
11901190
&self.query_kinds[k as usize]
11911191
}
11921192

‎compiler/rustc_query_impl/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ extern crate rustc_middle;
1717
use rustc_data_structures::fingerprint::Fingerprint;
1818
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
1919
use rustc_errors::DiagnosticBuilder;
20+
use rustc_middle::arena::Arena;
2021
use rustc_middle::dep_graph::{self, DepKindStruct};
2122
use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
2223
use rustc_middle::ty::query::{Providers, QueryEngine};

‎compiler/rustc_query_impl/src/plumbing.rs

Lines changed: 70 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ macro_rules! define_queries {
331331
}
332332
})*
333333

334-
#[allow(non_upper_case_globals)]
334+
#[allow(nonstandard_style)]
335335
pub mod query_callbacks {
336336
use super::*;
337337
use rustc_middle::dep_graph::DepNode;
@@ -341,74 +341,83 @@ macro_rules! define_queries {
341341
use rustc_query_system::dep_graph::FingerprintStyle;
342342

343343
// We use this for most things when incr. comp. is turned off.
344-
pub const Null: DepKindStruct = DepKindStruct {
345-
is_anon: false,
346-
is_eval_always: false,
347-
348-
fingerprint_style: || FingerprintStyle::Unit,
349-
force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
350-
try_load_from_on_disk_cache: |_, _| {},
351-
};
352-
353-
pub const TraitSelect: DepKindStruct = DepKindStruct {
354-
is_anon: true,
355-
is_eval_always: false,
344+
pub fn Null() -> DepKindStruct {
345+
DepKindStruct {
346+
is_anon: false,
347+
is_eval_always: false,
348+
fingerprint_style: FingerprintStyle::Unit,
349+
force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
350+
try_load_from_on_disk_cache: None,
351+
}
352+
}
356353

357-
fingerprint_style: || FingerprintStyle::Unit,
358-
force_from_dep_node: |_, _| false,
359-
try_load_from_on_disk_cache: |_, _| {},
360-
};
354+
pub fn TraitSelect() -> DepKindStruct {
355+
DepKindStruct {
356+
is_anon: true,
357+
is_eval_always: false,
358+
fingerprint_style: FingerprintStyle::Unit,
359+
force_from_dep_node: None,
360+
try_load_from_on_disk_cache: None,
361+
}
362+
}
361363

362-
pub const CompileCodegenUnit: DepKindStruct = DepKindStruct {
363-
is_anon: false,
364-
is_eval_always: false,
364+
pub fn CompileCodegenUnit() -> DepKindStruct {
365+
DepKindStruct {
366+
is_anon: false,
367+
is_eval_always: false,
368+
fingerprint_style: FingerprintStyle::Opaque,
369+
force_from_dep_node: None,
370+
try_load_from_on_disk_cache: None,
371+
}
372+
}
365373

366-
fingerprint_style: || FingerprintStyle::Opaque,
367-
force_from_dep_node: |_, _| false,
368-
try_load_from_on_disk_cache: |_, _| {},
369-
};
374+
pub fn CompileMonoItem() -> DepKindStruct {
375+
DepKindStruct {
376+
is_anon: false,
377+
is_eval_always: false,
378+
fingerprint_style: FingerprintStyle::Opaque,
379+
force_from_dep_node: None,
380+
try_load_from_on_disk_cache: None,
381+
}
382+
}
370383

371-
pub const CompileMonoItem: DepKindStruct = DepKindStruct {
372-
is_anon: false,
373-
is_eval_always: false,
384+
$(pub fn $name()-> DepKindStruct {
385+
let is_anon = is_anon!([$($modifiers)*]);
386+
let is_eval_always = is_eval_always!([$($modifiers)*]);
374387

375-
fingerprint_style: || FingerprintStyle::Opaque,
376-
force_from_dep_node: |_, _| false,
377-
try_load_from_on_disk_cache: |_, _| {},
378-
};
388+
let fingerprint_style =
389+
<query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::fingerprint_style();
379390

380-
$(pub const $name: DepKindStruct = {
381-
const is_anon: bool = is_anon!([$($modifiers)*]);
382-
const is_eval_always: bool = is_eval_always!([$($modifiers)*]);
391+
if is_anon || !fingerprint_style.reconstructible() {
392+
return DepKindStruct {
393+
is_anon,
394+
is_eval_always,
395+
fingerprint_style,
396+
force_from_dep_node: None,
397+
try_load_from_on_disk_cache: None,
398+
}
399+
}
383400

384401
#[inline(always)]
385-
fn fingerprint_style() -> FingerprintStyle {
386-
<query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>
387-
::fingerprint_style()
402+
fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> Option<query_keys::$name<'tcx>> {
403+
<query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, &dep_node)
388404
}
389405

390-
fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option<query_keys::$name<'tcx>> {
391-
<query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
406+
fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool {
407+
if let Some(key) = recover(tcx, dep_node) {
408+
let tcx = QueryCtxt::from_tcx(tcx);
409+
force_query::<queries::$name<'_>, _>(tcx, key, dep_node);
410+
true
411+
} else {
412+
false
413+
}
392414
}
393415

394-
fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool {
395-
let tcx = QueryCtxt::from_tcx(tcx);
396-
force_query::<queries::$name<'_>, _>(tcx, dep_node)
397-
}
416+
fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: DepNode) {
417+
debug_assert!(tcx.dep_graph.is_green(&dep_node));
398418

399-
fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: &DepNode) {
419+
let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
400420
let tcx = QueryCtxt::from_tcx(tcx);
401-
if is_anon {
402-
return
403-
}
404-
405-
if !fingerprint_style().reconstructible() {
406-
return
407-
}
408-
409-
debug_assert!(tcx.dep_graph.is_green(dep_node));
410-
411-
let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
412421
if queries::$name::cache_on_disk(tcx, &key, None) {
413422
let _ = tcx.$name(key);
414423
}
@@ -418,13 +427,15 @@ macro_rules! define_queries {
418427
is_anon,
419428
is_eval_always,
420429
fingerprint_style,
421-
force_from_dep_node,
422-
try_load_from_on_disk_cache,
430+
force_from_dep_node: Some(force_from_dep_node),
431+
try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache),
423432
}
424-
};)*
433+
})*
425434
}
426435

427-
pub static QUERY_CALLBACKS: &[DepKindStruct] = &make_dep_kind_array!(query_callbacks);
436+
pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] {
437+
arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
438+
}
428439
}
429440
}
430441

‎compiler/rustc_query_system/src/dep_graph/graph.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,7 @@ impl<K: DepKind> DepGraph<K> {
576576
"try_mark_previous_green({:?}) --- trying to force dependency {:?}",
577577
dep_node, dep_dep_node
578578
);
579-
if !tcx.dep_context().try_force_from_dep_node(dep_dep_node) {
579+
if !tcx.dep_context().try_force_from_dep_node(*dep_dep_node) {
580580
// The DepNode could not be forced.
581581
debug!(
582582
"try_mark_previous_green({:?}) - END - dependency {:?} could not be forced",
@@ -749,7 +749,7 @@ impl<K: DepKind> DepGraph<K> {
749749
match data.colors.get(prev_index) {
750750
Some(DepNodeColor::Green(_)) => {
751751
let dep_node = data.previous.index_to_node(prev_index);
752-
tcx.try_load_from_on_disk_cache(&dep_node);
752+
tcx.try_load_from_on_disk_cache(dep_node);
753753
}
754754
None | Some(DepNodeColor::Red) => {
755755
// We can skip red nodes because a node can only be marked

‎compiler/rustc_query_system/src/dep_graph/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ pub trait DepContext: Copy {
3939
fn fingerprint_style(&self, kind: Self::DepKind) -> FingerprintStyle;
4040

4141
/// Try to force a dep node to execute and see if it's green.
42-
fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
42+
fn try_force_from_dep_node(&self, dep_node: DepNode<Self::DepKind>) -> bool;
4343

4444
/// Load data from the on-disk cache.
45-
fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
45+
fn try_load_from_on_disk_cache(&self, dep_node: DepNode<Self::DepKind>);
4646
}
4747

4848
pub trait HasDepContext: Copy {

‎compiler/rustc_query_system/src/query/plumbing.rs

Lines changed: 16 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -665,41 +665,6 @@ where
665665
}
666666
}
667667

668-
#[inline(never)]
669-
fn force_query_impl<CTX, C>(
670-
tcx: CTX,
671-
state: &QueryState<CTX::DepKind, C::Key>,
672-
cache: &QueryCacheStore<C>,
673-
key: C::Key,
674-
dep_node: DepNode<CTX::DepKind>,
675-
query: &QueryVtable<CTX, C::Key, C::Value>,
676-
compute: fn(CTX::DepContext, C::Key) -> C::Value,
677-
) -> bool
678-
where
679-
C: QueryCache,
680-
C::Key: DepNodeParams<CTX::DepContext>,
681-
CTX: QueryContext,
682-
{
683-
debug_assert!(!query.anon);
684-
685-
// We may be concurrently trying both execute and force a query.
686-
// Ensure that only one of them runs the query.
687-
let cached = cache.cache.lookup(cache, &key, |_, index| {
688-
if unlikely!(tcx.dep_context().profiler().enabled()) {
689-
tcx.dep_context().profiler().query_cache_hit(index.into());
690-
}
691-
});
692-
693-
let lookup = match cached {
694-
Ok(()) => return true,
695-
Err(lookup) => lookup,
696-
};
697-
698-
let _ =
699-
try_execute_query(tcx, state, cache, DUMMY_SP, key, lookup, Some(dep_node), query, compute);
700-
true
701-
}
702-
703668
pub enum QueryMode {
704669
Get,
705670
Ensure,
@@ -747,34 +712,30 @@ where
747712
Some(result)
748713
}
749714

750-
pub fn force_query<Q, CTX>(tcx: CTX, dep_node: &DepNode<CTX::DepKind>) -> bool
715+
pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, dep_node: DepNode<CTX::DepKind>)
751716
where
752717
Q: QueryDescription<CTX>,
753718
Q::Key: DepNodeParams<CTX::DepContext>,
754719
CTX: QueryContext,
755720
{
756-
if Q::ANON {
757-
return false;
758-
}
721+
let query = &Q::VTABLE;
722+
debug_assert!(!Q::ANON);
759723

760-
if !<Q::Key as DepNodeParams<CTX::DepContext>>::fingerprint_style().reconstructible() {
761-
return false;
762-
}
724+
// We may be concurrently trying both execute and force a query.
725+
// Ensure that only one of them runs the query.
726+
let cache = Q::query_cache(tcx);
727+
let cached = cache.cache.lookup(cache, &key, |_, index| {
728+
if unlikely!(tcx.dep_context().profiler().enabled()) {
729+
tcx.dep_context().profiler().query_cache_hit(index.into());
730+
}
731+
});
763732

764-
let Some(key) =
765-
<Q::Key as DepNodeParams<CTX::DepContext>>::recover(*tcx.dep_context(), &dep_node)
766-
else {
767-
return false;
733+
let lookup = match cached {
734+
Ok(()) => return,
735+
Err(lookup) => lookup,
768736
};
769737

770738
let compute = Q::compute_fn(tcx, &key);
771-
force_query_impl(
772-
tcx,
773-
Q::query_state(tcx),
774-
Q::query_cache(tcx),
775-
key,
776-
*dep_node,
777-
&Q::VTABLE,
778-
compute,
779-
)
739+
let state = Q::query_state(tcx);
740+
try_execute_query(tcx, state, cache, DUMMY_SP, key, lookup, Some(dep_node), query, compute);
780741
}

0 commit comments

Comments
 (0)