Auto merge of #69808 - cjgillot:vtbl, r=pnkfelix

Avoid duplicating code for each query

There are at the moment roughly 170 queries in librustc.
The way `ty::query` is structured, a lot of code is duplicated for each query.
I suspect this to be responsible for a part of librustc'c compile time.

The first part of this PR reduces the amount of code generic on the query,
replacing it by code generic on the key-value types. I can split it out if needed.

In a second part, the non-inlined methods in the `QueryAccessors` and `QueryDescription` traits
are made into a virtual dispatch table. This allows to reduce even more the number of generated
functions.

This allows to save 1.5s on check build, and 10% on the size of the librustc.rlib.
(Attributed roughly half and half).
My computer is not good enough to measure properly compiling time.
I have no idea of the effect on performance. A perf run may be required.

cc #65031

src/librustc_macros/src/query.rs

@@ -365,7 +365,7 @@ fn try_load_from_disk(
             #[allow(unused_variables, unused_braces)]
             fn cache_on_disk(
                 #tcx: TyCtxt<'tcx>,
-                #key: Self::Key,
+                #key: &Self::Key,
                 #value: Option<&Self::Value>
             ) -> bool {
                 #expr
@@ -441,7 +441,7 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream {
                                             .unwrap_or(false));
 
                             let key = <#arg as DepNodeParams<TyCtxt<'_>>>::recover($tcx, $dep_node).unwrap();
-                            if queries::#name::cache_on_disk($tcx, key, None) {
+                            if queries::#name::cache_on_disk($tcx, &key, None) {
                                 let _ = $tcx.#name(key);
                             }
                         }

src/librustc_middle/dep_graph/dep_node.rs

@@ -183,31 +183,10 @@ pub fn $variant(_tcx: TyCtxt<'_>, $(arg: $tuple_arg_ty)*) -> DepNode {
                     // tuple args
                     $({
                         erase!($tuple_arg_ty);
-                        let hash = DepNodeParams::to_fingerprint(&arg, _tcx);
-                        let dep_node = DepNode {
-                            kind: DepKind::$variant,
-                            hash
-                        };
-
-                        #[cfg(debug_assertions)]
-                        {
-                            if !dep_node.kind.can_reconstruct_query_key() &&
-                            (_tcx.sess.opts.debugging_opts.incremental_info ||
-                                _tcx.sess.opts.debugging_opts.query_dep_graph)
-                            {
-                                _tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
-                                    arg.to_debug_str(_tcx)
-                                });
-                            }
-                        }
-
-                        return dep_node;
+                        return DepNode::construct(_tcx, DepKind::$variant, &arg)
                     })*
 
-                    DepNode {
-                        kind: DepKind::$variant,
-                        hash: Fingerprint::ZERO,
-                    }
+                    return DepNode::construct(_tcx, DepKind::$variant, &())
                 }
             )*
         }

src/librustc_middle/dep_graph/mod.rs

@@ -98,6 +98,10 @@ fn create_stable_hashing_context(&self) -> Self::StableHashingContext {
     fn debug_dep_tasks(&self) -> bool {
         self.sess.opts.debugging_opts.dep_tasks
     }
+    fn debug_dep_node(&self) -> bool {
+        self.sess.opts.debugging_opts.incremental_info
+            || self.sess.opts.debugging_opts.query_dep_graph
+    }
 
     fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
         // FIXME: This match is just a workaround for incremental bugs and should

src/librustc_middle/ty/query/mod.rs

-use crate::dep_graph::{self, DepConstructor, DepNode, DepNodeParams};
+use crate::dep_graph::{self, DepNode, DepNodeParams};
 use crate::hir::exports::Export;
 use crate::hir::map;
 use crate::infer::canonical::{self, Canonical};

src/librustc_middle/ty/query/on_disk_cache.rs

@@ -1009,7 +1009,7 @@ fn encode_query_results<'a, 'tcx, Q, E>(
 
     state.iter_results(|results| {
         for (key, value, dep_node) in results {
-            if Q::cache_on_disk(tcx, key.clone(), Some(&value)) {
+            if Q::cache_on_disk(tcx, &key, Some(&value)) {
                 let dep_node = SerializedDepNodeIndex::new(dep_node.index());
 
                 // Record position of the cache entry.

src/librustc_middle/ty/query/plumbing.rs

@@ -348,12 +348,6 @@ fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<TyCtxt<$tcx>, Self::Cach
                 &tcx.queries.$name
             }
 
-            #[allow(unused)]
-            #[inline(always)]
-            fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode {
-                DepConstructor::$node(tcx, *key)
-            }
-
             #[inline]
             fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
                 let provider = tcx.queries.providers.get(key.query_crate())

src/librustc_query_system/dep_graph/dep_node.rs

@@ -64,6 +64,24 @@ pub fn new_no_params(kind: K) -> DepNode<K> {
         debug_assert!(!kind.has_params());
         DepNode { kind, hash: Fingerprint::ZERO }
     }
+
+    pub fn construct<Ctxt, Key>(tcx: Ctxt, kind: K, arg: &Key) -> DepNode<K>
+    where
+        Ctxt: crate::query::QueryContext<DepKind = K>,
+        Key: DepNodeParams<Ctxt>,
+    {
+        let hash = arg.to_fingerprint(tcx);
+        let dep_node = DepNode { kind, hash };
+
+        #[cfg(debug_assertions)]
+        {
+            if !kind.can_reconstruct_query_key() && tcx.debug_dep_node() {
+                tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx));
+            }
+        }
+
+        return dep_node;
+    }
 }
 
 impl<K: DepKind> fmt::Debug for DepNode<K> {
@@ -120,6 +138,12 @@ impl<Ctxt: DepContext, T> DepNodeParams<Ctxt> for T
     }
 }
 
+impl<Ctxt: DepContext> DepNodeParams<Ctxt> for () {
+    fn to_fingerprint(&self, _: Ctxt) -> Fingerprint {
+        Fingerprint::ZERO
+    }
+}
+
 /// A "work product" corresponds to a `.o` (or other) file that we
 /// save in between runs. These IDs do not have a `DefId` but rather
 /// some independent path or string that persists between runs without

src/librustc_query_system/dep_graph/mod.rs

@@ -28,6 +28,7 @@ pub trait DepContext: Copy {
     fn create_stable_hashing_context(&self) -> Self::StableHashingContext;
 
     fn debug_dep_tasks(&self) -> bool;
+    fn debug_dep_node(&self) -> bool;
 
     /// Try to force a dep node to execute and see if it's green.
     fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;

src/librustc_query_system/query/config.rs

@@ -24,6 +24,53 @@ pub trait QueryConfig<CTX> {
     type Stored: Clone;
 }
 
+pub(crate) struct QueryVtable<CTX: QueryContext, K, V> {
+    pub anon: bool,
+    pub dep_kind: CTX::DepKind,
+    pub eval_always: bool,
+
+    // Don't use this method to compute query results, instead use the methods on TyCtxt
+    pub compute: fn(CTX, K) -> V,
+
+    pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option<Fingerprint>,
+    pub handle_cycle_error: fn(CTX, CycleError<CTX::Query>) -> V,
+    pub cache_on_disk: fn(CTX, &K, Option<&V>) -> bool,
+    pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option<V>,
+}
+
+impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
+    pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode<CTX::DepKind>
+    where
+        K: crate::dep_graph::DepNodeParams<CTX>,
+    {
+        DepNode::construct(tcx, self.dep_kind, key)
+    }
+
+    pub(crate) fn compute(&self, tcx: CTX, key: K) -> V {
+        (self.compute)(tcx, key)
+    }
+
+    pub(crate) fn hash_result(
+        &self,
+        hcx: &mut CTX::StableHashingContext,
+        value: &V,
+    ) -> Option<Fingerprint> {
+        (self.hash_result)(hcx, value)
+    }
+
+    pub(crate) fn handle_cycle_error(&self, tcx: CTX, error: CycleError<CTX::Query>) -> V {
+        (self.handle_cycle_error)(tcx, error)
+    }
+
+    pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool {
+        (self.cache_on_disk)(tcx, key, value)
+    }
+
+    pub(crate) fn try_load_from_disk(&self, tcx: CTX, index: SerializedDepNodeIndex) -> Option<V> {
+        (self.try_load_from_disk)(tcx, index)
+    }
+}
+
 pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
     const ANON: bool;
     const EVAL_ALWAYS: bool;
@@ -34,7 +81,12 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
     // Don't use this method to access query results, instead use the methods on TyCtxt
     fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;
 
-    fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>;
+    fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>
+    where
+        Self::Key: crate::dep_graph::DepNodeParams<CTX>,
+    {
+        DepNode::construct(tcx, Self::DEP_KIND, key)
+    }
 
     // Don't use this method to compute query results, instead use the methods on TyCtxt
     fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
@@ -51,7 +103,7 @@ pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
     fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>;
 
     #[inline]
-    fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
+    fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool {
         false
     }
 
@@ -60,6 +112,27 @@ fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value>
     }
 }
 
+pub(crate) trait QueryVtableExt<CTX: QueryContext, K, V> {
+    const VTABLE: QueryVtable<CTX, K, V>;
+}
+
+impl<CTX, Q> QueryVtableExt<CTX, Q::Key, Q::Value> for Q
+where
+    CTX: QueryContext,
+    Q: QueryDescription<CTX>,
+{
+    const VTABLE: QueryVtable<CTX, Q::Key, Q::Value> = QueryVtable {
+        anon: Q::ANON,
+        dep_kind: Q::DEP_KIND,
+        eval_always: Q::EVAL_ALWAYS,
+        compute: Q::compute,
+        hash_result: Q::hash_result,
+        handle_cycle_error: Q::handle_cycle_error,
+        cache_on_disk: Q::cache_on_disk,
+        try_load_from_disk: Q::try_load_from_disk,
+    };
+}
+
 impl<CTX: QueryContext, M> QueryDescription<CTX> for M
 where
     M: QueryAccessors<CTX, Key = DefId>,
@@ -73,7 +146,7 @@ impl<CTX: QueryContext, M> QueryDescription<CTX> for M
         }
     }
 
-    default fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool {
+    default fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool {
         false
     }
 

src/librustc_query_system/query/plumbing.rs

@@ -5,7 +5,7 @@
 use crate::dep_graph::{DepKind, DepNode};
 use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
 use crate::query::caches::QueryCache;
-use crate::query::config::QueryDescription;
+use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt};
 use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
 use crate::query::QueryContext;
 
@@ -29,7 +29,7 @@
 #[cfg(debug_assertions)]
 use std::sync::atomic::{AtomicUsize, Ordering};
 
-pub struct QueryStateShard<CTX: QueryContext, K, C> {
+pub(super) struct QueryStateShard<CTX: QueryContext, K, C> {
     pub(super) cache: C,
     active: FxHashMap<K, QueryResult<CTX>>,
 
@@ -80,6 +80,7 @@ enum QueryResult<CTX: QueryContext> {
 }
 
 impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
+    #[inline(always)]
     pub fn iter_results<R>(
         &self,
         f: impl for<'a> FnOnce(
@@ -89,6 +90,7 @@ pub fn iter_results<R>(
         self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
     }
 
+    #[inline(always)]
     pub fn all_inactive(&self) -> bool {
         let shards = self.shards.lock_shards();
         shards.iter().all(|shard| shard.active.is_empty())
@@ -168,14 +170,15 @@ impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C>
     /// This function is inlined because that results in a noticeable speed-up
     /// for some compile-time benchmarks.
     #[inline(always)]
-    fn try_start<'a, 'b, Q>(
+    fn try_start<'a, 'b>(
         tcx: CTX,
+        state: &'b QueryState<CTX, C>,
         span: Span,
         key: &C::Key,
         mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
+        query: &QueryVtable<CTX, C::Key, C::Value>,
     ) -> TryGetJob<'b, CTX, C>
     where
-        Q: QueryDescription<CTX, Key = C::Key, Stored = C::Stored, Value = C::Value, Cache = C>,
         CTX: QueryContext,
     {
         let lock = &mut *lookup.lock;
@@ -194,7 +197,7 @@ fn try_start<'a, 'b, Q>(
                         };
 
                         // Create the id of the job we're waiting for
-                        let id = QueryJobId::new(job.id, lookup.shard, Q::DEP_KIND);
+                        let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind);
 
                         (job.latch(id), _query_blocked_prof_timer)
                     }
@@ -209,15 +212,14 @@ fn try_start<'a, 'b, Q>(
                 lock.jobs = id;
                 let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
 
-                let global_id = QueryJobId::new(id, lookup.shard, Q::DEP_KIND);
+                let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind);
 
                 let job = tcx.current_query_job();
                 let job = QueryJob::new(id, span, job);
 
                 entry.insert(QueryResult::Started(job));
 
-                let owner =
-                    JobOwner { state: Q::query_state(tcx), id: global_id, key: (*key).clone() };
+                let owner = JobOwner { state, id: global_id, key: (*key).clone() };
                 return TryGetJob::NotYetStarted(owner);
             }
         };
@@ -227,8 +229,8 @@ fn try_start<'a, 'b, Q>(
         // so we just return the error.
         #[cfg(not(parallel_compiler))]
         return TryGetJob::Cycle(cold_path(|| {
-            let value = Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
-            Q::query_state(tcx).cache.store_nocache(value)
+            let value = query.handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
+            state.cache.store_nocache(value)
         }));
 
         // With parallel queries we might just have to wait on some other
@@ -238,14 +240,14 @@ fn try_start<'a, 'b, Q>(
             let result = latch.wait_on(tcx, span);
 
             if let Err(cycle) = result {
-                let value = Q::handle_cycle_error(tcx, cycle);
-                let value = Q::query_state(tcx).cache.store_nocache(value);
+                let value = query.handle_cycle_error(tcx, cycle);
+                let value = state.cache.store_nocache(value);
                 return TryGetJob::Cycle(value);
             }
 
             let cached = try_get_cached(
                 tcx,
-                Q::query_state(tcx),
+                state,
                 (*key).clone(),
                 |value, index| (value.clone(), index),
                 |_, _| panic!("value must be in cache after waiting"),
@@ -382,17 +384,21 @@ fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
 }
 
 #[inline(always)]
-fn try_execute_query<Q, CTX>(
+fn try_execute_query<CTX, C>(
     tcx: CTX,
+    state: &QueryState<CTX, C>,
     span: Span,
-    key: Q::Key,
-    lookup: QueryLookup<'_, CTX, Q::Key, <Q::Cache as QueryCache>::Sharded>,
-) -> Q::Stored
+    key: C::Key,
+    lookup: QueryLookup<'_, CTX, C::Key, C::Sharded>,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) -> C::Stored
 where
-    Q: QueryDescription<CTX>,
+    C: QueryCache,
+    C::Key: Eq + Clone + Debug + crate::dep_graph::DepNodeParams<CTX>,
+    C::Stored: Clone,
     CTX: QueryContext,
 {
-    let job = match JobOwner::try_start::<Q>(tcx, span, &key, lookup) {
+    let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
         TryGetJob::NotYetStarted(job) => job,
         TryGetJob::Cycle(result) => return result,
         #[cfg(parallel_compiler)]
@@ -406,15 +412,15 @@ fn try_execute_query<Q, CTX>(
     // expensive for some `DepKind`s.
     if !tcx.dep_graph().is_fully_enabled() {
         let null_dep_node = DepNode::new_no_params(DepKind::NULL);
-        return force_query_with_job::<Q, _>(tcx, key, job, null_dep_node).0;
+        return force_query_with_job(tcx, key, job, null_dep_node, query).0;
     }
 
-    if Q::ANON {
+    if query.anon {
         let prof_timer = tcx.profiler().query_provider();
 
         let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
             tcx.start_query(job.id, diagnostics, |tcx| {
-                tcx.dep_graph().with_anon_task(Q::DEP_KIND, || Q::compute(tcx, key))
+                tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key))
             })
         });
 
@@ -429,9 +435,9 @@ fn try_execute_query<Q, CTX>(
         return job.complete(tcx, result, dep_node_index);
     }
 
-    let dep_node = Q::to_dep_node(tcx, &key);
+    let dep_node = query.to_dep_node(tcx, &key);
 
-    if !Q::EVAL_ALWAYS {
+    if !query.eval_always {
         // The diagnostics for this query will be
         // promoted to the current session during
         // `try_mark_green()`, so we can ignore them here.
@@ -439,12 +445,13 @@ fn try_execute_query<Q, CTX>(
             let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
             marked.map(|(prev_dep_node_index, dep_node_index)| {
                 (
-                    load_from_disk_and_cache_in_memory::<Q, _>(
+                    load_from_disk_and_cache_in_memory(
                         tcx,
                         key.clone(),
                         prev_dep_node_index,
                         dep_node_index,
                         &dep_node,
+                        query,
                     ),
                     dep_node_index,
                 )
@@ -455,21 +462,21 @@ fn try_execute_query<Q, CTX>(
         }
     }
 
-    let (result, dep_node_index) = force_query_with_job::<Q, _>(tcx, key, job, dep_node);
+    let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query);
     tcx.dep_graph().read_index(dep_node_index);
     result
 }
 
-fn load_from_disk_and_cache_in_memory<Q, CTX>(
+fn load_from_disk_and_cache_in_memory<CTX, K, V>(
     tcx: CTX,
-    key: Q::Key,
+    key: K,
     prev_dep_node_index: SerializedDepNodeIndex,
     dep_node_index: DepNodeIndex,
     dep_node: &DepNode<CTX::DepKind>,
-) -> Q::Value
+    query: &QueryVtable<CTX, K, V>,
+) -> V
 where
     CTX: QueryContext,
-    Q: QueryDescription<CTX>,
 {
     // Note this function can be called concurrently from the same query
     // We must ensure that this is handled correctly.
@@ -477,9 +484,9 @@ fn load_from_disk_and_cache_in_memory<Q, CTX>(
     debug_assert!(tcx.dep_graph().is_green(dep_node));
 
     // First we try to load the result from the on-disk cache.
-    let result = if Q::cache_on_disk(tcx, key.clone(), None) {
+    let result = if query.cache_on_disk(tcx, &key, None) {
         let prof_timer = tcx.profiler().incr_cache_loading();
-        let result = Q::try_load_from_disk(tcx, prev_dep_node_index);
+        let result = query.try_load_from_disk(tcx, prev_dep_node_index);
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
         // We always expect to find a cached result for things that
@@ -503,7 +510,7 @@ fn load_from_disk_and_cache_in_memory<Q, CTX>(
         let prof_timer = tcx.profiler().query_provider();
 
         // The dep-graph for this computation is already in-place.
-        let result = tcx.dep_graph().with_ignore(|| Q::compute(tcx, key));
+        let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key));
 
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -513,7 +520,7 @@ fn load_from_disk_and_cache_in_memory<Q, CTX>(
     // If `-Zincremental-verify-ich` is specified, re-hash results from
     // the cache and make sure that they have the expected fingerprint.
     if unlikely!(tcx.incremental_verify_ich()) {
-        incremental_verify_ich::<Q, _>(tcx, &result, dep_node, dep_node_index);
+        incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query);
     }
 
     result
@@ -521,14 +528,14 @@ fn load_from_disk_and_cache_in_memory<Q, CTX>(
 
 #[inline(never)]
 #[cold]
-fn incremental_verify_ich<Q, CTX>(
+fn incremental_verify_ich<CTX, K, V>(
     tcx: CTX,
-    result: &Q::Value,
+    result: &V,
     dep_node: &DepNode<CTX::DepKind>,
     dep_node_index: DepNodeIndex,
+    query: &QueryVtable<CTX, K, V>,
 ) where
     CTX: QueryContext,
-    Q: QueryDescription<CTX>,
 {
     assert!(
         Some(tcx.dep_graph().fingerprint_of(dep_node_index))
@@ -540,7 +547,7 @@ fn incremental_verify_ich<Q, CTX>(
     debug!("BEGIN verify_ich({:?})", dep_node);
     let mut hcx = tcx.create_stable_hashing_context();
 
-    let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
+    let new_hash = query.hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
     debug!("END verify_ich({:?})", dep_node);
 
     let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
@@ -549,14 +556,17 @@ fn incremental_verify_ich<Q, CTX>(
 }
 
 #[inline(always)]
-fn force_query_with_job<Q, CTX>(
+fn force_query_with_job<C, CTX>(
     tcx: CTX,
-    key: Q::Key,
-    job: JobOwner<'_, CTX, Q::Cache>,
+    key: C::Key,
+    job: JobOwner<'_, CTX, C>,
     dep_node: DepNode<CTX::DepKind>,
-) -> (Q::Stored, DepNodeIndex)
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) -> (C::Stored, DepNodeIndex)
 where
-    Q: QueryDescription<CTX>,
+    C: QueryCache,
+    C::Key: Eq + Clone + Debug,
+    C::Stored: Clone,
     CTX: QueryContext,
 {
     // If the following assertion triggers, it can have two reasons:
@@ -577,16 +587,16 @@ fn force_query_with_job<Q, CTX>(
 
     let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
         tcx.start_query(job.id, diagnostics, |tcx| {
-            if Q::EVAL_ALWAYS {
+            if query.eval_always {
                 tcx.dep_graph().with_eval_always_task(
                     dep_node,
                     tcx,
                     key,
-                    Q::compute,
-                    Q::hash_result,
+                    query.compute,
+                    query.hash_result,
                 )
             } else {
-                tcx.dep_graph().with_task(dep_node, tcx, key, Q::compute, Q::hash_result)
+                tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result)
             }
         })
     });
@@ -605,22 +615,28 @@ fn force_query_with_job<Q, CTX>(
 }
 
 #[inline(never)]
-pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
+fn get_query_impl<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    span: Span,
+    key: C::Key,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) -> C::Stored
 where
-    Q: QueryDescription<CTX>,
     CTX: QueryContext,
+    C: QueryCache,
+    C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
+    C::Stored: Clone,
 {
-    debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
-
     try_get_cached(
         tcx,
-        Q::query_state(tcx),
+        state,
         key,
         |value, index| {
             tcx.dep_graph().read_index(index);
             value.clone()
         },
-        |key, lookup| try_execute_query::<Q, _>(tcx, span, key, lookup),
+        |key, lookup| try_execute_query(tcx, state, span, key, lookup, query),
     )
 }
 
@@ -631,20 +647,26 @@ pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
 /// side-effects -- e.g., in order to report errors for erroneous programs.
 ///
 /// Note: The optimization is only available during incr. comp.
-pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
-where
-    Q: QueryDescription<CTX>,
+#[inline(never)]
+fn ensure_query_impl<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    key: C::Key,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) where
+    C: QueryCache,
+    C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
     CTX: QueryContext,
 {
-    if Q::EVAL_ALWAYS {
-        let _ = get_query::<Q, _>(tcx, DUMMY_SP, key);
+    if query.eval_always {
+        let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
         return;
     }
 
     // Ensuring an anonymous query makes no sense
-    assert!(!Q::ANON);
+    assert!(!query.anon);
 
-    let dep_node = Q::to_dep_node(tcx, &key);
+    let dep_node = query.to_dep_node(tcx, &key);
 
     match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) {
         None => {
@@ -654,7 +676,7 @@ pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
             // DepNodeIndex. We must invoke the query itself. The performance cost
             // this introduces should be negligible as we'll immediately hit the
             // in-memory cache, or another query down the line will.
-            let _ = get_query::<Q, _>(tcx, DUMMY_SP, key);
+            let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
         }
         Some((_, dep_node_index)) => {
             tcx.profiler().query_cache_hit(dep_node_index.into());
@@ -662,9 +684,17 @@ pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
     }
 }
 
-pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
-where
-    Q: QueryDescription<CTX>,
+#[inline(never)]
+fn force_query_impl<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    key: C::Key,
+    span: Span,
+    dep_node: DepNode<CTX::DepKind>,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) where
+    C: QueryCache,
+    C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
     CTX: QueryContext,
 {
     // We may be concurrently trying both execute and force a query.
@@ -672,19 +702,51 @@ pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<
 
     try_get_cached(
         tcx,
-        Q::query_state(tcx),
+        state,
         key,
         |_, _| {
             // Cache hit, do nothing
         },
         |key, lookup| {
-            let job = match JobOwner::try_start::<Q>(tcx, span, &key, lookup) {
+            let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
                 TryGetJob::NotYetStarted(job) => job,
                 TryGetJob::Cycle(_) => return,
                 #[cfg(parallel_compiler)]
                 TryGetJob::JobCompleted(_) => return,
             };
-            force_query_with_job::<Q, _>(tcx, key, job, dep_node);
+            force_query_with_job(tcx, key, job, dep_node, query);
         },
     );
 }
+
+#[inline(always)]
+pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
+where
+    Q: QueryDescription<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
+
+    get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE)
+}
+
+#[inline(always)]
+pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
+where
+    Q: QueryDescription<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE)
+}
+
+#[inline(always)]
+pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
+where
+    Q: QueryDescription<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE)
+}