hide `SmallCanonicalVarValues` in `OriginalQueryValues` struct

src/librustc/infer/canonical/canonicalizer.rs

@@ -17,7 +17,7 @@
 
 use infer::canonical::{
     Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, Canonicalized,
-    SmallCanonicalVarValues,
+    OriginalQueryValues,
 };
 use infer::InferCtxt;
 use std::sync::atomic::Ordering;
@@ -48,7 +48,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> {
     pub fn canonicalize_query<V>(
         &self,
         value: &V,
-        var_values: &mut SmallCanonicalVarValues<'tcx>,
+        query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
@@ -64,7 +64,7 @@ pub fn canonicalize_query<V>(
             Some(self),
             self.tcx,
             &CanonicalizeAllFreeRegions,
-            var_values,
+            query_state,
         )
     }
 
@@ -97,13 +97,13 @@ pub fn canonicalize_response<V>(&self, value: &V) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
     {
-        let mut var_values = SmallVec::new();
+        let mut query_state = OriginalQueryValues::default();
         Canonicalizer::canonicalize(
             value,
             Some(self),
             self.tcx,
             &CanonicalizeQueryResponse,
-            &mut var_values,
+            &mut query_state,
         )
     }
 
@@ -119,7 +119,7 @@ pub fn canonicalize_response<V>(&self, value: &V) -> Canonicalized<'gcx, V>
     pub fn canonicalize_hr_query_hack<V>(
         &self,
         value: &V,
-        var_values: &mut SmallCanonicalVarValues<'tcx>,
+        query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
@@ -135,7 +135,7 @@ pub fn canonicalize_hr_query_hack<V>(
             Some(self),
             self.tcx,
             &CanonicalizeFreeRegionsOtherThanStatic,
-            var_values,
+            query_state,
         )
     }
 }
@@ -222,7 +222,7 @@ struct Canonicalizer<'cx, 'gcx: 'tcx, 'tcx: 'cx> {
     infcx: Option<&'cx InferCtxt<'cx, 'gcx, 'tcx>>,
     tcx: TyCtxt<'cx, 'gcx, 'tcx>,
     variables: SmallVec<[CanonicalVarInfo; 8]>,
-    var_values: &'cx mut SmallCanonicalVarValues<'tcx>,
+    query_state: &'cx mut OriginalQueryValues<'tcx>,
     // Note that indices is only used once `var_values` is big enough to be
     // heap-allocated.
     indices: FxHashMap<Kind<'tcx>, CanonicalVar>,
@@ -330,7 +330,7 @@ fn canonicalize<V>(
         infcx: Option<&InferCtxt<'_, 'gcx, 'tcx>>,
         tcx: TyCtxt<'_, 'gcx, 'tcx>,
         canonicalize_region_mode: &dyn CanonicalizeRegionMode,
-        var_values: &mut SmallCanonicalVarValues<'tcx>,
+        query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
@@ -365,7 +365,7 @@ fn canonicalize<V>(
             canonicalize_region_mode,
             needs_canonical_flags,
             variables: SmallVec::new(),
-            var_values,
+            query_state,
             indices: FxHashMap::default(),
         };
         let out_value = value.fold_with(&mut canonicalizer);
@@ -396,11 +396,13 @@ fn canonicalize<V>(
     fn canonical_var(&mut self, info: CanonicalVarInfo, kind: Kind<'tcx>) -> CanonicalVar {
         let Canonicalizer {
             variables,
-            var_values,
+            query_state,
             indices,
             ..
         } = self;
 
+        let var_values = &mut query_state.var_values;
+
         // This code is hot. `variables` and `var_values` are usually small
         // (fewer than 8 elements ~95% of the time). They are SmallVec's to
         // avoid allocations in those cases. We also don't use `indices` to

src/librustc/infer/canonical/mod.rs

@@ -75,9 +75,16 @@ pub struct CanonicalVarValues<'tcx> {
     pub var_values: IndexVec<CanonicalVar, Kind<'tcx>>,
 }
 
-/// Like CanonicalVarValues, but for use in places where a SmallVec is
-/// appropriate.
-pub type SmallCanonicalVarValues<'tcx> = SmallVec<[Kind<'tcx>; 8]>;
+/// When we canonicalize a value to form a query, we wind up replacing
+/// various parts of it with canonical variables. This struct stores
+/// those replaced bits to remember for when we process the query
+/// result.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, RustcDecodable, RustcEncodable)]
+pub struct OriginalQueryValues<'tcx> {
+    /// This is equivalent to `CanonicalVarValues`, but using a
+    /// `SmallVec` yields a significant performance win.
+    pub var_values: SmallVec<[Kind<'tcx>; 8]>,
+}
 
 /// Information about a canonical variable that is included with the
 /// canonical value. This is sufficient information for code to create

src/librustc/infer/canonical/query_response.rs

@@ -20,7 +20,7 @@
 use infer::canonical::substitute::substitute_value;
 use infer::canonical::{
     Canonical, CanonicalVarKind, CanonicalVarValues, CanonicalizedQueryResponse, Certainty,
-    QueryRegionConstraint, QueryResponse, SmallCanonicalVarValues,
+    OriginalQueryValues, QueryRegionConstraint, QueryResponse,
 };
 use infer::region_constraints::{Constraint, RegionConstraintData};
 use infer::InferCtxtBuilder;
@@ -64,11 +64,15 @@ pub fn enter_canonical_trait_query<K, R>(
         K: TypeFoldable<'tcx>,
         R: Debug + Lift<'gcx> + TypeFoldable<'tcx>,
     {
-        self.enter_with_canonical(DUMMY_SP, canonical_key, |ref infcx, key, canonical_inference_vars| {
-            let fulfill_cx = &mut FulfillmentContext::new();
-            let value = operation(infcx, fulfill_cx, key)?;
-            infcx.make_canonicalized_query_response(canonical_inference_vars, value, fulfill_cx)
-        })
+        self.enter_with_canonical(
+            DUMMY_SP,
+            canonical_key,
+            |ref infcx, key, canonical_inference_vars| {
+                let fulfill_cx = &mut FulfillmentContext::new();
+                let value = operation(infcx, fulfill_cx, key)?;
+                infcx.make_canonicalized_query_response(canonical_inference_vars, value, fulfill_cx)
+            },
+        )
     }
 }
 
@@ -153,7 +157,8 @@ fn make_query_response<T>(
                 region_obligations
                     .iter()
                     .map(|(_, r_o)| (r_o.sup_type, r_o.sub_region)),
-                region_constraints)
+                region_constraints,
+            )
         });
 
         let certainty = if ambig_errors.is_empty() {
@@ -184,7 +189,7 @@ pub fn instantiate_query_response_and_region_obligations<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &SmallCanonicalVarValues<'tcx>,
+        original_values: &OriginalQueryValues<'tcx>,
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, R>
     where
@@ -250,7 +255,7 @@ pub fn instantiate_nll_query_response_and_region_obligations<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &SmallCanonicalVarValues<'tcx>,
+        original_values: &OriginalQueryValues<'tcx>,
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
         output_query_region_constraints: &mut Vec<QueryRegionConstraint<'tcx>>,
     ) -> InferResult<'tcx, R>
@@ -272,7 +277,7 @@ pub fn instantiate_nll_query_response_and_region_obligations<R>(
         // variable...
         let mut obligations = vec![];
 
-        for (index, original_value) in original_values.iter().enumerate() {
+        for (index, original_value) in original_values.var_values.iter().enumerate() {
             // ...with the value `v_r` of that variable from the query.
             let result_value = query_response.substitute_projected(self.tcx, &result_subst, |v| {
                 &v.var_values[CanonicalVar::new(index)]
@@ -344,7 +349,7 @@ fn query_response_substitution<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &SmallCanonicalVarValues<'tcx>,
+        original_values: &OriginalQueryValues<'tcx>,
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, CanonicalVarValues<'tcx>>
     where
@@ -385,7 +390,7 @@ fn query_response_substitution<R>(
     fn query_response_substitution_guess<R>(
         &self,
         cause: &ObligationCause<'tcx>,
-        original_values: &SmallCanonicalVarValues<'tcx>,
+        original_values: &OriginalQueryValues<'tcx>,
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> CanonicalVarValues<'tcx>
     where
@@ -401,7 +406,7 @@ fn query_response_substitution_guess<R>(
         // these values with the original inputs that were
         // canonicalized.
         let result_values = &query_response.value.var_values;
-        assert_eq!(original_values.len(), result_values.len());
+        assert_eq!(original_values.var_values.len(), result_values.len());
 
         // Quickly try to find initial values for the canonical
         // variables in the result in terms of the query. We do this
@@ -415,7 +420,7 @@ fn query_response_substitution_guess<R>(
 
         // In terms of our example above, we are iterating over pairs like:
         // [(?A, Vec<?0>), ('static, '?1), (?B, ?0)]
-        for (original_value, result_value) in original_values.iter().zip(result_values) {
+        for (original_value, result_value) in original_values.var_values.iter().zip(result_values) {
             match result_value.unpack() {
                 UnpackedKind::Type(result_value) => {
                     // e.g., here `result_value` might be `?0` in the example above...
@@ -461,7 +466,7 @@ fn unify_query_response_substitution_guess<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &SmallCanonicalVarValues<'tcx>,
+        original_values: &OriginalQueryValues<'tcx>,
         result_subst: &CanonicalVarValues<'tcx>,
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, ()>
@@ -478,7 +483,12 @@ fn unify_query_response_substitution_guess<R>(
 
         // Unify the original value for each variable with the value
         // taken from `query_response` (after applying `result_subst`).
-        Ok(self.unify_canonical_vars(cause, param_env, original_values, substituted_query_response)?)
+        Ok(self.unify_canonical_vars(
+            cause,
+            param_env,
+            original_values,
+            substituted_query_response,
+        )?)
     }
 
     /// Converts the region constraints resulting from a query into an
@@ -522,12 +532,12 @@ fn unify_canonical_vars(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        variables1: &SmallCanonicalVarValues<'tcx>,
+        variables1: &OriginalQueryValues<'tcx>,
         variables2: impl Fn(CanonicalVar) -> Kind<'tcx>,
     ) -> InferResult<'tcx, ()> {
         self.commit_if_ok(|_| {
             let mut obligations = vec![];
-            for (index, value1) in variables1.iter().enumerate() {
+            for (index, value1) in variables1.var_values.iter().enumerate() {
                 let value2 = variables2(CanonicalVar::new(index));
 
                 match (value1.unpack(), value2.unpack()) {

src/librustc/traits/query/dropck_outlives.rs

@@ -10,7 +10,7 @@
 
 use infer::at::At;
 use infer::InferOk;
-use smallvec::SmallVec;
+use infer::canonical::OriginalQueryValues;
 use std::iter::FromIterator;
 use syntax::source_map::Span;
 use ty::subst::Kind;
@@ -51,7 +51,7 @@ pub fn dropck_outlives(&self, ty: Ty<'tcx>) -> InferOk<'tcx, Vec<Kind<'tcx>>> {
         }
 
         let gcx = tcx.global_tcx();
-        let mut orig_values = SmallVec::new();
+        let mut orig_values = OriginalQueryValues::default();
         let c_ty = self.infcx.canonicalize_query(&self.param_env.and(ty), &mut orig_values);
         let span = self.cause.span;
         debug!("c_ty = {:?}", c_ty);

src/librustc/traits/query/evaluate_obligation.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use infer::InferCtxt;
-use smallvec::SmallVec;
+use infer::canonical::OriginalQueryValues;
 use traits::{EvaluationResult, PredicateObligation, SelectionContext,
              TraitQueryMode, OverflowError};
 
@@ -38,7 +38,7 @@ pub fn evaluate_obligation(
         &self,
         obligation: &PredicateObligation<'tcx>,
     ) -> Result<EvaluationResult, OverflowError> {
-        let mut _orig_values = SmallVec::new();
+        let mut _orig_values = OriginalQueryValues::default();
         let c_pred = self.canonicalize_query(&obligation.param_env.and(obligation.predicate),
                                              &mut _orig_values);
         // Run canonical query. If overflow occurs, rerun from scratch but this time

src/librustc/traits/query/normalize.rs

@@ -13,9 +13,9 @@
 //! `normalize_projection_ty` query when it encounters projections.
 
 use infer::at::At;
+use infer::canonical::OriginalQueryValues;
 use infer::{InferCtxt, InferOk};
 use mir::interpret::{ConstValue, GlobalId};
-use smallvec::SmallVec;
 use traits::project::Normalized;
 use traits::{Obligation, ObligationCause, PredicateObligation, Reveal};
 use ty::fold::{TypeFoldable, TypeFolder};
@@ -154,7 +154,7 @@ fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
 
                 let gcx = self.infcx.tcx.global_tcx();
 
-                let mut orig_values = SmallVec::new();
+                let mut orig_values = OriginalQueryValues::default();
                 let c_data = self.infcx.canonicalize_query(
                     &self.param_env.and(*data), &mut orig_values);
                 debug!("QueryNormalizer: c_data = {:#?}", c_data);

src/librustc/traits/query/outlives_bounds.rs

@@ -9,9 +9,9 @@
 // except according to those terms.
 
 use infer::InferCtxt;
+use infer::canonical::OriginalQueryValues;
 use syntax::ast;
 use syntax::source_map::Span;
-use smallvec::SmallVec;
 use traits::{FulfillmentContext, ObligationCause, TraitEngine, TraitEngineExt};
 use traits::query::NoSolution;
 use ty::{self, Ty, TyCtxt};
@@ -105,7 +105,7 @@ pub fn implied_outlives_bounds(
     ) -> Vec<OutlivesBound<'tcx>> {
         debug!("implied_outlives_bounds(ty = {:?})", ty);
 
-        let mut orig_values = SmallVec::new();
+        let mut orig_values = OriginalQueryValues::default();
         let key = self.canonicalize_query(&param_env.and(ty), &mut orig_values);
         let result = match self.tcx.global_tcx().implied_outlives_bounds(key) {
             Ok(r) => r,

src/librustc/traits/query/type_op/mod.rs

@@ -9,10 +9,10 @@
 // except according to those terms.
 
 use infer::canonical::{
-    Canonical, Canonicalized, CanonicalizedQueryResponse, QueryRegionConstraint, QueryResponse,
+    Canonical, Canonicalized, CanonicalizedQueryResponse, OriginalQueryValues,
+    QueryRegionConstraint, QueryResponse,
 };
 use infer::{InferCtxt, InferOk};
-use smallvec::SmallVec;
 use std::fmt;
 use std::rc::Rc;
 use traits::query::Fallible;
@@ -106,7 +106,7 @@ fn fully_perform_into(
         // `canonicalize_hr_query_hack` here because of things
         // like the subtype query, which go awry around
         // `'static` otherwise.
-        let mut canonical_var_values = SmallVec::new();
+        let mut canonical_var_values = OriginalQueryValues::default();
         let canonical_self =
             infcx.canonicalize_hr_query_hack(&query_key, &mut canonical_var_values);
         let canonical_result = Self::perform_query(infcx.tcx, canonical_self)?;

src/librustc_traits/chalk_context.rs

@@ -10,7 +10,9 @@
 
 use chalk_engine::fallible::Fallible as ChalkEngineFallible;
 use chalk_engine::{context, hh::HhGoal, DelayedLiteral, ExClause};
-use rustc::infer::canonical::{Canonical, CanonicalVarValues, QueryRegionConstraint, QueryResponse};
+use rustc::infer::canonical::{
+    Canonical, CanonicalVarValues, OriginalQueryValues, QueryRegionConstraint, QueryResponse,
+};
 use rustc::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime};
 use rustc::traits::{
     WellFormed,
@@ -26,7 +28,6 @@
 use rustc::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor};
 use rustc::ty::subst::Kind;
 use rustc::ty::{self, TyCtxt};
-use smallvec::SmallVec;
 
 use std::fmt::{self, Debug};
 use std::marker::PhantomData;
@@ -390,7 +391,7 @@ fn canonicalize_goal(
         &mut self,
         value: &ty::ParamEnvAnd<'tcx, Goal<'tcx>>,
     ) -> Canonical<'gcx, ty::ParamEnvAnd<'gcx, Goal<'gcx>>> {
-        let mut _orig_values = SmallVec::new();
+        let mut _orig_values = OriginalQueryValues::default();
         self.infcx.canonicalize_query(value, &mut _orig_values)
     }