Avoid most allocations in `Canonicalizer`.

Extra allocations are a significant cost of NLL, and the most common
ones come from within `Canonicalizer`. In particular, `canonical_var()`
contains this code:

    indices
	.entry(kind)
	.or_insert_with(|| {
	    let cvar1 = variables.push(info);
	    let cvar2 = var_values.push(kind);
	    assert_eq!(cvar1, cvar2);
	    cvar1
	})
	.clone()

`variables` and `var_values` are `Vec`s. `indices` is a `HashMap` used
to track what elements have been inserted into `var_values`. If `kind`
hasn't been seen before, `indices`, `variables` and `var_values` all get
a new element. (The number of elements in each container is always the
same.) This results in lots of allocations.

In practice, most of the time these containers only end up holding a few
elements. This PR changes them to avoid heap allocations in the common
case, by changing the `Vec`s to `SmallVec`s and only using `indices`
once enough elements are present. (When the number of elements is small,
a direct linear search of `var_values` is as good or better than a
hashmap lookup.)

The changes to `variables` are straightforward and contained within
`Canonicalizer`. The changes to `indices` are more complex but also
contained within `Canonicalizer`. The changes to `var_values` are more
intrusive because they require defining a new type
`SmallCanonicalVarValues` -- which is to `CanonicalVarValues` as
`SmallVec` is to `Vec -- and passing stack-allocated values of that type
in from outside.

All this speeds up a number of NLL "check" builds, the best by 2%.

src/librustc/infer/canonical/canonicalizer.rs

@@ -16,8 +16,8 @@
 //! [c]: https://rust-lang-nursery.github.io/rustc-guide/traits/canonicalization.html
 
 use infer::canonical::{
-    Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, CanonicalVarValues,
-    Canonicalized,
+    Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, Canonicalized,
+    SmallCanonicalVarValues,
 };
 use infer::InferCtxt;
 use std::sync::atomic::Ordering;
@@ -26,7 +26,8 @@
 use ty::{self, CanonicalVar, Lift, Slice, Ty, TyCtxt, TypeFlags};
 
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::indexed_vec::IndexVec;
+use rustc_data_structures::indexed_vec::Idx;
+use rustc_data_structures::small_vec::SmallVec;
 
 impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> {
     /// Canonicalizes a query value `V`. When we canonicalize a query,
@@ -47,7 +48,8 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> {
     pub fn canonicalize_query<V>(
         &self,
         value: &V,
-    ) -> (Canonicalized<'gcx, V>, CanonicalVarValues<'tcx>)
+        var_values: &mut SmallCanonicalVarValues<'tcx>
+    ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
     {
@@ -65,6 +67,7 @@ pub fn canonicalize_query<V>(
                 static_region: true,
                 other_free_regions: true,
             },
+            var_values,
         )
     }
 
@@ -96,10 +99,11 @@ pub fn canonicalize_query<V>(
     pub fn canonicalize_response<V>(
         &self,
         value: &V,
-    ) -> (Canonicalized<'gcx, V>, CanonicalVarValues<'tcx>)
+    ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
     {
+        let mut var_values = SmallVec::new();
         Canonicalizer::canonicalize(
             value,
             Some(self),
@@ -108,6 +112,7 @@ pub fn canonicalize_response<V>(
                 static_region: false,
                 other_free_regions: false,
             },
+            &mut var_values
         )
     }
 
@@ -123,7 +128,8 @@ pub fn canonicalize_response<V>(
     pub fn canonicalize_hr_query_hack<V>(
         &self,
         value: &V,
-    ) -> (Canonicalized<'gcx, V>, CanonicalVarValues<'tcx>)
+        var_values: &mut SmallCanonicalVarValues<'tcx>
+    ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
     {
@@ -141,6 +147,7 @@ pub fn canonicalize_hr_query_hack<V>(
                 static_region: false,
                 other_free_regions: true,
             },
+            var_values
         )
     }
 }
@@ -163,9 +170,11 @@ fn any(&self) -> bool {
 struct Canonicalizer<'cx, 'gcx: 'tcx, 'tcx: 'cx> {
     infcx: Option<&'cx InferCtxt<'cx, 'gcx, 'tcx>>,
     tcx: TyCtxt<'cx, 'gcx, 'tcx>,
-    variables: IndexVec<CanonicalVar, CanonicalVarInfo>,
+    variables: SmallVec<[CanonicalVarInfo; 8]>,
+    var_values: &'cx mut SmallCanonicalVarValues<'tcx>,
+    // Note that indices is only used once `var_values` is big enough to be
+    // heap-allocated.
     indices: FxHashMap<Kind<'tcx>, CanonicalVar>,
-    var_values: IndexVec<CanonicalVar, Kind<'tcx>>,
     canonicalize_region_mode: CanonicalizeRegionMode,
     needs_canonical_flags: TypeFlags,
 }
@@ -295,7 +304,8 @@ fn canonicalize<V>(
         infcx: Option<&'cx InferCtxt<'cx, 'gcx, 'tcx>>,
         tcx: TyCtxt<'cx, 'gcx, 'tcx>,
         canonicalize_region_mode: CanonicalizeRegionMode,
-    ) -> (Canonicalized<'gcx, V>, CanonicalVarValues<'tcx>)
+        var_values: &'cx mut SmallCanonicalVarValues<'tcx>
+    ) -> Canonicalized<'gcx, V>
     where
         V: TypeFoldable<'tcx> + Lift<'gcx>,
     {
@@ -320,10 +330,7 @@ fn canonicalize<V>(
                 variables: Slice::empty(),
                 value: out_value,
             };
-            let values = CanonicalVarValues {
-                var_values: IndexVec::default(),
-            };
-            return (canon_value, values);
+            return canon_value;
         }
 
         let mut canonicalizer = Canonicalizer {
@@ -331,9 +338,9 @@ fn canonicalize<V>(
             tcx,
             canonicalize_region_mode,
             needs_canonical_flags,
-            variables: IndexVec::default(),
+            variables: SmallVec::new(),
+            var_values,
             indices: FxHashMap::default(),
-            var_values: IndexVec::default(),
         };
         let out_value = value.fold_with(&mut canonicalizer);
 
@@ -348,16 +355,12 @@ fn canonicalize<V>(
             )
         });
 
-        let canonical_variables = tcx.intern_canonical_var_infos(&canonicalizer.variables.raw);
+        let canonical_variables = tcx.intern_canonical_var_infos(&canonicalizer.variables);
 
-        let canonical_value = Canonical {
+        Canonical {
             variables: canonical_variables,
             value: out_value,
-        };
-        let canonical_var_values = CanonicalVarValues {
-            var_values: canonicalizer.var_values,
-        };
-        (canonical_value, canonical_var_values)
+        }
     }
 
     /// Creates a canonical variable replacing `kind` from the input,
@@ -366,21 +369,54 @@ fn canonicalize<V>(
     /// potentially a free region).
     fn canonical_var(&mut self, info: CanonicalVarInfo, kind: Kind<'tcx>) -> CanonicalVar {
         let Canonicalizer {
-            indices,
             variables,
             var_values,
+            indices,
             ..
         } = self;
 
-        indices
-            .entry(kind)
-            .or_insert_with(|| {
-                let cvar1 = variables.push(info);
-                let cvar2 = var_values.push(kind);
-                assert_eq!(cvar1, cvar2);
-                cvar1
-            })
-            .clone()
+        // This code is hot. `variables` and `var_values` are usually small
+        // (fewer than 8 elements ~95% of the time). They are SmallVec's to
+        // avoid allocations in those cases. We also don't use `indices` to
+        // determine if a kind has been seen before until the limit of 8 has
+        // been exceeded, to also avoid allocations for `indices`.
+        if var_values.is_array() {
+            // `var_values` is stack-allocated. `indices` isn't used yet. Do a
+            // direct linear search of `var_values`.
+            if let Some(idx) = var_values.iter().position(|&k| k == kind) {
+                // `kind` is already present in `var_values`.
+                CanonicalVar::new(idx)
+            } else {
+                // `kind` isn't present in `var_values`. Append it. Likewise
+                // for `info` and `variables`.
+                variables.push(info);
+                var_values.push(kind);
+                assert_eq!(variables.len(), var_values.len());
+
+                // If `var_values` has become big enough to be heap-allocated,
+                // fill up `indices` to facilitate subsequent lookups.
+                if !var_values.is_array() {
+                    assert!(indices.is_empty());
+                    *indices =
+                        var_values.iter()
+                            .enumerate()
+                            .map(|(i, &kind)| (kind, CanonicalVar::new(i)))
+                            .collect();
+                }
+                // The cv is the index of the appended element.
+                CanonicalVar::new(var_values.len() - 1)
+            }
+        } else {
+            // `var_values` is large. Do a hashmap search via `indices`.
+            *indices
+                .entry(kind)
+                .or_insert_with(|| {
+                    variables.push(info);
+                    var_values.push(kind);
+                    assert_eq!(variables.len(), var_values.len());
+                    CanonicalVar::new(variables.len() - 1)
+                })
+        }
     }
 
     /// Given a type variable `ty_var` of the given kind, first check

src/librustc/infer/canonical/mod.rs

@@ -33,6 +33,7 @@
 
 use infer::{InferCtxt, RegionVariableOrigin, TypeVariableOrigin};
 use rustc_data_structures::indexed_vec::IndexVec;
+use rustc_data_structures::small_vec::SmallVec;
 use rustc_data_structures::sync::Lrc;
 use serialize::UseSpecializedDecodable;
 use std::ops::Index;
@@ -74,6 +75,10 @@ pub struct CanonicalVarValues<'tcx> {
     pub var_values: IndexVec<CanonicalVar, Kind<'tcx>>,
 }
 
+/// Like CanonicalVarValues, but for use in places where a SmallVec is
+/// appropriate.
+pub type SmallCanonicalVarValues<'tcx> = SmallVec<[Kind<'tcx>; 8]>;
+
 /// Information about a canonical variable that is included with the
 /// canonical value. This is sufficient information for code to create
 /// a copy of the canonical value in some other inference context,
@@ -281,10 +286,6 @@ impl<'a, 'tcx, T> Lift<'tcx> for Canonical<'a, T> {
 }
 
 impl<'tcx> CanonicalVarValues<'tcx> {
-    fn iter<'a>(&'a self) -> impl Iterator<Item = Kind<'tcx>> + 'a {
-        self.var_values.iter().cloned()
-    }
-
     fn len(&self) -> usize {
         self.var_values.len()
     }

src/librustc/infer/canonical/query_result.rs

@@ -19,7 +19,7 @@
 
 use infer::canonical::substitute::substitute_value;
 use infer::canonical::{Canonical, CanonicalVarKind, CanonicalVarValues, CanonicalizedQueryResult,
-                       Certainty, QueryRegionConstraint, QueryResult};
+                       Certainty, QueryRegionConstraint, QueryResult, SmallCanonicalVarValues};
 use infer::region_constraints::{Constraint, RegionConstraintData};
 use infer::InferCtxtBuilder;
 use infer::{InferCtxt, InferOk, InferResult, RegionObligation};
@@ -103,7 +103,7 @@ pub fn make_canonicalized_query_result<T>(
         T: Debug + Lift<'gcx> + TypeFoldable<'tcx>,
     {
         let query_result = self.make_query_result(inference_vars, answer, fulfill_cx)?;
-        let (canonical_result, _) = self.canonicalize_response(&query_result);
+        let canonical_result = self.canonicalize_response(&query_result);
 
         debug!(
             "make_canonicalized_query_result: canonical_result = {:#?}",
@@ -186,7 +186,7 @@ pub fn instantiate_query_result_and_region_obligations<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &CanonicalVarValues<'tcx>,
+        original_values: &SmallCanonicalVarValues<'tcx>,
         query_result: &Canonical<'tcx, QueryResult<'tcx, R>>,
     ) -> InferResult<'tcx, R>
     where
@@ -252,7 +252,7 @@ pub fn instantiate_nll_query_result_and_region_obligations<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &CanonicalVarValues<'tcx>,
+        original_values: &SmallCanonicalVarValues<'tcx>,
         query_result: &Canonical<'tcx, QueryResult<'tcx, R>>,
         output_query_region_constraints: &mut Vec<QueryRegionConstraint<'tcx>>,
     ) -> InferResult<'tcx, R>
@@ -274,10 +274,11 @@ pub fn instantiate_nll_query_result_and_region_obligations<R>(
         // variable...
         let mut obligations = vec![];
 
-        for (index, original_value) in original_values.var_values.iter_enumerated() {
+        for (index, original_value) in original_values.iter().enumerate() {
             // ...with the value `v_r` of that variable from the query.
             let result_value = query_result
-                .substitute_projected(self.tcx, &result_subst, |v| &v.var_values[index]);
+                .substitute_projected(self.tcx, &result_subst,
+                                      |v| &v.var_values[CanonicalVar::new(index)]);
             match (original_value.unpack(), result_value.unpack()) {
                 (UnpackedKind::Lifetime(ty::ReErased), UnpackedKind::Lifetime(ty::ReErased)) => {
                     // no action needed
@@ -341,7 +342,7 @@ fn query_result_substitution<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &CanonicalVarValues<'tcx>,
+        original_values: &SmallCanonicalVarValues<'tcx>,
         query_result: &Canonical<'tcx, QueryResult<'tcx, R>>,
     ) -> InferResult<'tcx, CanonicalVarValues<'tcx>>
     where
@@ -382,7 +383,7 @@ fn query_result_substitution<R>(
     fn query_result_substitution_guess<R>(
         &self,
         cause: &ObligationCause<'tcx>,
-        original_values: &CanonicalVarValues<'tcx>,
+        original_values: &SmallCanonicalVarValues<'tcx>,
         query_result: &Canonical<'tcx, QueryResult<'tcx, R>>,
     ) -> CanonicalVarValues<'tcx>
     where
@@ -418,14 +419,14 @@ fn query_result_substitution_guess<R>(
                     // e.g., here `result_value` might be `?0` in the example above...
                     if let ty::TyInfer(ty::InferTy::CanonicalTy(index)) = result_value.sty {
                         // in which case we would set `canonical_vars[0]` to `Some(?U)`.
-                        opt_values[index] = Some(original_value);
+                        opt_values[index] = Some(*original_value);
                     }
                 }
                 UnpackedKind::Lifetime(result_value) => {
                     // e.g., here `result_value` might be `'?1` in the example above...
                     if let &ty::RegionKind::ReCanonical(index) = result_value {
                         // in which case we would set `canonical_vars[0]` to `Some('static)`.
-                        opt_values[index] = Some(original_value);
+                        opt_values[index] = Some(*original_value);
                     }
                 }
             }
@@ -459,7 +460,7 @@ fn unify_query_result_substitution_guess<R>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        original_values: &CanonicalVarValues<'tcx>,
+        original_values: &SmallCanonicalVarValues<'tcx>,
         result_subst: &CanonicalVarValues<'tcx>,
         query_result: &Canonical<'tcx, QueryResult<'tcx, R>>,
     ) -> InferResult<'tcx, ()>
@@ -522,13 +523,13 @@ fn unify_canonical_vars(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        variables1: &CanonicalVarValues<'tcx>,
+        variables1: &SmallCanonicalVarValues<'tcx>,
         variables2: impl Fn(CanonicalVar) -> Kind<'tcx>,
     ) -> InferResult<'tcx, ()> {
         self.commit_if_ok(|_| {
             let mut obligations = vec![];
-            for (index, value1) in variables1.var_values.iter_enumerated() {
-                let value2 = variables2(index);
+            for (index, value1) in variables1.iter().enumerate() {
+                let value2 = variables2(CanonicalVar::new(index));
 
                 match (value1.unpack(), value2.unpack()) {
                     (UnpackedKind::Type(v1), UnpackedKind::Type(v2)) => {

src/librustc/infer/canonical/substitute.rs

@@ -46,7 +46,7 @@ pub fn substitute_projected<T>(
     where
         T: TypeFoldable<'tcx>,
     {
-        assert_eq!(self.variables.len(), var_values.var_values.len());
+        assert_eq!(self.variables.len(), var_values.len());
         let value = projection_fn(&self.value);
         substitute_value(tcx, var_values, value)
     }

src/librustc/traits/query/dropck_outlives.rs

@@ -10,6 +10,7 @@
 
 use infer::at::At;
 use infer::InferOk;
+use rustc_data_structures::small_vec::SmallVec;
 use std::iter::FromIterator;
 use syntax::codemap::Span;
 use ty::subst::Kind;
@@ -50,7 +51,8 @@ pub fn dropck_outlives(&self, ty: Ty<'tcx>) -> InferOk<'tcx, Vec<Kind<'tcx>>> {
         }
 
         let gcx = tcx.global_tcx();
-        let (c_ty, orig_values) = self.infcx.canonicalize_query(&self.param_env.and(ty));
+        let mut orig_values = SmallVec::new();
+        let c_ty = self.infcx.canonicalize_query(&self.param_env.and(ty), &mut orig_values);
         let span = self.cause.span;
         debug!("c_ty = {:?}", c_ty);
         match &gcx.dropck_outlives(c_ty) {

src/librustc/traits/query/evaluate_obligation.rs

@@ -9,6 +9,7 @@
 // except according to those terms.
 
 use infer::InferCtxt;
+use rustc_data_structures::small_vec::SmallVec;
 use traits::{EvaluationResult, PredicateObligation, SelectionContext,
              TraitQueryMode, OverflowError};
 
@@ -38,8 +39,9 @@ fn evaluate_obligation(
         &self,
         obligation: &PredicateObligation<'tcx>,
     ) -> EvaluationResult {
-        let (c_pred, _) =
-            self.canonicalize_query(&obligation.param_env.and(obligation.predicate));
+        let mut _orig_values = SmallVec::new();
+        let c_pred = self.canonicalize_query(&obligation.param_env.and(obligation.predicate),
+                                             &mut _orig_values);
         // Run canonical query. If overflow occurs, rerun from scratch but this time
         // in standard trait query mode so that overflow is handled appropriately
         // within `SelectionContext`.

src/librustc/traits/query/normalize.rs

@@ -15,6 +15,7 @@
 use infer::{InferCtxt, InferOk};
 use infer::at::At;
 use mir::interpret::{GlobalId, ConstValue};
+use rustc_data_structures::small_vec::SmallVec;
 use traits::{Obligation, ObligationCause, PredicateObligation, Reveal};
 use traits::project::Normalized;
 use ty::{self, Ty, TyCtxt};
@@ -147,8 +148,9 @@ fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
 
                 let gcx = self.infcx.tcx.global_tcx();
 
-                let (c_data, orig_values) =
-                    self.infcx.canonicalize_query(&self.param_env.and(*data));
+                let mut orig_values = SmallVec::new();
+                let c_data =
+                    self.infcx.canonicalize_query(&self.param_env.and(*data), &mut orig_values);
                 debug!("QueryNormalizer: c_data = {:#?}", c_data);
                 debug!("QueryNormalizer: orig_values = {:#?}", orig_values);
                 match gcx.normalize_projection_ty(c_data) {

src/librustc/traits/query/type_op/mod.rs

@@ -11,6 +11,7 @@
 use infer::canonical::{Canonical, Canonicalized, CanonicalizedQueryResult, QueryRegionConstraint,
                        QueryResult};
 use infer::{InferCtxt, InferOk};
+use rustc_data_structures::small_vec::SmallVec;
 use std::fmt;
 use std::rc::Rc;
 use traits::query::Fallible;
@@ -103,7 +104,9 @@ fn fully_perform_into(
         // `canonicalize_hr_query_hack` here because of things
         // like the subtype query, which go awry around
         // `'static` otherwise.
-        let (canonical_self, canonical_var_values) = infcx.canonicalize_hr_query_hack(&query_key);
+        let mut canonical_var_values = SmallVec::new();
+        let canonical_self =
+            infcx.canonicalize_hr_query_hack(&query_key, &mut canonical_var_values);
         let canonical_result = Self::perform_query(infcx.tcx, canonical_self)?;
         let canonical_result = Self::shrink_to_tcx_lifetime(&canonical_result);
 

src/librustc_data_structures/accumulate_vec.rs

@@ -46,6 +46,13 @@ pub fn new() -> AccumulateVec<A> {
         AccumulateVec::Array(ArrayVec::new())
     }
 
+    pub fn is_array(&self) -> bool {
+        match self {
+            AccumulateVec::Array(..) => true,
+            AccumulateVec::Heap(..) => false,
+        }
+    }
+
     pub fn one(el: A::Element) -> Self {
         iter::once(el).collect()
     }

src/librustc_data_structures/small_vec.rs

@@ -50,6 +50,10 @@ pub fn new() -> Self {
         SmallVec(AccumulateVec::new())
     }
 
+    pub fn is_array(&self) -> bool {
+        self.0.is_array()
+    }
+
     pub fn with_capacity(cap: usize) -> Self {
         let mut vec = SmallVec::new();
         vec.reserve(cap);

src/librustc_traits/chalk_context.rs

@@ -25,6 +25,7 @@
 use rustc::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor};
 use rustc::ty::subst::Kind;
 use rustc::ty::{self, TyCtxt};
+use rustc_data_structures::small_vec::SmallVec;
 
 use std::fmt::{self, Debug};
 use std::marker::PhantomData;
@@ -388,14 +389,15 @@ fn canonicalize_goal(
         &mut self,
         value: &ty::ParamEnvAnd<'tcx, Goal<'tcx>>,
     ) -> Canonical<'gcx, ty::ParamEnvAnd<'gcx, Goal<'gcx>>> {
-        self.infcx.canonicalize_query(value).0
+        let mut _orig_values = SmallVec::new();
+        self.infcx.canonicalize_query(value, &mut _orig_values)
     }
 
     fn canonicalize_ex_clause(
         &mut self,
         value: &ChalkExClause<'tcx>,
     ) -> Canonical<'gcx, ChalkExClause<'gcx>> {
-        self.infcx.canonicalize_response(value).0
+        self.infcx.canonicalize_response(value)
     }
 
     fn canonicalize_constrained_subst(
@@ -403,9 +405,7 @@ fn canonicalize_constrained_subst(
         subst: CanonicalVarValues<'tcx>,
         constraints: Vec<QueryRegionConstraint<'tcx>>,
     ) -> Canonical<'gcx, ConstrainedSubst<'gcx>> {
-        self.infcx
-            .canonicalize_response(&ConstrainedSubst { subst, constraints })
-            .0
+        self.infcx.canonicalize_response(&ConstrainedSubst { subst, constraints })
     }
 
     fn u_canonicalize_goal(

src/librustc_typeck/check/mod.rs

@@ -942,7 +942,7 @@ fn visit_local(&mut self, local: &'gcx hir::Local) {
             Some(ref ty) => {
                 let o_ty = self.fcx.to_ty(&ty);
 
-                let (c_ty, _orig_values) = self.fcx.inh.infcx.canonicalize_response(&o_ty);
+                let c_ty = self.fcx.inh.infcx.canonicalize_response(&o_ty);
                 debug!("visit_local: ty.hir_id={:?} o_ty={:?} c_ty={:?}", ty.hir_id, o_ty, c_ty);
                 self.fcx.tables.borrow_mut().user_provided_tys_mut().insert(ty.hir_id, c_ty);