diff --git a/src/librustc_trans/collector.rs b/src/librustc_trans/collector.rs index a76abcf7b49a69ab5e187c19bd9385f6ee344c05..19d7198b49b7b1b614fac1b256c6488ebd0d10bd 100644 --- a/src/librustc_trans/collector.rs +++ b/src/librustc_trans/collector.rs @@ -208,6 +208,8 @@ use trans_item::{TransItem, DefPathBasedNames, InstantiationMode}; +use rustc_data_structures::bitvec::BitVector; + #[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)] pub enum TransItemCollectionMode { Eager, @@ -217,12 +219,16 @@ pub enum TransItemCollectionMode { /// Maps every translation item to all translation items it references in its /// body. pub struct InliningMap<'tcx> { - // Maps a source translation item to a range of target translation items - // that are potentially inlined by LLVM into the source. + // Maps a source translation item to the range of translation items + // accessed by it. // The two numbers in the tuple are the start (inclusive) and // end index (exclusive) within the `targets` vecs. index: FxHashMap, (usize, usize)>, targets: Vec>, + + // Contains one bit per translation item in the `targets` field. That bit + // is true if that translation item needs to be inlined into every CGU. + inlines: BitVector, } impl<'tcx> InliningMap<'tcx> { @@ -231,18 +237,33 @@ fn new() -> InliningMap<'tcx> { InliningMap { index: FxHashMap(), targets: Vec::new(), + inlines: BitVector::new(1024), } } - fn record_inlining_canditates(&mut self, - source: TransItem<'tcx>, - targets: I) - where I: Iterator> + fn record_accesses(&mut self, + source: TransItem<'tcx>, + targets: I) + where I: Iterator, bool)> { assert!(!self.index.contains_key(&source)); let start_index = self.targets.len(); - self.targets.extend(targets); + let (targets_size_hint, targets_size_hint_max) = targets.size_hint(); + debug_assert_eq!(targets_size_hint_max, Some(targets_size_hint)); + let new_items_count = targets_size_hint; + let new_items_count_total = new_items_count + self.targets.len(); + + self.targets.reserve(new_items_count); + self.inlines.grow(new_items_count_total); + + for (i, (target, inline)) in targets.enumerate() { + self.targets.push(target); + if inline { + self.inlines.insert(i + start_index); + } + } + let end_index = self.targets.len(); self.index.insert(source, (start_index, end_index)); } @@ -250,14 +271,27 @@ fn record_inlining_canditates(&mut self, // Internally iterate over all items referenced by `source` which will be // made available for inlining. pub fn with_inlining_candidates(&self, source: TransItem<'tcx>, mut f: F) - where F: FnMut(TransItem<'tcx>) { - if let Some(&(start_index, end_index)) = self.index.get(&source) - { - for candidate in &self.targets[start_index .. end_index] { - f(*candidate) + where F: FnMut(TransItem<'tcx>) + { + if let Some(&(start_index, end_index)) = self.index.get(&source) { + for (i, candidate) in self.targets[start_index .. end_index] + .iter() + .enumerate() { + if self.inlines.contains(start_index + i) { + f(*candidate); + } } } } + + // Internally iterate over all items and the things each accesses. + pub fn iter_accesses(&self, mut f: F) + where F: FnMut(TransItem<'tcx>, &[TransItem<'tcx>]) + { + for (&accessor, &(start_index, end_index)) in &self.index { + f(accessor, &self.targets[start_index .. end_index]) + } + } } pub fn collect_crate_translation_items<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>, @@ -340,7 +374,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>, recursion_depth_reset = None; - collect_neighbours(scx, instance, &mut neighbors); + collect_neighbours(scx, instance, true, &mut neighbors); } TransItem::Fn(instance) => { // Sanity check whether this ended up being collected accidentally @@ -352,14 +386,14 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>, recursion_depths)); check_type_length_limit(scx.tcx(), instance); - collect_neighbours(scx, instance, &mut neighbors); + collect_neighbours(scx, instance, false, &mut neighbors); } TransItem::GlobalAsm(..) => { recursion_depth_reset = None; } } - record_inlining_canditates(scx.tcx(), starting_point, &neighbors[..], inlining_map); + record_accesses(scx.tcx(), starting_point, &neighbors[..], inlining_map); for neighbour in neighbors { collect_items_rec(scx, neighbour, visited, recursion_depths, inlining_map); @@ -372,7 +406,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>, debug!("END collect_items_rec({})", starting_point.to_string(scx.tcx())); } -fn record_inlining_canditates<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, +fn record_accesses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, caller: TransItem<'tcx>, callees: &[TransItem<'tcx>], inlining_map: &mut InliningMap<'tcx>) { @@ -380,11 +414,12 @@ fn record_inlining_canditates<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, trans_item.instantiation_mode(tcx) == InstantiationMode::LocalCopy }; - let inlining_candidates = callees.into_iter() - .map(|x| *x) - .filter(is_inlining_candidate); + let accesses = callees.into_iter() + .map(|trans_item| { + (*trans_item, is_inlining_candidate(trans_item)) + }); - inlining_map.record_inlining_canditates(caller, inlining_candidates); + inlining_map.record_accesses(caller, accesses); } fn check_recursion_limit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, @@ -459,7 +494,8 @@ struct MirNeighborCollector<'a, 'tcx: 'a> { scx: &'a SharedCrateContext<'a, 'tcx>, mir: &'a mir::Mir<'tcx>, output: &'a mut Vec>, - param_substs: &'tcx Substs<'tcx> + param_substs: &'tcx Substs<'tcx>, + const_context: bool, } impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { @@ -540,7 +576,7 @@ fn visit_constant(&mut self, constant: &mir::Constant<'tcx>, location: Location) let substs = self.scx.tcx().trans_apply_param_substs(self.param_substs, &substs); let instance = monomorphize::resolve(self.scx, def_id, substs); - collect_neighbours(self.scx, instance, self.output); + collect_neighbours(self.scx, instance, true, self.output); } self.super_constant(constant, location); @@ -556,8 +592,16 @@ fn visit_terminator_kind(&mut self, match *kind { mir::TerminatorKind::Call { ref func, .. } => { let callee_ty = func.ty(self.mir, tcx); - let callee_ty = tcx.trans_apply_param_substs(self.param_substs, &callee_ty); - visit_fn_use(self.scx, callee_ty, true, &mut self.output); + + let skip_const = self.const_context && match callee_ty.sty { + ty::TyFnDef(def_id, _) => self.scx.tcx().is_const_fn(def_id), + _ => false + }; + + if !skip_const { + let callee_ty = tcx.trans_apply_param_substs(self.param_substs, &callee_ty); + visit_fn_use(self.scx, callee_ty, true, &mut self.output); + } } mir::TerminatorKind::Drop { ref location, .. } | mir::TerminatorKind::DropAndReplace { ref location, .. } => { @@ -576,6 +620,22 @@ fn visit_terminator_kind(&mut self, self.super_terminator_kind(block, kind, location); } + + fn visit_static(&mut self, + static_: &mir::Static<'tcx>, + context: mir::visit::LvalueContext<'tcx>, + location: Location) { + debug!("visiting static {:?} @ {:?}", static_.def_id, location); + + let tcx = self.scx.tcx(); + let instance = Instance::mono(tcx, static_.def_id); + if should_trans_locally(tcx, &instance) { + let node_id = tcx.hir.as_local_node_id(static_.def_id).unwrap(); + self.output.push(TransItem::Static(node_id)); + } + + self.super_static(static_, context, location); + } } fn visit_drop_use<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>, @@ -850,8 +910,14 @@ fn visit_item(&mut self, item: &'v hir::Item) { // const items only generate translation items if they are // actually used somewhere. Just declaring them is insufficient. } - hir::ItemFn(.., ref generics, _) => { - if !generics.is_type_parameterized() { + hir::ItemFn(_, _, constness, _, ref generics, _) => { + let is_const = match constness { + hir::Constness::Const => true, + hir::Constness::NotConst => false, + }; + + if !generics.is_type_parameterized() && + (!is_const || self.mode == TransItemCollectionMode::Eager) { let def_id = self.scx.tcx().hir.local_def_id(item.id); debug!("RootCollector: ItemFn({})", @@ -872,12 +938,13 @@ fn visit_trait_item(&mut self, _: &'v hir::TraitItem) { fn visit_impl_item(&mut self, ii: &'v hir::ImplItem) { match ii.node { hir::ImplItemKind::Method(hir::MethodSig { + constness, ref generics, .. }, _) => { let hir_map = &self.scx.tcx().hir; let parent_node_id = hir_map.get_parent_node(ii.id); - let is_impl_generic = match hir_map.expect_item(parent_node_id) { + let is_impl_generic = || match hir_map.expect_item(parent_node_id) { &hir::Item { node: hir::ItemImpl(_, _, _, ref generics, ..), .. @@ -889,7 +956,14 @@ fn visit_impl_item(&mut self, ii: &'v hir::ImplItem) { } }; - if !generics.is_type_parameterized() && !is_impl_generic { + let is_const = match constness { + hir::Constness::Const => true, + hir::Constness::NotConst => false, + }; + + if (!is_const || self.mode == TransItemCollectionMode::Eager) && + !generics.is_type_parameterized() && + !is_impl_generic() { let def_id = self.scx.tcx().hir.local_def_id(ii.id); debug!("RootCollector: MethodImplItem({})", @@ -958,6 +1032,7 @@ fn create_trans_items_for_default_impls<'a, 'tcx>(scx: &SharedCrateContext<'a, ' /// Scan the MIR in order to find function calls, closures, and drop-glue fn collect_neighbours<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>, instance: Instance<'tcx>, + const_context: bool, output: &mut Vec>) { let mir = scx.tcx().instance_mir(instance.def); @@ -966,7 +1041,8 @@ fn collect_neighbours<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>, scx: scx, mir: &mir, output: output, - param_substs: instance.substs + param_substs: instance.substs, + const_context, }; visitor.visit_mir(&mir);