From 884ba4f11f4cac31b9fda402cbd925af4bc286d4 Mon Sep 17 00:00:00 2001 From: Dylan MacKenzie Date: Sun, 12 Apr 2020 10:30:07 -0700 Subject: [PATCH] Use internal mutability for predecessor cache --- src/librustc_middle/mir/cache.rs | 276 ------------------------ src/librustc_middle/mir/mod.rs | 77 ++++++- src/librustc_middle/mir/predecessors.rs | 67 ++++++ 3 files changed, 133 insertions(+), 287 deletions(-) delete mode 100644 src/librustc_middle/mir/cache.rs create mode 100644 src/librustc_middle/mir/predecessors.rs diff --git a/src/librustc_middle/mir/cache.rs b/src/librustc_middle/mir/cache.rs deleted file mode 100644 index 12822cea418..00000000000 --- a/src/librustc_middle/mir/cache.rs +++ /dev/null @@ -1,276 +0,0 @@ -use crate::ich::StableHashingContext; -use crate::mir::{BasicBlock, BasicBlockData, Body, LocalDecls, Location, Successors}; -use rustc_data_structures::graph::dominators::{dominators, Dominators}; -use rustc_data_structures::graph::{self, GraphPredecessors, GraphSuccessors}; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_index::vec::IndexVec; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; -use smallvec::SmallVec; -use std::iter; -use std::ops::{Deref, DerefMut, Index, IndexMut}; -use std::vec::IntoIter; - -#[derive(Clone, Debug)] -pub struct Cache { - // Typically 95%+ of the inner vectors have 4 or fewer elements. - predecessors: Option>>, -} - -impl rustc_serialize::Encodable for Cache { - fn encode(&self, s: &mut S) -> Result<(), S::Error> { - Encodable::encode(&(), s) - } -} - -impl rustc_serialize::Decodable for Cache { - fn decode(d: &mut D) -> Result { - Decodable::decode(d).map(|_v: ()| Self::new()) - } -} - -impl<'a> HashStable> for Cache { - fn hash_stable(&self, _: &mut StableHashingContext<'a>, _: &mut StableHasher) { - // Do nothing. - } -} - -impl Cache { - pub fn new() -> Self { - Self { predecessors: None } - } - - pub fn invalidate_predecessors(&mut self) { - // FIXME: consider being more fine-grained - self.predecessors = None; - } - - pub fn ensure_predecessors(&mut self, body: &Body<'_>) { - if self.predecessors.is_none() { - let mut result = IndexVec::from_elem(smallvec![], body.basic_blocks()); - for (bb, data) in body.basic_blocks().iter_enumerated() { - if let Some(ref term) = data.terminator { - for &tgt in term.successors() { - result[tgt].push(bb); - } - } - } - - self.predecessors = Some(result) - } - } - - /// This will recompute the predecessors cache if it is not available - fn predecessors( - &mut self, - body: &Body<'_>, - ) -> &IndexVec> { - self.ensure_predecessors(body); - self.predecessors.as_ref().unwrap() - } - - fn unwrap_predecessors_for(&self, bb: BasicBlock) -> &[BasicBlock] { - &self.predecessors.as_ref().unwrap()[bb] - } - - fn unwrap_predecessor_locations<'a>( - &'a self, - loc: Location, - body: &'a Body<'a>, - ) -> impl Iterator + 'a { - let if_zero_locations = if loc.statement_index == 0 { - let predecessor_blocks = self.unwrap_predecessors_for(loc.block); - let num_predecessor_blocks = predecessor_blocks.len(); - Some( - (0..num_predecessor_blocks) - .map(move |i| predecessor_blocks[i]) - .map(move |bb| body.terminator_loc(bb)), - ) - } else { - None - }; - - let if_not_zero_locations = if loc.statement_index == 0 { - None - } else { - Some(Location { block: loc.block, statement_index: loc.statement_index - 1 }) - }; - - if_zero_locations.into_iter().flatten().chain(if_not_zero_locations) - } - - pub fn basic_blocks_mut<'a, 'tcx>( - &mut self, - body: &'a mut Body<'tcx>, - ) -> &'a mut IndexVec> { - debug!("bbm: Clearing predecessors cache for body at: {:?}", body.span.data()); - self.invalidate_predecessors(); - &mut body.basic_blocks - } - - pub fn basic_blocks_and_local_decls_mut<'a, 'tcx>( - &mut self, - body: &'a mut Body<'tcx>, - ) -> (&'a mut IndexVec>, &'a mut LocalDecls<'tcx>) { - debug!("bbaldm: Clearing predecessors cache for body at: {:?}", body.span.data()); - self.invalidate_predecessors(); - (&mut body.basic_blocks, &mut body.local_decls) - } -} - -#[derive(Clone, Debug, HashStable, RustcEncodable, RustcDecodable, TypeFoldable)] -pub struct BodyAndCache<'tcx> { - body: Body<'tcx>, - cache: Cache, -} - -impl BodyAndCache<'tcx> { - pub fn new(body: Body<'tcx>) -> Self { - Self { body, cache: Cache::new() } - } -} - -#[macro_export] -macro_rules! read_only { - ($body:expr) => {{ - $body.ensure_predecessors(); - $body.unwrap_read_only() - }}; -} - -impl BodyAndCache<'tcx> { - pub fn ensure_predecessors(&mut self) { - self.cache.ensure_predecessors(&self.body); - } - - pub fn predecessors(&mut self) -> &IndexVec> { - self.cache.predecessors(&self.body) - } - - pub fn unwrap_read_only(&self) -> ReadOnlyBodyAndCache<'_, 'tcx> { - ReadOnlyBodyAndCache::new(&self.body, &self.cache) - } - - pub fn basic_blocks_mut(&mut self) -> &mut IndexVec> { - self.cache.basic_blocks_mut(&mut self.body) - } - - pub fn basic_blocks_and_local_decls_mut( - &mut self, - ) -> (&mut IndexVec>, &mut LocalDecls<'tcx>) { - self.cache.basic_blocks_and_local_decls_mut(&mut self.body) - } -} - -impl<'tcx> Index for BodyAndCache<'tcx> { - type Output = BasicBlockData<'tcx>; - - fn index(&self, index: BasicBlock) -> &BasicBlockData<'tcx> { - &self.body[index] - } -} - -impl<'tcx> IndexMut for BodyAndCache<'tcx> { - fn index_mut(&mut self, index: BasicBlock) -> &mut Self::Output { - &mut self.basic_blocks_mut()[index] - } -} - -impl<'tcx> Deref for BodyAndCache<'tcx> { - type Target = Body<'tcx>; - - fn deref(&self) -> &Self::Target { - &self.body - } -} - -impl<'tcx> DerefMut for BodyAndCache<'tcx> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.body - } -} - -#[derive(Copy, Clone, Debug)] -pub struct ReadOnlyBodyAndCache<'a, 'tcx> { - body: &'a Body<'tcx>, - cache: &'a Cache, -} - -impl ReadOnlyBodyAndCache<'a, 'tcx> { - fn new(body: &'a Body<'tcx>, cache: &'a Cache) -> Self { - assert!( - cache.predecessors.is_some(), - "Cannot construct ReadOnlyBodyAndCache without computed predecessors" - ); - Self { body, cache } - } - - pub fn predecessors(&self) -> &IndexVec> { - self.cache.predecessors.as_ref().unwrap() - } - - pub fn predecessors_for(&self, bb: BasicBlock) -> &[BasicBlock] { - self.cache.unwrap_predecessors_for(bb) - } - - pub fn predecessor_locations(&self, loc: Location) -> impl Iterator + '_ { - self.cache.unwrap_predecessor_locations(loc, self.body) - } - - pub fn basic_blocks(&self) -> &IndexVec> { - &self.body.basic_blocks - } - - pub fn dominators(&self) -> Dominators { - dominators(self) - } -} - -impl graph::DirectedGraph for ReadOnlyBodyAndCache<'a, 'tcx> { - type Node = BasicBlock; -} - -impl graph::GraphPredecessors<'graph> for ReadOnlyBodyAndCache<'a, 'tcx> { - type Item = BasicBlock; - type Iter = IntoIter; -} - -impl graph::WithPredecessors for ReadOnlyBodyAndCache<'a, 'tcx> { - fn predecessors(&self, node: Self::Node) -> >::Iter { - self.cache.unwrap_predecessors_for(node).to_vec().into_iter() - } -} - -impl graph::WithNumNodes for ReadOnlyBodyAndCache<'a, 'tcx> { - fn num_nodes(&self) -> usize { - self.body.num_nodes() - } -} - -impl graph::WithStartNode for ReadOnlyBodyAndCache<'a, 'tcx> { - fn start_node(&self) -> Self::Node { - self.body.start_node() - } -} - -impl graph::WithSuccessors for ReadOnlyBodyAndCache<'a, 'tcx> { - fn successors(&self, node: Self::Node) -> >::Iter { - self.body.successors(node) - } -} - -impl<'a, 'b, 'tcx> graph::GraphSuccessors<'b> for ReadOnlyBodyAndCache<'a, 'tcx> { - type Item = BasicBlock; - type Iter = iter::Cloned>; -} - -impl Deref for ReadOnlyBodyAndCache<'a, 'tcx> { - type Target = &'a Body<'tcx>; - - fn deref(&self) -> &Self::Target { - &self.body - } -} - -CloneTypeFoldableAndLiftImpls! { - Cache, -} diff --git a/src/librustc_middle/mir/mod.rs b/src/librustc_middle/mir/mod.rs index 70ae2fd8ac9..3295e48b7b4 100644 --- a/src/librustc_middle/mir/mod.rs +++ b/src/librustc_middle/mir/mod.rs @@ -21,27 +21,28 @@ pub use rustc_ast::ast::Mutability; use rustc_ast::ast::Name; use rustc_data_structures::fx::FxHashSet; -use rustc_data_structures::graph::dominators::Dominators; +use rustc_data_structures::graph::dominators::{dominators, Dominators}; use rustc_data_structures::graph::{self, GraphSuccessors}; +use rustc_data_structures::sync::MappedLockGuard; use rustc_index::bit_set::BitMatrix; use rustc_index::vec::{Idx, IndexVec}; use rustc_macros::HashStable; use rustc_serialize::{Decodable, Encodable}; use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; +use smallvec::SmallVec; use std::borrow::Cow; use std::fmt::{self, Debug, Display, Formatter, Write}; -use std::ops::Index; +use std::ops::{Index, IndexMut}; use std::slice; use std::{iter, mem, option}; -pub use self::cache::{BodyAndCache, ReadOnlyBodyAndCache}; +use self::predecessors::{PredecessorCache, Predecessors}; pub use self::query::*; -pub use crate::read_only; -mod cache; pub mod interpret; pub mod mono; +mod predecessors; mod query; pub mod tcx; pub mod traversal; @@ -108,7 +109,7 @@ pub struct Body<'tcx> { pub yield_ty: Option>, /// Generator drop glue. - pub generator_drop: Option>>, + pub generator_drop: Option>>, /// The layout of a generator. Produced by the state transformation. pub generator_layout: Option>, @@ -164,6 +165,8 @@ pub struct Body<'tcx> { /// implementation without the flag hid this situation silently. /// FIXME(oli-obk): rewrite the promoted during promotion to eliminate the cell components. pub ignore_interior_mut_in_const_validation: bool, + + pub predecessor_cache: PredecessorCache, } impl<'tcx> Body<'tcx> { @@ -202,6 +205,7 @@ pub fn new( span, ignore_interior_mut_in_const_validation: false, control_flow_destroyed, + predecessor_cache: PredecessorCache::new(), } } @@ -227,6 +231,7 @@ pub fn new_cfg_only(basic_blocks: IndexVec>) -> generator_kind: None, var_debug_info: Vec::new(), ignore_interior_mut_in_const_validation: false, + predecessor_cache: PredecessorCache::new(), } } @@ -235,6 +240,25 @@ pub fn basic_blocks(&self) -> &IndexVec> { &self.basic_blocks } + #[inline] + pub fn basic_blocks_mut(&mut self) -> &mut IndexVec> { + // Because the user could mutate basic block terminators via this reference, we need to + // invalidate the predecessor cache. + // + // FIXME: Use a finer-grained API for this, so only transformations that alter terminators + // invalidate the predecessor cache. + self.predecessor_cache.invalidate(); + &mut self.basic_blocks + } + + #[inline] + pub fn basic_blocks_and_local_decls_mut( + &mut self, + ) -> (&mut IndexVec>, &mut LocalDecls<'tcx>) { + self.predecessor_cache.invalidate(); + (&mut self.basic_blocks, &mut self.local_decls) + } + /// Returns `true` if a cycle exists in the control-flow graph that is reachable from the /// `START_BLOCK`. pub fn is_cfg_cyclic(&self) -> bool { @@ -365,6 +389,23 @@ pub fn return_ty(&self) -> Ty<'tcx> { pub fn terminator_loc(&self, bb: BasicBlock) -> Location { Location { block: bb, statement_index: self[bb].statements.len() } } + + pub fn predecessors_for( + &self, + bb: BasicBlock, + ) -> impl std::ops::Deref> + '_ { + let predecessors = self.predecessor_cache.compute(&self.basic_blocks); + MappedLockGuard::map(predecessors, |preds| &mut preds[bb]) + } + + pub fn predecessors(&self) -> impl std::ops::Deref + '_ { + self.predecessor_cache.compute(&self.basic_blocks) + } + + #[inline] + pub fn dominators(&self) -> Dominators { + dominators(self) + } } #[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, HashStable)] @@ -387,6 +428,13 @@ fn index(&self, index: BasicBlock) -> &BasicBlockData<'tcx> { } } +impl<'tcx> IndexMut for Body<'tcx> { + #[inline] + fn index_mut(&mut self, index: BasicBlock) -> &mut BasicBlockData<'tcx> { + &mut self.basic_blocks_mut()[index] + } +} + #[derive(Copy, Clone, Debug, HashStable, TypeFoldable)] pub enum ClearCrossCrate { Clear, @@ -2613,6 +2661,17 @@ impl<'a, 'b> graph::GraphSuccessors<'b> for Body<'a> { type Iter = iter::Cloned>; } +impl graph::GraphPredecessors<'graph> for Body<'tcx> { + type Item = BasicBlock; + type Iter = smallvec::IntoIter<[BasicBlock; 4]>; +} + +impl graph::WithPredecessors for Body<'tcx> { + fn predecessors(&self, node: Self::Node) -> >::Iter { + self.predecessors_for(node).clone().into_iter() + } +} + /// `Location` represents the position of the start of the statement; or, if /// `statement_index` equals the number of statements, then the start of the /// terminator. @@ -2642,11 +2701,7 @@ pub fn successor_within_block(&self) -> Location { } /// Returns `true` if `other` is earlier in the control flow graph than `self`. - pub fn is_predecessor_of<'tcx>( - &self, - other: Location, - body: ReadOnlyBodyAndCache<'_, 'tcx>, - ) -> bool { + pub fn is_predecessor_of<'tcx>(&self, other: Location, body: &Body<'tcx>) -> bool { // If we are in the same block as the other location and are an earlier statement // then we are a predecessor of `other`. if self.block == other.block && self.statement_index < other.statement_index { diff --git a/src/librustc_middle/mir/predecessors.rs b/src/librustc_middle/mir/predecessors.rs new file mode 100644 index 00000000000..90cf1e3ce4e --- /dev/null +++ b/src/librustc_middle/mir/predecessors.rs @@ -0,0 +1,67 @@ +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::sync::{Lock, LockGuard, MappedLockGuard}; +use rustc_index::vec::IndexVec; +use rustc_serialize as serialize; +use smallvec::SmallVec; + +use crate::mir::{BasicBlock, BasicBlockData}; + +// Typically 95%+ of basic blocks have 4 or fewer predecessors. +pub type Predecessors = IndexVec>; + +#[derive(Clone, Debug)] +pub struct PredecessorCache { + cache: Lock>, +} + +impl PredecessorCache { + pub fn new() -> Self { + PredecessorCache { cache: Lock::new(None) } + } + + pub fn invalidate(&mut self) { + *self.cache.get_mut() = None; + } + + pub fn compute( + &self, + basic_blocks: &IndexVec>, + ) -> MappedLockGuard<'_, Predecessors> { + LockGuard::map(self.cache.lock(), |cache| { + cache.get_or_insert_with(|| { + let mut preds = IndexVec::from_elem(SmallVec::new(), basic_blocks); + for (bb, data) in basic_blocks.iter_enumerated() { + if let Some(term) = &data.terminator { + for &succ in term.successors() { + preds[succ].push(bb); + } + } + } + + preds + }) + }) + } +} + +impl serialize::Encodable for PredecessorCache { + fn encode(&self, s: &mut S) -> Result<(), S::Error> { + serialize::Encodable::encode(&(), s) + } +} + +impl serialize::Decodable for PredecessorCache { + fn decode(d: &mut D) -> Result { + serialize::Decodable::decode(d).map(|_v: ()| Self::new()) + } +} + +impl HashStable for PredecessorCache { + fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) { + // do nothing + } +} + +CloneTypeFoldableAndLiftImpls! { + PredecessorCache, +} -- GitLab