Auto merge of #47181 - michaelwoerister:var-len-def-index, r=eddyb

Use DefIndex encoding that works better with on-disk variable length integer representations.

Use the least instead of the most significant bit for representing the address space.

r? @EddyB

src/librustc/hir/def_id.rs

@@ -71,27 +71,23 @@ impl serialize::UseSpecializedDecodable for CrateNum {}
 /// particular definition. It should really be considered an interned
 /// shorthand for a particular DefPath.
 ///
-/// At the moment we are allocating the numerical values of DefIndexes into two
-/// ranges: the "low" range (starting at zero) and the "high" range (starting at
-/// DEF_INDEX_HI_START). This allows us to allocate the DefIndexes of all
-/// item-likes (Items, TraitItems, and ImplItems) into one of these ranges and
+/// At the moment we are allocating the numerical values of DefIndexes from two
+/// address spaces: DefIndexAddressSpace::Low and DefIndexAddressSpace::High.
+/// This allows us to allocate the DefIndexes of all item-likes
+/// (Items, TraitItems, and ImplItems) into one of these spaces and
 /// consequently use a simple array for lookup tables keyed by DefIndex and
 /// known to be densely populated. This is especially important for the HIR map.
 ///
 /// Since the DefIndex is mostly treated as an opaque ID, you probably
-/// don't have to care about these ranges.
-newtype_index!(DefIndex
-    {
-        ENCODABLE = custom
-        DEBUG_FORMAT = custom,
+/// don't have to care about these address spaces.
 
-        /// The start of the "high" range of DefIndexes.
-        const DEF_INDEX_HI_START = 1 << 31,
+#[derive(Clone, Eq, Ord, PartialOrd, PartialEq, Hash, Copy)]
+pub struct DefIndex(u32);
+
+/// The crate root is always assigned index 0 by the AST Map code,
+/// thanks to `NodeCollector::new`.
+pub const CRATE_DEF_INDEX: DefIndex = DefIndex(0);
 
-        /// The crate root is always assigned index 0 by the AST Map code,
-        /// thanks to `NodeCollector::new`.
-        const CRATE_DEF_INDEX = 0,
-    });
 
 impl fmt::Debug for DefIndex {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -104,40 +100,50 @@ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 
 impl DefIndex {
     #[inline]
-    pub fn from_u32(x: u32) -> DefIndex {
-        DefIndex(x)
+    pub fn address_space(&self) -> DefIndexAddressSpace {
+        match self.0 & 1 {
+            0 => DefIndexAddressSpace::Low,
+            1 => DefIndexAddressSpace::High,
+            _ => unreachable!()
+        }
     }
 
+    /// Converts this DefIndex into a zero-based array index.
+    /// This index is the offset within the given DefIndexAddressSpace.
     #[inline]
-    pub fn as_usize(&self) -> usize {
-        self.0 as usize
+    pub fn as_array_index(&self) -> usize {
+        (self.0 >> 1) as usize
     }
 
     #[inline]
-    pub fn as_u32(&self) -> u32 {
-        self.0
+    pub fn from_array_index(i: usize, address_space: DefIndexAddressSpace) -> DefIndex {
+        DefIndex::from_raw_u32(((i << 1) | (address_space as usize)) as u32)
     }
 
-    #[inline]
-    pub fn address_space(&self) -> DefIndexAddressSpace {
-        if self.0 < DEF_INDEX_HI_START.0 {
-            DefIndexAddressSpace::Low
-        } else {
-            DefIndexAddressSpace::High
+    // Proc macros from a proc-macro crate have a kind of virtual DefIndex. This
+    // function maps the index of the macro within the crate (which is also the
+    // index of the macro in the CrateMetadata::proc_macros array) to the
+    // corresponding DefIndex.
+    pub fn from_proc_macro_index(proc_macro_index: usize) -> DefIndex {
+        let def_index = DefIndex::from_array_index(proc_macro_index,
+                                                   DefIndexAddressSpace::High);
+        assert!(def_index != CRATE_DEF_INDEX);
+        def_index
     }
+
+    // This function is the reverse of from_proc_macro_index() above.
+    pub fn to_proc_macro_index(self: DefIndex) -> usize {
+        self.as_array_index()
     }
 
-    /// Converts this DefIndex into a zero-based array index.
-    /// This index is the offset within the given "range" of the DefIndex,
-    /// that is, if the DefIndex is part of the "high" range, the resulting
-    /// index will be (DefIndex - DEF_INDEX_HI_START).
-    #[inline]
-    pub fn as_array_index(&self) -> usize {
-        (self.0 & !DEF_INDEX_HI_START.0) as usize
+    // Don't use this if you don't know about the DefIndex encoding.
+    pub fn from_raw_u32(x: u32) -> DefIndex {
+        DefIndex(x)
     }
 
-    pub fn from_array_index(i: usize, address_space: DefIndexAddressSpace) -> DefIndex {
-        DefIndex::new(address_space.start() + i)
+    // Don't use this if you don't know about the DefIndex encoding.
+    pub fn as_raw_u32(&self) -> u32 {
+        self.0
     }
 }
 
@@ -155,11 +161,6 @@ impl DefIndexAddressSpace {
     pub fn index(&self) -> usize {
         *self as usize
     }
-
-    #[inline]
-    pub fn start(&self) -> usize {
-        self.index() * DEF_INDEX_HI_START.as_usize()
-    }
 }
 
 /// A DefId identifies a particular *definition*, by combining a crate

src/librustc/hir/map/definitions.rs

@@ -19,7 +19,7 @@
                   CRATE_DEF_INDEX};
 use ich::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::indexed_vec::{IndexVec, Idx};
+use rustc_data_structures::indexed_vec::{IndexVec};
 use rustc_data_structures::stable_hasher::StableHasher;
 use serialize::{Encodable, Decodable, Encoder, Decoder};
 use session::CrateDisambiguator;
@@ -61,7 +61,7 @@ fn allocate(&mut self,
                 -> DefIndex {
         let index = {
             let index_to_key = &mut self.index_to_key[address_space.index()];
-            let index = DefIndex::new(index_to_key.len() + address_space.start());
+            let index = DefIndex::from_array_index(index_to_key.len(), address_space);
             debug!("DefPathTable::insert() - {:?} <-> {:?}", key, index);
             index_to_key.push(key);
             index
@@ -89,8 +89,7 @@ pub fn def_path_hash(&self, index: DefIndex) -> DefPathHash {
     pub fn add_def_path_hashes_to(&self,
                                   cnum: CrateNum,
                                   out: &mut FxHashMap<DefPathHash, DefId>) {
-        for address_space in &[DefIndexAddressSpace::Low, DefIndexAddressSpace::High] {
-            let start_index = address_space.start();
+        for &address_space in &[DefIndexAddressSpace::Low, DefIndexAddressSpace::High] {
             out.extend(
                 (&self.def_path_hashes[address_space.index()])
                     .iter()
@@ -98,7 +97,7 @@ pub fn add_def_path_hashes_to(&self,
                     .map(|(index, &hash)| {
                         let def_id = DefId {
                             krate: cnum,
-                            index: DefIndex::new(index + start_index),
+                            index: DefIndex::from_array_index(index, address_space),
                         };
                         (hash, def_id)
                     })

src/librustc/infer/lexical_region_resolve/graphviz.rs

@@ -19,7 +19,6 @@
 use graphviz as dot;
 
 use hir::def_id::DefIndex;
-use rustc_data_structures::indexed_vec::Idx;
 use ty;
 use middle::free_region::RegionRelations;
 use middle::region;
@@ -68,7 +67,7 @@ pub fn maybe_print_constraints_for<'a, 'gcx, 'tcx>(
     }
 
     let requested_node = env::var("RUST_REGION_GRAPH_NODE")
-        .ok().and_then(|s| s.parse().map(DefIndex::new).ok());
+        .ok().and_then(|s| s.parse().map(DefIndex::from_raw_u32).ok());
 
     if requested_node.is_some() && requested_node != Some(context.index) {
         return;
@@ -102,7 +101,7 @@ pub fn maybe_print_constraints_for<'a, 'gcx, 'tcx>(
             let mut new_str = String::new();
             for c in output_template.chars() {
                 if c == '%' {
-                    new_str.push_str(&context.index.as_usize().to_string());
+                    new_str.push_str(&context.index.as_raw_u32().to_string());
                 } else {
                     new_str.push(c);
                 }

src/librustc/session/mod.rs

@@ -562,14 +562,14 @@ pub fn generate_plugin_registrar_symbol(&self, disambiguator: CrateDisambiguator
                                             index: DefIndex)
                                             -> String {
         format!("__rustc_plugin_registrar__{}_{}", disambiguator.to_fingerprint().to_hex(),
-                                                   index.as_usize())
+                                                   index.to_proc_macro_index())
     }
 
     pub fn generate_derive_registrar_symbol(&self, disambiguator: CrateDisambiguator,
                                             index: DefIndex)
                                             -> String {
         format!("__rustc_derive_registrar__{}_{}", disambiguator.to_fingerprint().to_hex(),
-                                                   index.as_usize())
+                                                   index.to_proc_macro_index())
     }
 
     pub fn sysroot<'a>(&'a self) -> &'a Path {

src/librustc_metadata/cstore_impl.rs

@@ -463,7 +463,7 @@ fn item_children_untracked(&self, def_id: DefId, sess: &Session) -> Vec<def::Exp
     fn load_macro_untracked(&self, id: DefId, sess: &Session) -> LoadedMacro {
         let data = self.get_crate_data(id.krate);
         if let Some(ref proc_macros) = data.proc_macros {
-            return LoadedMacro::ProcMacro(proc_macros[id.index.as_usize() - 1].1.clone());
+            return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
         } else if data.name == "proc_macro" &&
                   self.get_crate_data(id.krate).item_name(id.index) == "quote" {
             let ext = SyntaxExtension::ProcMacro(Box::new(::proc_macro::__internal::Quoter));

src/librustc_metadata/decoder.rs

@@ -18,7 +18,8 @@
 use rustc::middle::cstore::{LinkagePreference, ExternConstBody,
                             ExternBodyNestedBodies};
 use rustc::hir::def::{self, Def, CtorKind};
-use rustc::hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE};
+use rustc::hir::def_id::{CrateNum, DefId, DefIndex,
+                         CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc::ich::Fingerprint;
 use rustc::middle::lang_items;
 use rustc::mir;
@@ -36,7 +37,6 @@
 use std::u32;
 
 use rustc_serialize::{Decodable, Decoder, SpecializedDecoder, opaque};
-use rustc_data_structures::indexed_vec::Idx;
 use syntax::attr;
 use syntax::ast::{self, Ident};
 use syntax::codemap;
@@ -264,7 +264,7 @@ fn specialized_decode(&mut self) -> Result<DefId, Self::Error> {
 impl<'a, 'tcx> SpecializedDecoder<DefIndex> for DecodeContext<'a, 'tcx> {
     #[inline]
     fn specialized_decode(&mut self) -> Result<DefIndex, Self::Error> {
-        Ok(DefIndex::from_u32(self.read_u32()?))
+        Ok(DefIndex::from_raw_u32(self.read_u32()?))
     }
 }
 
@@ -453,7 +453,7 @@ pub fn get_def(&self, index: DefIndex) -> Option<Def> {
         if !self.is_proc_macro(index) {
             self.entry(index).kind.to_def(self.local_def_id(index))
         } else {
-            let kind = self.proc_macros.as_ref().unwrap()[index.as_usize() - 1].1.kind();
+            let kind = self.proc_macros.as_ref().unwrap()[index.to_proc_macro_index()].1.kind();
             Some(Def::Macro(self.local_def_id(index), kind))
         }
     }
@@ -634,7 +634,7 @@ pub fn each_child_of_item<F>(&self, id: DefIndex, mut callback: F, sess: &Sessio
                     let def = Def::Macro(
                         DefId {
                             krate: self.cnum,
-                            index: DefIndex::new(id + 1)
+                            index: DefIndex::from_proc_macro_index(id),
                         },
                         ext.kind()
                     );

src/librustc_metadata/encoder.rs

@@ -140,7 +140,7 @@ fn specialized_encode(&mut self, def_id: &DefId) -> Result<(), Self::Error> {
 impl<'a, 'tcx> SpecializedEncoder<DefIndex> for EncodeContext<'a, 'tcx> {
     #[inline]
     fn specialized_encode(&mut self, def_index: &DefIndex) -> Result<(), Self::Error> {
-        self.emit_u32(def_index.as_u32())
+        self.emit_u32(def_index.as_raw_u32())
     }
 }
 

src/librustc_metadata/index.rs

@@ -74,10 +74,9 @@ impl<'tcx> LazySeq<Index> {
     #[inline(never)]
     pub fn lookup(&self, bytes: &[u8], def_index: DefIndex) -> Option<Lazy<Entry<'tcx>>> {
         let words = &bytes_to_words(&bytes[self.position..])[..self.len];
-        let index = def_index.as_usize();
 
         debug!("Index::lookup: index={:?} words.len={:?}",
-               index,
+               def_index,
                words.len());
 
         let positions = match def_index.address_space() {

src/librustc_resolve/macros.rs

@@ -13,8 +13,8 @@
 use Namespace::{self, MacroNS};
 use build_reduced_graph::BuildReducedGraphVisitor;
 use resolve_imports::ImportResolver;
-use rustc_data_structures::indexed_vec::Idx;
-use rustc::hir::def_id::{DefId, BUILTIN_MACROS_CRATE, CRATE_DEF_INDEX, DefIndex};
+use rustc::hir::def_id::{DefId, BUILTIN_MACROS_CRATE, CRATE_DEF_INDEX, DefIndex,
+                         DefIndexAddressSpace};
 use rustc::hir::def::{Def, Export};
 use rustc::hir::map::{self, DefCollector};
 use rustc::{ty, lint};
@@ -188,7 +188,8 @@ fn visit_expansion(&mut self, mark: Mark, expansion: &Expansion, derives: &[Mark
     fn add_builtin(&mut self, ident: ast::Ident, ext: Rc<SyntaxExtension>) {
         let def_id = DefId {
             krate: BUILTIN_MACROS_CRATE,
-            index: DefIndex::new(self.macro_map.len()),
+            index: DefIndex::from_array_index(self.macro_map.len(),
+                                              DefIndexAddressSpace::Low),
         };
         let kind = ext.kind();
         self.macro_map.insert(def_id, ext);

src/librustc_save_analysis/lib.rs

@@ -1123,7 +1123,7 @@ fn generated_code(span: Span) -> bool {
 fn id_from_def_id(id: DefId) -> rls_data::Id {
     rls_data::Id {
         krate: id.krate.as_u32(),
-        index: id.index.as_u32(),
+        index: id.index.as_raw_u32(),
     }
 }