Store metadata separately in rlib files

Right now whenever an rlib file is linked against, all of the metadata from the
rlib is pulled in to the final staticlib or binary. The reason for this is that
the metadata is currently stored in a section of the object file. Note that this
is intentional for dynamic libraries in order to distribute metadata bundled
with static libraries.

This commit alters the situation for rlib libraries to instead store the
metadata in a separate file in the archive. In doing so, when the archive is
passed to the linker, none of the metadata will get pulled into the result
executable. Furthermore, the metadata file is skipped when assembling rlibs into
an archive.

The snag in this implementation comes with multiple output formats. When
generating a dylib, the metadata needs to be in the object file, but when
generating an rlib this needs to be separate. In order to accomplish this, the
metadata variable is inserted into an entirely separate LLVM Module which is
then codegen'd into a different location (foo.metadata.o). This is then linked
into dynamic libraries and silently ignored for rlib files.

While changing how metadata is inserted into archives, I have also stopped
compressing metadata when inserted into rlib files. We have wanted to stop
compressing metadata, but the sections it creates in object file sections are
apparently too large. Thankfully if it's just an arbitrary file it doesn't
matter how large it is.

I have seen massive reductions in executable sizes, as well as staticlib output
sizes (to confirm that this is all working).

src/librustc/back/archive.rs

@@ -20,6 +20,8 @@
 use extra::tempfile::TempDir;
 use syntax::abi;
 
+pub static METADATA_FILENAME: &'static str = "metadata";
+
 pub struct Archive {
     priv sess: Session,
     priv dst: Path,
@@ -81,17 +83,22 @@ pub fn read(&self, file: &str) -> ~[u8] {
     /// search in the relevant locations for a library named `name`.
     pub fn add_native_library(&mut self, name: &str) {
         let location = self.find_library(name);
-        self.add_archive(&location, name);
+        self.add_archive(&location, name, []);
     }
 
     /// Adds all of the contents of the rlib at the specified path to this
     /// archive.
     pub fn add_rlib(&mut self, rlib: &Path) {
         let name = rlib.filename_str().unwrap().split('-').next().unwrap();
-        self.add_archive(rlib, name);
+        self.add_archive(rlib, name, [METADATA_FILENAME]);
+    }
+
+    /// Adds an arbitrary file to this archive
+    pub fn add_file(&mut self, file: &Path) {
+        run_ar(self.sess, "r", None, [&self.dst, file]);
     }
 
-    fn add_archive(&mut self, archive: &Path, name: &str) {
+    fn add_archive(&mut self, archive: &Path, name: &str, skip: &[&str]) {
         let loc = TempDir::new("rsar").unwrap();
 
         // First, extract the contents of the archive to a temporary directory
@@ -106,6 +113,7 @@ fn add_archive(&mut self, archive: &Path, name: &str) {
         let mut inputs = ~[];
         for file in files.iter() {
             let filename = file.filename_str().unwrap();
+            if skip.iter().any(|s| *s == filename) { continue }
             let filename = format!("r-{}-{}", name, filename);
             let new_filename = file.with_filename(filename);
             fs::rename(file, &new_filename);

src/librustc/back/link.rs

@@ -9,8 +9,9 @@
 // except according to those terms.
 
 
-use back::archive::Archive;
+use back::archive::{Archive, METADATA_FILENAME};
 use back::rpath;
+use driver::driver::CrateTranslation;
 use driver::session::Session;
 use driver::session;
 use lib::llvm::llvm;
@@ -88,10 +89,11 @@ pub mod write {
     use back::link::{output_type_assembly, output_type_bitcode};
     use back::link::{output_type_exe, output_type_llvm_assembly};
     use back::link::{output_type_object};
+    use driver::driver::CrateTranslation;
     use driver::session::Session;
     use driver::session;
     use lib::llvm::llvm;
-    use lib::llvm::{ModuleRef, ContextRef};
+    use lib::llvm::ModuleRef;
     use lib;
 
     use std::c_str::ToCStr;
@@ -101,10 +103,11 @@ pub mod write {
     use std::str;
 
     pub fn run_passes(sess: Session,
-                      llcx: ContextRef,
-                      llmod: ModuleRef,
+                      trans: &CrateTranslation,
                       output_type: output_type,
                       output: &Path) {
+        let llmod = trans.module;
+        let llcx = trans.context;
         unsafe {
             llvm::LLVMInitializePasses();
 
@@ -204,12 +207,23 @@ pub fn run_passes(sess: Session,
                 })
             }
 
-            // Create a codegen-specific pass manager to emit the actual
-            // assembly or object files. This may not end up getting used,
-            // but we make it anyway for good measure.
-            let cpm = llvm::LLVMCreatePassManager();
-            llvm::LLVMRustAddAnalysisPasses(tm, cpm, llmod);
-            llvm::LLVMRustAddLibraryInfo(cpm, llmod);
+            // A codegen-specific pass manager is used to generate object
+            // files for an LLVM module.
+            //
+            // Apparently each of these pass managers is a one-shot kind of
+            // thing, so we create a new one for each type of output. The
+            // pass manager passed to the closure should be ensured to not
+            // escape the closure itself, and the manager should only be
+            // used once.
+            fn with_codegen(tm: TargetMachineRef, llmod: ModuleRef,
+                            f: |PassManagerRef|) {
+                let cpm = llvm::LLVMCreatePassManager();
+                llvm::LLVMRustAddAnalysisPasses(tm, cpm, llmod);
+                llvm::LLVMRustAddLibraryInfo(cpm, llmod);
+                f(cpm);
+                llvm::LLVMDisposePassManager(cpm);
+
+            }
 
             match output_type {
                 output_type_none => {}
@@ -220,20 +234,47 @@ pub fn run_passes(sess: Session,
                 }
                 output_type_llvm_assembly => {
                     output.with_c_str(|output| {
-                        llvm::LLVMRustPrintModule(cpm, llmod, output)
+                        with_codegen(tm, llmod, |cpm| {
+                            llvm::LLVMRustPrintModule(cpm, llmod, output);
+                        })
                     })
                 }
                 output_type_assembly => {
-                    WriteOutputFile(sess, tm, cpm, llmod, output, lib::llvm::AssemblyFile);
+                    with_codegen(tm, llmod, |cpm| {
+                        WriteOutputFile(sess, tm, cpm, llmod, output,
+                                        lib::llvm::AssemblyFile);
+                    });
+
+                    // windows will invoke this function with an assembly output
+                    // type when it's actually generating an object file. This
+                    // is because g++ is used to compile the assembly instead of
+                    // having LLVM directly output an object file. Regardless,
+                    // in this case, we're going to possibly need a metadata
+                    // file.
+                    if sess.opts.output_type != output_type_assembly {
+                        with_codegen(tm, trans.metadata_module, |cpm| {
+                            let out = output.with_extension("metadata.o");
+                            WriteOutputFile(sess, tm, cpm,
+                                            trans.metadata_module, &out,
+                                            lib::llvm::ObjectFile);
+                        })
+                    }
                 }
                 output_type_exe | output_type_object => {
-                    WriteOutputFile(sess, tm, cpm, llmod, output, lib::llvm::ObjectFile);
+                    with_codegen(tm, llmod, |cpm| {
+                        WriteOutputFile(sess, tm, cpm, llmod, output,
+                                        lib::llvm::ObjectFile);
+                    });
+                    with_codegen(tm, trans.metadata_module, |cpm| {
+                        WriteOutputFile(sess, tm, cpm, trans.metadata_module,
+                                        &output.with_extension("metadata.o"),
+                                        lib::llvm::ObjectFile);
+                    })
                 }
             }
 
-            llvm::LLVMDisposePassManager(cpm);
-
             llvm::LLVMRustDisposeTargetMachine(tm);
+            llvm::LLVMDisposeModule(trans.metadata_module);
             llvm::LLVMDisposeModule(llmod);
             llvm::LLVMContextDispose(llcx);
             if sess.time_llvm_passes() { llvm::LLVMRustPrintPassTimings(); }
@@ -782,10 +823,9 @@ pub fn get_cc_prog(sess: Session) -> ~str {
 /// Perform the linkage portion of the compilation phase. This will generate all
 /// of the requested outputs for this compilation session.
 pub fn link_binary(sess: Session,
-                   crate_types: &[~str],
+                   trans: &CrateTranslation,
                    obj_filename: &Path,
-                   out_filename: &Path,
-                   lm: LinkMeta) {
+                   out_filename: &Path) {
     let outputs = if sess.opts.test {
         // If we're generating a test executable, then ignore all other output
         // styles at all other locations
@@ -795,7 +835,7 @@ pub fn link_binary(sess: Session,
         // look at what was in the crate file itself for generating output
         // formats.
         let mut outputs = sess.opts.outputs.clone();
-        for ty in crate_types.iter() {
+        for ty in trans.crate_types.iter() {
             if "bin" == *ty {
                 outputs.push(session::OutputExecutable);
             } else if "dylib" == *ty || "lib" == *ty {
@@ -813,12 +853,13 @@ pub fn link_binary(sess: Session,
     };
 
     for output in outputs.move_iter() {
-        link_binary_output(sess, output, obj_filename, out_filename, lm);
+        link_binary_output(sess, trans, output, obj_filename, out_filename);
     }
 
-    // Remove the temporary object file if we aren't saving temps
+    // Remove the temporary object file and metadata if we aren't saving temps
     if !sess.opts.save_temps {
         fs::unlink(obj_filename);
+        fs::unlink(&obj_filename.with_extension("metadata.o"));
     }
 }
 
@@ -832,11 +873,11 @@ fn is_writeable(p: &Path) -> bool {
 }
 
 fn link_binary_output(sess: Session,
+                      trans: &CrateTranslation,
                       output: session::OutputStyle,
                       obj_filename: &Path,
-                      out_filename: &Path,
-                      lm: LinkMeta) {
-    let libname = output_lib_filename(lm);
+                      out_filename: &Path) {
+    let libname = output_lib_filename(trans.link);
     let out_filename = match output {
         session::OutputRlib => {
             out_filename.with_filename(format!("lib{}.rlib", libname))
@@ -874,7 +915,7 @@ fn link_binary_output(sess: Session,
 
     match output {
         session::OutputRlib => {
-            link_rlib(sess, obj_filename, &out_filename);
+            link_rlib(sess, Some(trans), obj_filename, &out_filename);
         }
         session::OutputStaticlib => {
             link_staticlib(sess, obj_filename, &out_filename);
@@ -894,9 +935,25 @@ fn link_binary_output(sess: Session,
 // rlib primarily contains the object file of the crate, but it also contains
 // all of the object files from native libraries. This is done by unzipping
 // native libraries and inserting all of the contents into this archive.
-fn link_rlib(sess: Session, obj_filename: &Path,
+//
+// Instead of putting the metadata in an object file section, instead rlibs
+// contain the metadata in a separate file.
+fn link_rlib(sess: Session,
+             trans: Option<&CrateTranslation>, // None == no metadata
+             obj_filename: &Path,
              out_filename: &Path) -> Archive {
     let mut a = Archive::create(sess, out_filename, obj_filename);
+
+    match trans {
+        Some(trans) => {
+            let metadata = obj_filename.with_filename(METADATA_FILENAME);
+            fs::File::create(&metadata).write(trans.metadata);
+            a.add_file(&metadata);
+            fs::unlink(&metadata);
+        }
+        None => {}
+    }
+
     for &(ref l, kind) in cstore::get_used_libraries(sess.cstore).iter() {
         match kind {
             cstore::NativeStatic => {
@@ -916,8 +973,12 @@ fn link_rlib(sess: Session, obj_filename: &Path,
 //
 // Additionally, there's no way for us to link dynamic libraries, so we warn
 // about all dynamic library dependencies that they're not linked in.
+//
+// There's no need to include metadata in a static archive, so ensure to not
+// link in the metadata object file (and also don't prepare the archive with a
+// metadata file).
 fn link_staticlib(sess: Session, obj_filename: &Path, out_filename: &Path) {
-    let mut a = link_rlib(sess, obj_filename, out_filename);
+    let mut a = link_rlib(sess, None, obj_filename, out_filename);
     a.add_native_library("morestack");
 
     let crates = cstore::get_used_crates(sess.cstore, cstore::RequireStatic);
@@ -998,6 +1059,14 @@ fn link_args(sess: Session,
         ~"-o", out_filename.as_str().unwrap().to_owned(),
         obj_filename.as_str().unwrap().to_owned()]);
 
+    // When linking a dynamic library, we put the metadata into a section of the
+    // executable. This metadata is in a separate object file from the main
+    // object file, so we link that in here.
+    if dylib {
+        let metadata = obj_filename.with_extension("metadata.o");
+        args.push(metadata.as_str().unwrap().to_owned());
+    }
+
     if sess.targ_cfg.os == abi::OsLinux {
         // GNU-style linkers will use this to omit linking to libraries which
         // don't actually fulfill any relocations, but only for libraries which

src/librustc/driver/driver.rs

@@ -335,8 +335,10 @@ pub fn phase_3_run_analysis_passes(sess: Session,
 pub struct CrateTranslation {
     context: ContextRef,
     module: ModuleRef,
+    metadata_module: ModuleRef,
     link: LinkMeta,
     crate_types: ~[~str],
+    metadata: ~[u8],
 }
 
 /// Run the translation phase to LLVM, after which the AST and analysis can
@@ -362,8 +364,7 @@ pub fn phase_5_run_llvm_passes(sess: Session,
 
         time(sess.time_passes(), "LLVM passes", (), |_|
             link::write::run_passes(sess,
-                                    trans.context,
-                                    trans.module,
+                                    trans,
                                     output_type,
                                     &asm_filename));
 
@@ -376,8 +377,7 @@ pub fn phase_5_run_llvm_passes(sess: Session,
     } else {
         time(sess.time_passes(), "LLVM passes", (), |_|
             link::write::run_passes(sess,
-                                    trans.context,
-                                    trans.module,
+                                    trans,
                                     sess.opts.output_type,
                                     &outputs.obj_filename));
     }
@@ -390,10 +390,9 @@ pub fn phase_6_link_output(sess: Session,
                            outputs: &OutputFilenames) {
     time(sess.time_passes(), "linking", (), |_|
          link::link_binary(sess,
-                           trans.crate_types,
+                           trans,
                            &outputs.obj_filename,
-                           &outputs.out_filename,
-                           trans.link));
+                           &outputs.out_filename));
 }
 
 pub fn stop_after_phase_3(sess: Session) -> bool {

src/librustc/metadata/encoder.rs

@@ -21,13 +21,14 @@
 use middle::typeck;
 use middle;
 
+use std::cast;
 use std::hashmap::{HashMap, HashSet};
-use std::io::{Writer, Seek, Decorator};
 use std::io::mem::MemWriter;
+use std::io::{Writer, Seek, Decorator};
 use std::str;
+use std::util;
 use std::vec;
 
-use extra::flate;
 use extra::serialize::Encodable;
 use extra;
 
@@ -47,8 +48,6 @@
 use syntax;
 use writer = extra::ebml::writer;
 
-use std::cast;
-
 // used by astencode:
 type abbrev_map = @mut HashMap<ty::t, tyencode::ty_abbrev>;
 
@@ -1871,10 +1870,9 @@ pub fn encode_metadata(parms: EncodeParams, crate: &Crate) -> ~[u8] {
     // remaining % 4 bytes.
     wr.write(&[0u8, 0u8, 0u8, 0u8]);
 
-    let writer_bytes: &mut ~[u8] = wr.inner_mut_ref();
-
-    metadata_encoding_version.to_owned() +
-        flate::deflate_bytes(*writer_bytes)
+    // This is a horrible thing to do to the outer MemWriter, but thankfully we
+    // don't use it again so... it's ok right?
+    return util::replace(wr.inner_mut_ref(), ~[]);
 }
 
 // Get the encoded string for a type

src/librustc/metadata/loader.rs

@@ -10,7 +10,7 @@
 
 //! Finds crate binaries and loads their metadata
 
-use back::archive::Archive;
+use back::archive::{Archive, METADATA_FILENAME};
 use driver::session::Session;
 use lib::llvm::{False, llvm, ObjectFile, mk_section_iter};
 use metadata::decoder;
@@ -27,7 +27,6 @@
 use std::c_str::ToCStr;
 use std::cast;
 use std::io;
-use std::libc;
 use std::num;
 use std::option;
 use std::os::consts::{macos, freebsd, linux, android, win32};
@@ -102,8 +101,7 @@ fn find_library_crate(&self) -> Option<Library> {
                     if candidate && existing {
                         FileMatches
                     } else if candidate {
-                        match get_metadata_section(self.sess, self.os, path,
-                                                   crate_name) {
+                        match get_metadata_section(self.sess, self.os, path) {
                             Some(cvec) =>
                                 if crate_matches(cvec, self.metas, self.hash) {
                                     debug!("found {} with matching metadata",
@@ -271,22 +269,15 @@ pub fn metadata_matches(extern_metas: &[@ast::MetaItem],
     local_metas.iter().all(|needed| attr::contains(extern_metas, *needed))
 }
 
-fn get_metadata_section(sess: Session, os: Os, filename: &Path,
-                        crate_name: &str) -> Option<@~[u8]> {
+fn get_metadata_section(sess: Session, os: Os, filename: &Path) -> Option<@~[u8]> {
+    if filename.filename_str().unwrap().ends_with(".rlib") {
+        let archive = Archive::open(sess, filename.clone());
+        return Some(@archive.read(METADATA_FILENAME));
+    }
     unsafe {
-        let mb = if filename.filename_str().unwrap().ends_with(".rlib") {
-            let archive = Archive::open(sess, filename.clone());
-            let contents = archive.read(crate_name + ".o");
-            let ptr = vec::raw::to_ptr(contents);
-            crate_name.with_c_str(|name| {
-                llvm::LLVMCreateMemoryBufferWithMemoryRangeCopy(
-                    ptr as *i8, contents.len() as libc::size_t, name)
-            })
-        } else {
-            filename.with_c_str(|buf| {
-                llvm::LLVMRustCreateMemoryBufferWithContentsOfFile(buf)
-            })
-        };
+        let mb = filename.with_c_str(|buf| {
+            llvm::LLVMRustCreateMemoryBufferWithContentsOfFile(buf)
+        });
         if mb as int == 0 { return None }
         let of = match ObjectFile::new(mb) {
             Some(of) => of,
@@ -356,12 +347,7 @@ pub fn list_file_metadata(sess: Session,
                           os: Os,
                           path: &Path,
                           out: @mut io::Writer) {
-    // guess the crate name from the pathname
-    let crate_name = path.filename_str().unwrap();
-    let crate_name = if crate_name.starts_with("lib") {
-        crate_name.slice_from(3) } else { crate_name };
-    let crate_name = crate_name.split('-').next().unwrap();
-    match get_metadata_section(sess, os, path, crate_name) {
+    match get_metadata_section(sess, os, path) {
       option::Some(bytes) => decoder::list_crate_metadata(intr, bytes, out),
       option::None => {
         write!(out, "could not find metadata in {}.\n", path.display())

src/librustc/middle/trans/base.rs

@@ -3044,19 +3044,24 @@ pub fn crate_ctxt_to_encode_parms<'r>(cx: &'r CrateContext, ie: encoder::encode_
         }
 }
 
-pub fn write_metadata(cx: &CrateContext, crate: &ast::Crate) {
-    if !*cx.sess.building_library { return; }
+pub fn write_metadata(cx: &CrateContext, crate: &ast::Crate) -> ~[u8] {
+    use extra::flate;
+
+    if !*cx.sess.building_library { return ~[]; }
 
     let encode_inlined_item: encoder::encode_inlined_item =
         |ecx, ebml_w, path, ii|
         astencode::encode_inlined_item(ecx, ebml_w, path, ii, cx.maps);
 
     let encode_parms = crate_ctxt_to_encode_parms(cx, encode_inlined_item);
-    let llmeta = C_bytes(encoder::encode_metadata(encode_parms, crate));
+    let metadata = encoder::encode_metadata(encode_parms, crate);
+    let compressed = encoder::metadata_encoding_version +
+                        flate::deflate_bytes(metadata);
+    let llmeta = C_bytes(compressed);
     let llconst = C_struct([llmeta], false);
     let mut llglobal = "rust_metadata".with_c_str(|buf| {
         unsafe {
-            llvm::LLVMAddGlobal(cx.llmod, val_ty(llconst).to_ref(), buf)
+            llvm::LLVMAddGlobal(cx.metadata_llmod, val_ty(llconst).to_ref(), buf)
         }
     });
     unsafe {
@@ -3069,11 +3074,13 @@ pub fn write_metadata(cx: &CrateContext, crate: &ast::Crate) {
         let t_ptr_i8 = Type::i8p();
         llglobal = llvm::LLVMConstBitCast(llglobal, t_ptr_i8.to_ref());
         let llvm_used = "llvm.used".with_c_str(|buf| {
-            llvm::LLVMAddGlobal(cx.llmod, Type::array(&t_ptr_i8, 1).to_ref(), buf)
+            llvm::LLVMAddGlobal(cx.metadata_llmod,
+                                Type::array(&t_ptr_i8, 1).to_ref(), buf)
         });
         lib::llvm::SetLinkage(llvm_used, lib::llvm::AppendingLinkage);
         llvm::LLVMSetInitializer(llvm_used, C_array(t_ptr_i8, [llglobal]));
     }
+    return metadata;
 }
 
 pub fn trans_crate(sess: session::Session,
@@ -3140,7 +3147,7 @@ pub fn trans_crate(sess: session::Session,
     }
 
     // Translate the metadata.
-    write_metadata(ccx, &crate);
+    let metadata = write_metadata(ccx, &crate);
     if ccx.sess.trans_stats() {
         println("--- trans stats ---");
         println!("n_static_tydescs: {}", ccx.stats.n_static_tydescs);
@@ -3187,5 +3194,7 @@ pub fn trans_crate(sess: session::Session,
         module: llmod,
         link: link_meta,
         crate_types: crate_types,
+        metadata_module: ccx.metadata_llmod,
+        metadata: metadata,
     };
 }

src/librustc/middle/trans/context.rs

@@ -42,6 +42,7 @@ pub struct CrateContext {
      sess: session::Session,
      llmod: ModuleRef,
      llcx: ContextRef,
+     metadata_llmod: ModuleRef,
      td: TargetData,
      tn: TypeNames,
      externs: ExternMap,
@@ -134,11 +135,18 @@ pub fn new(sess: session::Session,
             let llmod = name.with_c_str(|buf| {
                 llvm::LLVMModuleCreateWithNameInContext(buf, llcx)
             });
+            let metadata_llmod = format!("{}_metadata", name).with_c_str(|buf| {
+                llvm::LLVMModuleCreateWithNameInContext(buf, llcx)
+            });
             let data_layout: &str = sess.targ_cfg.target_strs.data_layout;
             let targ_triple: &str = sess.targ_cfg.target_strs.target_triple;
-            data_layout.with_c_str(|buf| llvm::LLVMSetDataLayout(llmod, buf));
+            data_layout.with_c_str(|buf| {
+                llvm::LLVMSetDataLayout(llmod, buf);
+                llvm::LLVMSetDataLayout(metadata_llmod, buf);
+            });
             targ_triple.with_c_str(|buf| {
-                llvm::LLVMRustSetNormalizedTarget(llmod, buf)
+                llvm::LLVMRustSetNormalizedTarget(llmod, buf);
+                llvm::LLVMRustSetNormalizedTarget(metadata_llmod, buf);
             });
             let targ_cfg = sess.targ_cfg;
 
@@ -174,6 +182,7 @@ pub fn new(sess: session::Session,
                   sess: sess,
                   llmod: llmod,
                   llcx: llcx,
+                  metadata_llmod: metadata_llmod,
                   td: td,
                   tn: tn,
                   externs: HashMap::new(),