rustc: Add a `#[wasm_custom_section]` attribute

This commit is an implementation of adding custom sections to wasm artifacts in
rustc. The intention here is to expose the ability of the wasm binary format to
contain custom sections with arbitrary user-defined data. Currently neither our
version of LLVM nor LLD supports this so the implementation is currently custom
to rustc itself.

The implementation here is to attach a `#[wasm_custom_section = "foo"]`
attribute to any `const` which has a type like `[u8; N]`. Other types of
constants aren't supported yet but may be added one day! This should hopefully
be enough to get off the ground with *some* custom section support.

The current semantics are that any constant tagged with `#[wasm_custom_section]`
section will be *appended* to the corresponding section in the final output wasm
artifact (and this affects dependencies linked in as well, not just the final
crate). This means that whatever is interpreting the contents must be able to
interpret binary-concatenated sections (or each constant needs to be in its own
custom section).

To test this change the existing `run-make` test suite was moved to a
`run-make-fulldeps` folder and a new `run-make` test suite was added which
applies to all targets by default. This test suite currently only has one test
which only runs for the wasm target (using a node.js script to use `WebAssembly`
in JS to parse the wasm output).

src/bootstrap/builder.rs

@@ -313,6 +313,7 @@ fn get_step_descriptions(kind: Kind) -> Vec<StepDescription> {
                 test::RunPassFullDepsPretty, test::RunFailFullDepsPretty,
                 test::Crate, test::CrateLibrustc, test::CrateRustdoc, test::Linkcheck,
                 test::Cargotest, test::Cargo, test::Rls, test::ErrorIndex, test::Distcheck,
+                test::RunMakeFullDeps,
                 test::Nomicon, test::Reference, test::RustdocBook, test::RustByExample,
                 test::TheBook, test::UnstableBook,
                 test::Rustfmt, test::Miri, test::Clippy, test::RustdocJS, test::RustdocTheme,

src/bootstrap/compile.rs

@@ -915,7 +915,7 @@ fn run(self, builder: &Builder) -> Compiler {
             }
         }
 
-        let lld_install = if build.config.lld_enabled && target_compiler.stage > 0 {
+        let lld_install = if build.config.lld_enabled {
             Some(builder.ensure(native::Lld {
                 target: target_compiler.host,
             }))

src/bootstrap/test.rs

@@ -759,12 +759,18 @@ fn run(self, builder: &Builder) {
     host: true
 });
 
-host_test!(RunMake {
+default_test!(RunMake {
     path: "src/test/run-make",
     mode: "run-make",
     suite: "run-make"
 });
 
+host_test!(RunMakeFullDeps {
+    path: "src/test/run-make-fulldeps",
+    mode: "run-make",
+    suite: "run-make-fulldeps"
+});
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 struct Compiletest {
     compiler: Compiler,
@@ -827,8 +833,7 @@ fn run(self, builder: &Builder) {
             // FIXME: Does pretty need librustc compiled? Note that there are
             // fulldeps test suites with mode = pretty as well.
             mode == "pretty" ||
-            mode == "rustdoc" ||
-            mode == "run-make" {
+            mode == "rustdoc" {
             builder.ensure(compile::Rustc { compiler, target });
         }
 
@@ -849,7 +854,7 @@ fn run(self, builder: &Builder) {
         cmd.arg("--rustc-path").arg(builder.rustc(compiler));
 
         // Avoid depending on rustdoc when we don't need it.
-        if mode == "rustdoc" || mode == "run-make" {
+        if mode == "rustdoc" || (mode == "run-make" && suite.ends_with("fulldeps")) {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler.host));
         }
 
@@ -931,7 +936,7 @@ fn run(self, builder: &Builder) {
 
             // Only pass correct values for these flags for the `run-make` suite as it
             // requires that a C++ compiler was configured which isn't always the case.
-            if suite == "run-make" {
+            if suite == "run-make-fulldeps" {
                 let llvm_components = output(Command::new(&llvm_config).arg("--components"));
                 let llvm_cxxflags = output(Command::new(&llvm_config).arg("--cxxflags"));
                 cmd.arg("--cc").arg(build.cc(target))
@@ -944,12 +949,12 @@ fn run(self, builder: &Builder) {
                 }
             }
         }
-        if suite == "run-make" && !build.config.llvm_enabled {
+        if suite == "run-make-fulldeps" && !build.config.llvm_enabled {
             println!("Ignoring run-make test suite as they generally don't work without LLVM");
             return;
         }
 
-        if suite != "run-make" {
+        if suite != "run-make-fulldeps" {
             cmd.arg("--cc").arg("")
                .arg("--cxx").arg("")
                .arg("--cflags").arg("")

src/librustc/dep_graph/dep_node.rs

@@ -650,6 +650,8 @@ pub fn fingerprint_needed_for_crate_hash(self) -> bool {
 
     [] GetSymbolExportLevel(DefId),
 
+    [] WasmCustomSections(CrateNum),
+
     [input] Features,
 
     [] ProgramClausesFor(DefId),

src/librustc/hir/check_attr.rs

@@ -25,6 +25,7 @@ enum Target {
     Struct,
     Union,
     Enum,
+    Const,
     Other,
 }
 
@@ -35,6 +36,7 @@ fn from_item(item: &hir::Item) -> Target {
             hir::ItemStruct(..) => Target::Struct,
             hir::ItemUnion(..) => Target::Union,
             hir::ItemEnum(..) => Target::Enum,
+            hir::ItemConst(..) => Target::Const,
             _ => Target::Other,
         }
     }
@@ -60,6 +62,17 @@ fn check_attributes(&self, item: &hir::Item, target: Target) {
                 if name == "inline" {
                     self.check_inline(attr, item, target)
                 }
+
+                if name == "wasm_custom_section" {
+                    if target != Target::Const {
+                        self.tcx.sess.span_err(attr.span, "only allowed on consts");
+                    }
+
+                    if attr.value_str().is_none() {
+                        self.tcx.sess.span_err(attr.span, "must be of the form \
+                            #[wasm_custom_section = \"foo\"]");
+                    }
+                }
             }
         }
 

src/librustc/middle/dead.rs

@@ -318,6 +318,11 @@ fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt,
         return true;
     }
 
+    // These constants are special for wasm
+    if attr::contains_name(attrs, "wasm_custom_section") {
+        return true;
+    }
+
     tcx.lint_level_at_node(lint::builtin::DEAD_CODE, id).0 == lint::Allow
 }
 

src/librustc/ty/maps/config.rs

@@ -678,6 +678,12 @@ fn describe(tcx: TyCtxt, def: ty::InstanceDef<'tcx>) -> String {
     }
 }
 
+impl<'tcx> QueryDescription<'tcx> for queries::wasm_custom_sections<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("custom wasm sections for a crate")
+    }
+}
+
 impl<'tcx> QueryDescription<'tcx> for queries::generics_of<'tcx> {
     #[inline]
     fn cache_on_disk(def_id: Self::Key) -> bool {

src/librustc/ty/maps/mod.rs

@@ -424,6 +424,8 @@
     [] fn features_query: features_node(CrateNum) -> Lrc<feature_gate::Features>,
 
     [] fn program_clauses_for: ProgramClausesFor(DefId) -> Lrc<Vec<Clause<'tcx>>>,
+
+    [] fn wasm_custom_sections: WasmCustomSections(CrateNum) -> Lrc<Vec<DefId>>,
 }
 
 //////////////////////////////////////////////////////////////////////

src/librustc/ty/maps/plumbing.rs

@@ -940,6 +940,7 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
         DepKind::Features => { force!(features_query, LOCAL_CRATE); }
 
         DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); }
+        DepKind::WasmCustomSections => { force!(wasm_custom_sections, krate!()); }
     }
 
     true

src/librustc_metadata/cstore_impl.rs

@@ -271,6 +271,8 @@ fn into_args(self) -> (DefId, DefId) { (self.0.as_def_id(), self.1) }
 
         Arc::new(cdata.exported_symbols())
     }
+
+    wasm_custom_sections => { Lrc::new(cdata.wasm_custom_sections()) }
 }
 
 pub fn provide<'tcx>(providers: &mut Providers<'tcx>) {

src/librustc_metadata/decoder.rs

@@ -1067,6 +1067,16 @@ pub fn exported_symbols(&self) -> Vec<(ExportedSymbol, SymbolExportLevel)> {
             .collect()
     }
 
+    pub fn wasm_custom_sections(&self) -> Vec<DefId> {
+        let sections = self.root
+            .wasm_custom_sections
+            .decode(self)
+            .map(|def_index| self.local_def_id(def_index))
+            .collect::<Vec<_>>();
+        info!("loaded wasm sections {:?}", sections);
+        return sections
+    }
+
     pub fn get_macro(&self, id: DefIndex) -> (InternedString, MacroDef) {
         let entry = self.entry(id);
         match entry.kind {

src/librustc_metadata/encoder.rs

@@ -435,6 +435,12 @@ fn encode_crate_root(&mut self) -> Lazy<CrateRoot> {
             &exported_symbols);
         let exported_symbols_bytes = self.position() - i;
 
+        // encode wasm custom sections
+        let wasm_custom_sections = self.tcx.wasm_custom_sections(LOCAL_CRATE);
+        let wasm_custom_sections = self.tracked(
+            IsolatedEncoder::encode_wasm_custom_sections,
+            &wasm_custom_sections);
+
         // Encode and index the items.
         i = self.position();
         let items = self.encode_info_for_items();
@@ -478,6 +484,7 @@ fn encode_crate_root(&mut self) -> Lazy<CrateRoot> {
             def_path_table,
             impls,
             exported_symbols,
+            wasm_custom_sections,
             index,
         });
 
@@ -1444,6 +1451,11 @@ fn encode_exported_symbols(&mut self,
             .cloned())
     }
 
+    fn encode_wasm_custom_sections(&mut self, statics: &[DefId]) -> LazySeq<DefIndex> {
+        info!("encoding custom wasm section constants {:?}", statics);
+        self.lazy_seq(statics.iter().map(|id| id.index))
+    }
+
     fn encode_dylib_dependency_formats(&mut self, _: ()) -> LazySeq<Option<LinkagePreference>> {
         match self.tcx.sess.dependency_formats.borrow().get(&config::CrateTypeDylib) {
             Some(arr) => {

src/librustc_metadata/schema.rs

@@ -204,6 +204,7 @@ pub struct CrateRoot {
     pub def_path_table: Lazy<hir::map::definitions::DefPathTable>,
     pub impls: LazySeq<TraitImpls>,
     pub exported_symbols: LazySeq<(ExportedSymbol, SymbolExportLevel)>,
+    pub wasm_custom_sections: LazySeq<DefIndex>,
 
     pub index: LazySeq<index::Index>,
 }

src/librustc_trans/attributes.rs

@@ -11,9 +11,11 @@
 
 use std::ffi::{CStr, CString};
 
-use rustc::hir::TransFnAttrFlags;
+use rustc::hir::{self, TransFnAttrFlags};
 use rustc::hir::def_id::{DefId, LOCAL_CRATE};
+use rustc::hir::itemlikevisit::ItemLikeVisitor;
 use rustc::session::config::Sanitizer;
+use rustc::ty::TyCtxt;
 use rustc::ty::maps::Providers;
 use rustc_data_structures::sync::Lrc;
 
@@ -161,4 +163,32 @@ pub fn provide(providers: &mut Providers) {
                 .collect())
         }
     };
+
+    providers.wasm_custom_sections = |tcx, cnum| {
+        assert_eq!(cnum, LOCAL_CRATE);
+        let mut finder = WasmSectionFinder { tcx, list: Vec::new() };
+        tcx.hir.krate().visit_all_item_likes(&mut finder);
+        Lrc::new(finder.list)
+    };
+}
+
+struct WasmSectionFinder<'a, 'tcx: 'a> {
+    tcx: TyCtxt<'a, 'tcx, 'tcx>,
+    list: Vec<DefId>,
+}
+
+impl<'a, 'tcx: 'a> ItemLikeVisitor<'tcx> for WasmSectionFinder<'a, 'tcx> {
+    fn visit_item(&mut self, i: &'tcx hir::Item) {
+        match i.node {
+            hir::ItemConst(..) => {}
+            _ => return,
+        }
+        if i.attrs.iter().any(|i| i.check_name("wasm_custom_section")) {
+            self.list.push(self.tcx.hir.local_def_id(i.id));
+        }
+    }
+
+    fn visit_trait_item(&mut self, _: &'tcx hir::TraitItem) {}
+
+    fn visit_impl_item(&mut self, _: &'tcx hir::ImplItem) {}
 }

src/librustc_trans/back/link.rs

@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+use back::wasm;
 use cc::windows_registry;
 use super::archive::{ArchiveBuilder, ArchiveConfig};
 use super::bytecode::RLIB_BYTECODE_EXTENSION;
@@ -810,6 +811,11 @@ fn escape_string(s: &[u8]) -> String {
             Err(e) => sess.fatal(&format!("failed to run dsymutil: {}", e)),
         }
     }
+
+    if sess.opts.target_triple == "wasm32-unknown-unknown" {
+        wasm::add_custom_sections(&out_filename,
+                                  &trans.crate_info.wasm_custom_sections);
+    }
 }
 
 fn exec_linker(sess: &Session, cmd: &mut Command, tmpdir: &Path)

src/librustc_trans/back/wasm.rs

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::fs;
+use std::path::Path;
+use std::collections::BTreeMap;
+
+use serialize::leb128;
+
+pub fn add_custom_sections(path: &Path, sections: &BTreeMap<String, Vec<u8>>) {
+    let mut wasm = fs::read(path).expect("failed to read wasm output");
+
+    // see https://webassembly.github.io/spec/core/binary/modules.html#custom-section
+    for (section, bytes) in sections {
+        // write the `id` identifier, 0 for a custom section
+        let len = wasm.len();
+        leb128::write_u32_leb128(&mut wasm, len, 0);
+
+        // figure out how long our name descriptor will be
+        let mut name = Vec::new();
+        leb128::write_u32_leb128(&mut name, 0, section.len() as u32);
+        name.extend_from_slice(section.as_bytes());
+
+        // write the length of the payload
+        let len = wasm.len();
+        let total_len = bytes.len() + name.len();
+        leb128::write_u32_leb128(&mut wasm, len, total_len as u32);
+
+        // write out the name section
+        wasm.extend(name);
+
+        // and now the payload itself
+        wasm.extend_from_slice(bytes);
+    }
+
+    fs::write(path, &wasm).expect("failed to write wasm output");
+}

src/librustc_trans/base.rs

@@ -74,6 +74,7 @@
 use CrateInfo;
 
 use std::any::Any;
+use std::collections::BTreeMap;
 use std::ffi::CString;
 use std::str;
 use std::sync::Arc;
@@ -1070,8 +1071,24 @@ pub fn new(tcx: TyCtxt) -> CrateInfo {
             used_crates_dynamic: cstore::used_crates(tcx, LinkagePreference::RequireDynamic),
             used_crates_static: cstore::used_crates(tcx, LinkagePreference::RequireStatic),
             used_crate_source: FxHashMap(),
+            wasm_custom_sections: BTreeMap::new(),
         };
 
+        let load_wasm_sections = tcx.sess.crate_types.borrow()
+            .iter()
+            .any(|c| *c != config::CrateTypeRlib) &&
+            tcx.sess.opts.target_triple == "wasm32-unknown-unknown";
+
+        if load_wasm_sections {
+            info!("attempting to load all wasm sections");
+            for &id in tcx.wasm_custom_sections(LOCAL_CRATE).iter() {
+                let (name, contents) = fetch_wasm_section(tcx, id);
+                info.wasm_custom_sections.entry(name)
+                    .or_insert(Vec::new())
+                    .extend(contents);
+            }
+        }
+
         for &cnum in tcx.crates().iter() {
             info.native_libraries.insert(cnum, tcx.native_libraries(cnum));
             info.crate_name.insert(cnum, tcx.crate_name(cnum).to_string());
@@ -1091,6 +1108,14 @@ pub fn new(tcx: TyCtxt) -> CrateInfo {
             if tcx.is_no_builtins(cnum) {
                 info.is_no_builtins.insert(cnum);
             }
+            if load_wasm_sections {
+                for &id in tcx.wasm_custom_sections(cnum).iter() {
+                    let (name, contents) = fetch_wasm_section(tcx, id);
+                    info.wasm_custom_sections.entry(name)
+                        .or_insert(Vec::new())
+                        .extend(contents);
+                }
+            }
         }
 
 
@@ -1270,3 +1295,44 @@ fn hash_stable<W: StableHasherResult>(&self,
         }
     }
 }
+
+fn fetch_wasm_section(tcx: TyCtxt, id: DefId) -> (String, Vec<u8>) {
+    use rustc::mir::interpret::{GlobalId, Value, PrimVal};
+    use rustc::middle::const_val::ConstVal;
+
+    info!("loading wasm section {:?}", id);
+
+    let section = tcx.get_attrs(id)
+        .iter()
+        .find(|a| a.check_name("wasm_custom_section"))
+        .expect("missing #[wasm_custom_section] attribute")
+        .value_str()
+        .expect("malformed #[wasm_custom_section] attribute");
+
+    let instance = ty::Instance::mono(tcx, id);
+    let cid = GlobalId {
+        instance,
+        promoted: None
+    };
+    let param_env = ty::ParamEnv::reveal_all();
+    let val = tcx.const_eval(param_env.and(cid)).unwrap();
+
+    let val = match val.val {
+        ConstVal::Value(val) => val,
+        ConstVal::Unevaluated(..) => bug!("should be evaluated"),
+    };
+    let val = match val {
+        Value::ByRef(ptr, _align) => ptr.into_inner_primval(),
+        ref v => bug!("should be ByRef, was {:?}", v),
+    };
+    let mem = match val {
+        PrimVal::Ptr(mem) => mem,
+        ref v => bug!("should be Ptr, was {:?}", v),
+    };
+    assert_eq!(mem.offset, 0);
+    let alloc = tcx
+        .interpret_interner
+        .get_alloc(mem.alloc_id)
+        .expect("miri allocation never successfully created");
+    (section.to_string(), alloc.bytes.clone())
+}

src/librustc_trans/lib.rs

@@ -72,6 +72,7 @@
 use std::any::Any;
 use std::path::PathBuf;
 use std::sync::mpsc;
+use std::collections::BTreeMap;
 use rustc_data_structures::sync::Lrc;
 
 use rustc::dep_graph::DepGraph;
@@ -98,6 +99,7 @@ mod back {
     pub mod symbol_export;
     pub mod write;
     mod rpath;
+    mod wasm;
 }
 
 mod abi;
@@ -400,6 +402,7 @@ struct CrateInfo {
     used_crate_source: FxHashMap<CrateNum, Lrc<CrateSource>>,
     used_crates_static: Vec<(CrateNum, LibSource)>,
     used_crates_dynamic: Vec<(CrateNum, LibSource)>,
+    wasm_custom_sections: BTreeMap<String, Vec<u8>>,
 }
 
 __build_diagnostic_array! { librustc_trans, DIAGNOSTICS }

src/librustc_typeck/check/mod.rs

@@ -1182,9 +1182,15 @@ pub fn check_item_type<'a,'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, it: &'tcx hir::Item
     let _indenter = indenter();
     match it.node {
       // Consts can play a role in type-checking, so they are included here.
-      hir::ItemStatic(..) |
+      hir::ItemStatic(..) => {
+        tcx.typeck_tables_of(tcx.hir.local_def_id(it.id));
+      }
       hir::ItemConst(..) => {
         tcx.typeck_tables_of(tcx.hir.local_def_id(it.id));
+        if it.attrs.iter().any(|a| a.check_name("wasm_custom_section")) {
+            let def_id = tcx.hir.local_def_id(it.id);
+            check_const_is_u8_array(tcx, def_id, it.span);
+        }
       }
       hir::ItemEnum(ref enum_definition, _) => {
         check_enum(tcx,
@@ -1256,6 +1262,21 @@ pub fn check_item_type<'a,'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, it: &'tcx hir::Item
     }
 }
 
+fn check_const_is_u8_array<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
+                                     def_id: DefId,
+                                     span: Span) {
+    match tcx.type_of(def_id).sty {
+        ty::TyArray(t, _) => {
+            match t.sty {
+                ty::TyUint(ast::UintTy::U8) => return,
+                _ => {}
+            }
+        }
+        _ => {}
+    }
+    tcx.sess.span_err(span, "must be an array of bytes like `[u8; N]`");
+}
+
 fn check_on_unimplemented<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                                     trait_def_id: DefId,
                                     item: &hir::Item) {

src/libsyntax/feature_gate.rs

@@ -451,6 +451,9 @@ pub fn walk_feature_fields<F>(&self, mut f: F)
 
     // `use path as _;` and `extern crate c as _;`
     (active, underscore_imports, "1.26.0", Some(48216), None),
+
+    // The #[wasm_custom_section] attribute
+    (active, wasm_custom_section, "1.26.0", None, None),
 );
 
 declare_features! (
@@ -1004,6 +1007,11 @@ pub fn is_builtin_attr(attr: &ast::Attribute) -> bool {
                                  "never will be stable",
                                  cfg_fn!(rustc_attrs))),
 
+    ("wasm_custom_section", Whitelisted, Gated(Stability::Unstable,
+                                 "wasm_custom_section",
+                                 "attribute is currently unstable",
+                                 cfg_fn!(wasm_custom_section))),
+
     // Crate level attributes
     ("crate_name", CrateLevel, Ungated),
     ("crate_type", CrateLevel, Ungated),