Rollup merge of rust-lang#48883 - alexcrichton:wasm-custom-sections, r=nikomatsakis

rustc: Add a `#[wasm_custom_section]` attribute

This commit is an implementation of adding custom sections to wasm artifacts in
rustc. The intention here is to expose the ability of the wasm binary format to
contain custom sections with arbitrary user-defined data. Currently neither our
version of LLVM nor LLD supports this so the implementation is currently custom
to rustc itself.

The implementation here is to attach a `#[wasm_custom_section = "foo"]`
attribute to any `const` which has a type like `[u8; N]`. Other types of
constants aren't supported yet but may be added one day! This should hopefully
be enough to get off the ground with *some* custom section support.

The current semantics are that any constant tagged with `#[wasm_custom_section]`
section will be *appended* to the corresponding section in the final output wasm
artifact (and this affects dependencies linked in as well, not just the final
crate). This means that whatever is interpreting the contents must be able to
interpret binary-concatenated sections (or each constant needs to be in its own
custom section).

To test this change the existing `run-make` test suite was moved to a
`run-make-fulldeps` folder and a new `run-make` test suite was added which
applies to all targets by default. This test suite currently only has one test
which only runs for the wasm target (using a node.js script to use `WebAssembly`
in JS to parse the wasm output).

Author: alexcrichton

src/bootstrap/builder.rs

@@ -313,6 +313,7 @@ impl<'a> Builder<'a> {
                 test::RunPassFullDepsPretty, test::RunFailFullDepsPretty,
                 test::Crate, test::CrateLibrustc, test::CrateRustdoc, test::Linkcheck,
                 test::Cargotest, test::Cargo, test::Rls, test::ErrorIndex, test::Distcheck,
+                test::RunMakeFullDeps,
                 test::Nomicon, test::Reference, test::RustdocBook, test::RustByExample,
                 test::TheBook, test::UnstableBook,
                 test::Rustfmt, test::Miri, test::Clippy, test::RustdocJS, test::RustdocTheme,

src/bootstrap/compile.rs

@@ -915,7 +915,7 @@ impl Step for Assemble {
             }
         }
 
-        let lld_install = if build.config.lld_enabled && target_compiler.stage > 0 {
+        let lld_install = if build.config.lld_enabled {
             Some(builder.ensure(native::Lld {
                 target: target_compiler.host,
             }))

src/bootstrap/test.rs

@@ -759,12 +759,18 @@ test!(RunFailFullDepsPretty {
     host: true
 });
 
-host_test!(RunMake {
+default_test!(RunMake {
     path: "src/test/run-make",
     mode: "run-make",
     suite: "run-make"
 });
 
+host_test!(RunMakeFullDeps {
+    path: "src/test/run-make-fulldeps",
+    mode: "run-make",
+    suite: "run-make-fulldeps"
+});
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 struct Compiletest {
     compiler: Compiler,
@@ -827,8 +833,7 @@ impl Step for Compiletest {
             // FIXME: Does pretty need librustc compiled? Note that there are
             // fulldeps test suites with mode = pretty as well.
             mode == "pretty" ||
-            mode == "rustdoc" ||
-            mode == "run-make" {
+            mode == "rustdoc" {
             builder.ensure(compile::Rustc { compiler, target });
         }
 
@@ -849,7 +854,7 @@ impl Step for Compiletest {
         cmd.arg("--rustc-path").arg(builder.rustc(compiler));
 
         // Avoid depending on rustdoc when we don't need it.
-        if mode == "rustdoc" || mode == "run-make" {
+        if mode == "rustdoc" || (mode == "run-make" && suite.ends_with("fulldeps")) {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler.host));
         }
 
@@ -931,7 +936,7 @@ impl Step for Compiletest {
 
             // Only pass correct values for these flags for the `run-make` suite as it
             // requires that a C++ compiler was configured which isn't always the case.
-            if suite == "run-make" {
+            if suite == "run-make-fulldeps" {
                 let llvm_components = output(Command::new(&llvm_config).arg("--components"));
                 let llvm_cxxflags = output(Command::new(&llvm_config).arg("--cxxflags"));
                 cmd.arg("--cc").arg(build.cc(target))
@@ -944,12 +949,12 @@ impl Step for Compiletest {
                 }
             }
         }
-        if suite == "run-make" && !build.config.llvm_enabled {
+        if suite == "run-make-fulldeps" && !build.config.llvm_enabled {
             println!("Ignoring run-make test suite as they generally don't work without LLVM");
             return;
         }
 
-        if suite != "run-make" {
+        if suite != "run-make-fulldeps" {
             cmd.arg("--cc").arg("")
                .arg("--cxx").arg("")
                .arg("--cflags").arg("")

src/ci/docker/wasm32-unknown/Dockerfile

@@ -26,6 +26,7 @@ ENV RUST_CONFIGURE_ARGS \
   --set rust.lld
 
 ENV SCRIPT python2.7 /checkout/x.py test --target $TARGETS \
+  src/test/run-make \
   src/test/ui \
   src/test/run-pass \
   src/test/compile-fail \

src/librustc/dep_graph/dep_node.rs

@@ -593,6 +593,7 @@ define_dep_nodes!( <'tcx>
     [] ImplementationsOfTrait { krate: CrateNum, trait_id: DefId },
     [] AllTraitImplementations(CrateNum),
 
+    [] DllimportForeignItems(CrateNum),
     [] IsDllimportForeignItem(DefId),
     [] IsStaticallyIncludedForeignItem(DefId),
     [] NativeLibraryKind(DefId),
@@ -650,9 +651,13 @@ define_dep_nodes!( <'tcx>
 
     [] GetSymbolExportLevel(DefId),
 
+    [] WasmCustomSections(CrateNum),
+
     [input] Features,
 
     [] ProgramClausesFor(DefId),
+    [] WasmImportModuleMap(CrateNum),
+    [] ForeignModules(CrateNum),
 );
 
 trait DepNodeParams<'a, 'gcx: 'tcx + 'a, 'tcx: 'a> : fmt::Debug {

src/librustc/hir/check_attr.rs

@@ -25,6 +25,8 @@ enum Target {
     Struct,
     Union,
     Enum,
+    Const,
+    ForeignMod,
     Other,
 }
 
@@ -35,6 +37,8 @@ impl Target {
             hir::ItemStruct(..) => Target::Struct,
             hir::ItemUnion(..) => Target::Union,
             hir::ItemEnum(..) => Target::Enum,
+            hir::ItemConst(..) => Target::Const,
+            hir::ItemForeignMod(..) => Target::ForeignMod,
             _ => Target::Other,
         }
     }
@@ -55,14 +59,42 @@ impl<'a, 'tcx> CheckAttrVisitor<'a, 'tcx> {
                 .emit();
         }
 
+        let mut has_wasm_import_module = false;
         for attr in &item.attrs {
-            if let Some(name) = attr.name() {
-                if name == "inline" {
-                    self.check_inline(attr, item, target)
+            if attr.check_name("inline") {
+                self.check_inline(attr, item, target)
+            } else if attr.check_name("wasm_import_module") {
+                has_wasm_import_module = true;
+                if attr.value_str().is_none() {
+                    self.tcx.sess.span_err(attr.span, "\
+                        must be of the form #[wasm_import_module = \"...\"]");
+                }
+                if target != Target::ForeignMod {
+                    self.tcx.sess.span_err(attr.span, "\
+                        must only be attached to foreign modules");
+                }
+            } else if attr.check_name("wasm_custom_section") {
+                if target != Target::Const {
+                    self.tcx.sess.span_err(attr.span, "only allowed on consts");
+                }
+
+                if attr.value_str().is_none() {
+                    self.tcx.sess.span_err(attr.span, "must be of the form \
+                        #[wasm_custom_section = \"foo\"]");
                 }
             }
         }
 
+        if target == Target::ForeignMod &&
+            !has_wasm_import_module &&
+            self.tcx.sess.target.target.arch == "wasm32" &&
+            false // FIXME: eventually enable this warning when stable
+        {
+            self.tcx.sess.span_warn(item.span, "\
+                must have a #[wasm_import_module = \"...\"] attribute, this \
+                will become a hard error before too long");
+        }
+
         self.check_repr(item, target);
     }
 

src/librustc/ich/impls_cstore.rs

@@ -33,7 +33,12 @@ impl_stable_hash_for!(struct middle::cstore::NativeLibrary {
     kind,
     name,
     cfg,
-    foreign_items
+    foreign_module
+});
+
+impl_stable_hash_for!(struct middle::cstore::ForeignModule {
+    foreign_items,
+    def_id
 });
 
 impl_stable_hash_for!(enum middle::cstore::LinkagePreference {

src/librustc/middle/cstore.rs

@@ -132,7 +132,13 @@ pub struct NativeLibrary {
     pub kind: NativeLibraryKind,
     pub name: Symbol,
     pub cfg: Option<ast::MetaItem>,
+    pub foreign_module: Option<DefId>,
+}
+
+#[derive(Clone, Hash, RustcEncodable, RustcDecodable)]
+pub struct ForeignModule {
     pub foreign_items: Vec<DefId>,
+    pub def_id: DefId,
 }
 
 pub enum LoadedMacro {

src/librustc/middle/dead.rs

@@ -318,6 +318,11 @@ fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt,
         return true;
     }
 
+    // These constants are special for wasm
+    if attr::contains_name(attrs, "wasm_custom_section") {
+        return true;
+    }
+
     tcx.lint_level_at_node(lint::builtin::DEAD_CODE, id).0 == lint::Allow
 }
 

src/librustc/ty/maps/config.rs

@@ -430,6 +430,12 @@ impl<'tcx> QueryDescription<'tcx> for queries::native_libraries<'tcx> {
     }
 }
 
+impl<'tcx> QueryDescription<'tcx> for queries::foreign_modules<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("looking up the foreign modules of a linked crate")
+    }
+}
+
 impl<'tcx> QueryDescription<'tcx> for queries::plugin_registrar_fn<'tcx> {
     fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
         format!("looking up the plugin registrar for a crate")
@@ -678,6 +684,12 @@ impl<'tcx> QueryDescription<'tcx> for queries::instance_def_size_estimate<'tcx>
     }
 }
 
+impl<'tcx> QueryDescription<'tcx> for queries::wasm_custom_sections<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("custom wasm sections for a crate")
+    }
+}
+
 impl<'tcx> QueryDescription<'tcx> for queries::generics_of<'tcx> {
     #[inline]
     fn cache_on_disk(def_id: Self::Key) -> bool {
@@ -699,6 +711,18 @@ impl<'tcx> QueryDescription<'tcx> for queries::program_clauses_for<'tcx> {
     }
 }
 
+impl<'tcx> QueryDescription<'tcx> for queries::wasm_import_module_map<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("wasm import module map")
+    }
+}
+
+impl<'tcx> QueryDescription<'tcx> for queries::dllimport_foreign_items<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("wasm import module map")
+    }
+}
+
 macro_rules! impl_disk_cacheable_query(
     ($query_name:ident, |$key:tt| $cond:expr) => {
         impl<'tcx> QueryDescription<'tcx> for queries::$query_name<'tcx> {

src/librustc/ty/maps/mod.rs

@@ -18,7 +18,7 @@ use infer::canonical::{Canonical, QueryResult};
 use lint;
 use middle::borrowck::BorrowCheckResult;
 use middle::cstore::{ExternCrate, LinkagePreference, NativeLibrary,
-                     ExternBodyNestedBodies};
+                     ExternBodyNestedBodies, ForeignModule};
 use middle::cstore::{NativeLibraryKind, DepKind, CrateSource, ExternConstBody};
 use middle::privacy::AccessLevels;
 use middle::reachable::ReachableSet;
@@ -320,6 +320,9 @@ define_maps! { <'tcx>
 
 
     [] fn native_libraries: NativeLibraries(CrateNum) -> Lrc<Vec<NativeLibrary>>,
+
+    [] fn foreign_modules: ForeignModules(CrateNum) -> Lrc<Vec<ForeignModule>>,
+
     [] fn plugin_registrar_fn: PluginRegistrarFn(CrateNum) -> Option<DefId>,
     [] fn derive_registrar_fn: DeriveRegistrarFn(CrateNum) -> Option<DefId>,
     [] fn crate_disambiguator: CrateDisambiguator(CrateNum) -> CrateDisambiguator,
@@ -331,6 +334,8 @@ define_maps! { <'tcx>
     [] fn all_trait_implementations: AllTraitImplementations(CrateNum)
         -> Lrc<Vec<DefId>>,
 
+    [] fn dllimport_foreign_items: DllimportForeignItems(CrateNum)
+        -> Lrc<FxHashSet<DefId>>,
     [] fn is_dllimport_foreign_item: IsDllimportForeignItem(DefId) -> bool,
     [] fn is_statically_included_foreign_item: IsStaticallyIncludedForeignItem(DefId) -> bool,
     [] fn native_library_kind: NativeLibraryKind(DefId)
@@ -424,6 +429,10 @@ define_maps! { <'tcx>
     [] fn features_query: features_node(CrateNum) -> Lrc<feature_gate::Features>,
 
     [] fn program_clauses_for: ProgramClausesFor(DefId) -> Lrc<Vec<Clause<'tcx>>>,
+
+    [] fn wasm_custom_sections: WasmCustomSections(CrateNum) -> Lrc<Vec<DefId>>,
+    [] fn wasm_import_module_map: WasmImportModuleMap(CrateNum)
+        -> Lrc<FxHashMap<DefId, String>>,
 }
 
 //////////////////////////////////////////////////////////////////////

src/librustc/ty/maps/plumbing.rs

@@ -886,6 +886,9 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
             force!(all_trait_implementations, krate!());
         }
 
+        DepKind::DllimportForeignItems => {
+            force!(dllimport_foreign_items, krate!());
+        }
         DepKind::IsDllimportForeignItem => {
             force!(is_dllimport_foreign_item, def_id!());
         }
@@ -940,6 +943,9 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
         DepKind::Features => { force!(features_query, LOCAL_CRATE); }
 
         DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); }
+        DepKind::WasmCustomSections => { force!(wasm_custom_sections, krate!()); }
+        DepKind::WasmImportModuleMap => { force!(wasm_import_module_map, krate!()); }
+        DepKind::ForeignModules => { force!(foreign_modules, krate!()); }
     }
 
     true

src/librustc_metadata/creader.rs

@@ -12,7 +12,6 @@
 
 use cstore::{self, CStore, CrateSource, MetadataBlob};
 use locator::{self, CratePaths};
-use native_libs::relevant_lib;
 use schema::CrateRoot;
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 
@@ -230,7 +229,7 @@ impl<'a> CrateLoader<'a> {
             .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
             .collect();
 
-        let mut cmeta = cstore::CrateMetadata {
+        let cmeta = cstore::CrateMetadata {
             name,
             extern_crate: Lock::new(None),
             def_path_table: Lrc::new(def_path_table),
@@ -250,25 +249,8 @@ impl<'a> CrateLoader<'a> {
                 rlib,
                 rmeta,
             },
-            // Initialize this with an empty set. The field is populated below
-            // after we were able to deserialize its contents.
-            dllimport_foreign_items: FxHashSet(),
         };
 
-        let dllimports: FxHashSet<_> = cmeta
-            .root
-            .native_libraries
-            .decode((&cmeta, self.sess))
-            .filter(|lib| relevant_lib(self.sess, lib) &&
-                          lib.kind == cstore::NativeLibraryKind::NativeUnknown)
-            .flat_map(|lib| {
-                assert!(lib.foreign_items.iter().all(|def_id| def_id.krate == cnum));
-                lib.foreign_items.into_iter().map(|def_id| def_id.index)
-            })
-            .collect();
-
-        cmeta.dllimport_foreign_items = dllimports;
-
         let cmeta = Lrc::new(cmeta);
         self.cstore.set_crate_data(cnum, cmeta.clone());
         (cnum, cmeta)

src/librustc_metadata/cstore.rs

@@ -20,7 +20,7 @@ use rustc::middle::cstore::{DepKind, ExternCrate, MetadataLoader};
 use rustc::session::{Session, CrateDisambiguator};
 use rustc_back::PanicStrategy;
 use rustc_data_structures::indexed_vec::IndexVec;
-use rustc::util::nodemap::{FxHashMap, FxHashSet, NodeMap};
+use rustc::util::nodemap::{FxHashMap, NodeMap};
 
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 use syntax::{ast, attr};
@@ -30,7 +30,7 @@ use syntax_pos;
 
 pub use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind, LinkagePreference};
 pub use rustc::middle::cstore::NativeLibraryKind::*;
-pub use rustc::middle::cstore::{CrateSource, LibSource};
+pub use rustc::middle::cstore::{CrateSource, LibSource, ForeignModule};
 
 pub use cstore_impl::{provide, provide_extern};
 
@@ -84,8 +84,6 @@ pub struct CrateMetadata {
     pub source: CrateSource,
 
     pub proc_macros: Option<Vec<(ast::Name, Lrc<SyntaxExtension>)>>,
-    // Foreign items imported from a dylib (Windows only)
-    pub dllimport_foreign_items: FxHashSet<DefIndex>,
 }
 
 pub struct CStore {