Introduce -Zsplit-metadata option

Cargo.lock

@@ -3595,7 +3595,6 @@ dependencies = [
  "rustc_symbol_mangling",
  "rustc_target",
  "smallvec",
- "snap",
  "tempfile",
  "thorin-dwp",
  "tracing",
@@ -3967,7 +3966,6 @@ dependencies = [
  "rustc_span",
  "rustc_target",
  "smallvec",
- "snap",
  "tracing",
 ]
 
@@ -4795,12 +4793,6 @@ version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
 
-[[package]]
-name = "snap"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da73c8f77aebc0e40c300b93f0a5f1bece7a248a36eee287d4e095f35c7b7d6e"
-
 [[package]]
 name = "socket2"
 version = "0.4.1"

compiler/rustc_codegen_ssa/Cargo.toml

@@ -16,7 +16,6 @@ jobserver = "0.1.22"
 tempfile = "3.2"
 thorin-dwp = "0.2"
 pathdiff = "0.2.0"
-snap = "1"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 regex = "1.4"
 

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -359,7 +359,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
             // metadata in rlib files is wrapped in a "dummy" object file for
             // the target platform so the rlib can be processed entirely by
             // normal linkers for the platform.
-            let metadata = create_rmeta_file(sess, codegen_results.metadata.raw_data());
+            let metadata = create_rmeta_file(sess, codegen_results.metadata.maybe_reference());
             ab.add_file(&emit_metadata(sess, &metadata, tmpdir));
         }
 

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -1,7 +1,6 @@
 //! Reading of the rustc metadata for rlibs and dylibs
 
 use std::fs::File;
-use std::io::Write;
 use std::path::Path;
 
 use object::write::{self, StandardSegment, Symbol, SymbolSection};
@@ -10,8 +9,6 @@ use object::{
     SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope,
 };
 
-use snap::write::FrameEncoder;
-
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::owning_ref::OwningRef;
 use rustc_data_structures::rustc_erase_owner;
@@ -255,17 +252,16 @@ pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> Vec<u8> {
 // As a result, we choose a slightly shorter name! As to why
 // `.note.rustc` works on MinGW, see
 // https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197
+// TODO rename function
 pub fn create_compressed_metadata_file(
     sess: &Session,
     metadata: &EncodedMetadata,
     symbol_name: &str,
 ) -> Vec<u8> {
-    let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
-    FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap();
     let mut file = if let Some(file) = create_object_file(sess) {
         file
     } else {
-        return compressed.to_vec();
+        return metadata.maybe_reference().to_vec();
     };
     let section = file.add_section(
         file.segment_name(StandardSegment::Data).to_vec(),
@@ -279,14 +275,14 @@ pub fn create_compressed_metadata_file(
         }
         _ => {}
     };
-    let offset = file.append_section_data(section, &compressed, 1);
+    let offset = file.append_section_data(section, metadata.maybe_reference(), 1);
 
     // For MachO and probably PE this is necessary to prevent the linker from throwing away the
     // .rustc section. For ELF this isn't necessary, but it also doesn't harm.
     file.add_symbol(Symbol {
         name: symbol_name.as_bytes().to_vec(),
         value: offset,
-        size: compressed.len() as u64,
+        size: metadata.maybe_reference().len() as u64,
         kind: SymbolKind::Data,
         scope: SymbolScope::Dynamic,
         weak: false,

compiler/rustc_interface/src/passes.rs

@@ -1036,7 +1036,7 @@ fn encode_and_write_metadata(
     enum MetadataKind {
         None,
         Uncompressed,
-        Compressed,
+        Compressed, // TODO remove this variant
     }
 
     let metadata_kind = tcx
@@ -1074,7 +1074,7 @@ fn encode_and_write_metadata(
             .tempdir_in(out_filename.parent().unwrap())
             .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
         let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
-        let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir);
+        let metadata_filename = emit_metadata(tcx.sess, metadata.full(), &metadata_tmpdir);
         if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) {
             tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
         }

compiler/rustc_interface/src/tests.rs

@@ -689,6 +689,7 @@ fn test_debugging_options_tracking_hash() {
     untracked!(self_profile_events, Some(vec![String::new()]));
     untracked!(span_debug, true);
     untracked!(span_free_formats, true);
+    untracked!(split_metadata, true);
     untracked!(temps_dir, Some(String::from("abc")));
     untracked!(terminal_width, Some(80));
     untracked!(threads, 99);

compiler/rustc_metadata/Cargo.toml

@@ -9,7 +9,6 @@ doctest = false
 [dependencies]
 libloading = "0.7.1"
 odht = { version = "0.3.1", features = ["nightly"] }
-snap = "1"
 tracing = "0.1"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_middle = { path = "../rustc_middle" }

compiler/rustc_metadata/src/locator.rs

@@ -213,7 +213,7 @@
 //! metadata::locator or metadata::creader for all the juicy details!
 
 use crate::creader::Library;
-use crate::rmeta::{rustc_version, MetadataBlob, METADATA_HEADER};
+use crate::rmeta::{rustc_version, MetadataBlob};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::memmap::Mmap;
@@ -231,18 +231,16 @@ use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
 use rustc_target::spec::{Target, TargetTriple};
 
-use snap::read::FrameDecoder;
 use std::fmt::Write as _;
-use std::io::{Read, Result as IoResult, Write};
+use std::io::{Result as IoResult, Write};
 use std::path::{Path, PathBuf};
-use std::{cmp, fmt, fs};
+use std::{fmt, fs};
 use tracing::{debug, info};
 
 #[derive(Clone)]
 crate struct CrateLocator<'a> {
     // Immutable per-session configuration.
     only_needs_metadata: bool,
-    sysroot: &'a Path,
     metadata_loader: &'a dyn MetadataLoader,
 
     // Immutable per-search configuration.
@@ -309,7 +307,6 @@ impl<'a> CrateLocator<'a> {
 
         CrateLocator {
             only_needs_metadata,
-            sysroot: &sess.sysroot,
             metadata_loader,
             crate_name,
             exact_paths: if hash.is_none() {
@@ -504,9 +501,8 @@ impl<'a> CrateLocator<'a> {
     // the errors and notes are emitted about the set of libraries.
     //
     // With only one library in the set, this function will extract it, and then
-    // read the metadata from it if `*slot` is `None`. If the metadata couldn't
-    // be read, it is assumed that the file isn't a valid rust library (no
-    // errors are emitted).
+    // read the metadata from it. If the metadata couldn't be read, it is assumed
+    // that the file isn't a valid rust library (no errors are emitted).
     fn extract_one(
         &mut self,
         m: FxHashMap<PathBuf, PathKind>,
@@ -522,17 +518,8 @@ impl<'a> CrateLocator<'a> {
         //
         // See also #68149 which provides more detail on why emitting the
         // dependency on the rlib is a bad thing.
-        //
-        // We currently do not verify that these other sources are even in sync,
-        // and this is arguably a bug (see #10786), but because reading metadata
-        // is quite slow (especially from dylibs) we currently do not read it
-        // from the other crate sources.
-        if slot.is_some() {
-            if m.is_empty() || !self.needs_crate_flavor(flavor) {
-                return Ok(None);
-            } else if m.len() == 1 {
-                return Ok(Some(m.into_iter().next().unwrap()));
-            }
+        if slot.is_some() && !self.needs_crate_flavor(flavor) {
+            return Ok(None);
         }
 
         let mut ret: Option<(PathBuf, PathKind)> = None;
@@ -552,12 +539,31 @@ impl<'a> CrateLocator<'a> {
                 match get_metadata_section(self.target, flavor, &lib, self.metadata_loader) {
                     Ok(blob) => {
                         if let Some(h) = self.crate_matches(&blob, &lib) {
+                            if blob.is_reference_only() {
+                                if slot.is_none() {
+                                    todo!("return error");
+                                }
+                            }
                             (h, blob)
                         } else {
                             info!("metadata mismatch");
                             continue;
                         }
                     }
+                    Err(MetadataError::VersionMismatch(found_version)) => {
+                        // The file was present and created by the same compiler version, but we
+                        // couldn't load it for some reason.  Give a hard error instead of silently
+                        // ignoring it, but only if we would have given an error anyway.
+                        let rustc_version = rustc_version();
+                        info!(
+                            "Rejecting via version: expected {} got {}",
+                            rustc_version, found_version
+                        );
+                        self.crate_rejections
+                            .via_version
+                            .push(CrateMismatch { path: lib, got: found_version });
+                        continue;
+                    }
                     Err(MetadataError::LoadFailure(err)) => {
                         info!("no metadata found: {}", err);
                         // The file was present and created by the same compiler version, but we
@@ -590,32 +596,6 @@ impl<'a> CrateLocator<'a> {
                 continue;
             }
 
-            // Ok so at this point we've determined that `(lib, kind)` above is
-            // a candidate crate to load, and that `slot` is either none (this
-            // is the first crate of its kind) or if some the previous path has
-            // the exact same hash (e.g., it's the exact same crate).
-            //
-            // In principle these two candidate crates are exactly the same so
-            // we can choose either of them to link. As a stupidly gross hack,
-            // however, we favor crate in the sysroot.
-            //
-            // You can find more info in rust-lang/rust#39518 and various linked
-            // issues, but the general gist is that during testing libstd the
-            // compilers has two candidates to choose from: one in the sysroot
-            // and one in the deps folder. These two crates are the exact same
-            // crate but if the compiler chooses the one in the deps folder
-            // it'll cause spurious errors on Windows.
-            //
-            // As a result, we favor the sysroot crate here. Note that the
-            // candidates are all canonicalized, so we canonicalize the sysroot
-            // as well.
-            if let Some((prev, _)) = &ret {
-                let sysroot = self.sysroot;
-                let sysroot = sysroot.canonicalize().unwrap_or_else(|_| sysroot.to_path_buf());
-                if prev.starts_with(&sysroot) {
-                    continue;
-                }
-            }
             *slot = Some((hash, metadata));
             ret = Some((lib, kind));
         }
@@ -628,14 +608,18 @@ impl<'a> CrateLocator<'a> {
     }
 
     fn crate_matches(&mut self, metadata: &MetadataBlob, libpath: &Path) -> Option<Svh> {
-        let rustc_version = rustc_version();
-        let found_version = metadata.get_rustc_version();
-        if found_version != rustc_version {
-            info!("Rejecting via version: expected {} got {}", rustc_version, found_version);
-            self.crate_rejections
-                .via_version
-                .push(CrateMismatch { path: libpath.to_path_buf(), got: found_version });
-            return None;
+        if metadata.is_reference_only() {
+            let hash = metadata.get_hash();
+            if let Some(expected_hash) = self.hash {
+                if hash != expected_hash {
+                    info!("Rejecting via hash: expected {} got {}", expected_hash, hash);
+                    self.crate_rejections
+                        .via_hash
+                        .push(CrateMismatch { path: libpath.to_path_buf(), got: hash.to_string() });
+                    return None;
+                }
+            }
+            return Some(hash);
         }
 
         let root = metadata.get_root();
@@ -757,34 +741,7 @@ fn get_metadata_section<'p>(
             loader.get_rlib_metadata(target, filename).map_err(MetadataError::LoadFailure)?
         }
         CrateFlavor::Dylib => {
-            let buf =
-                loader.get_dylib_metadata(target, filename).map_err(MetadataError::LoadFailure)?;
-            // The header is uncompressed
-            let header_len = METADATA_HEADER.len();
-            debug!("checking {} bytes of metadata-version stamp", header_len);
-            let header = &buf[..cmp::min(header_len, buf.len())];
-            if header != METADATA_HEADER {
-                return Err(MetadataError::LoadFailure(format!(
-                    "invalid metadata version found: {}",
-                    filename.display()
-                )));
-            }
-
-            // Header is okay -> inflate the actual metadata
-            let compressed_bytes = &buf[header_len..];
-            debug!("inflating {} bytes of compressed metadata", compressed_bytes.len());
-            // Assume the decompressed data will be at least the size of the compressed data, so we
-            // don't have to grow the buffer as much.
-            let mut inflated = Vec::with_capacity(compressed_bytes.len());
-            match FrameDecoder::new(compressed_bytes).read_to_end(&mut inflated) {
-                Ok(_) => rustc_erase_owner!(OwningRef::new(inflated).map_owner_box()),
-                Err(_) => {
-                    return Err(MetadataError::LoadFailure(format!(
-                        "failed to decompress metadata: {}",
-                        filename.display()
-                    )));
-                }
-            }
+            loader.get_dylib_metadata(target, filename).map_err(MetadataError::LoadFailure)?
         }
         CrateFlavor::Rmeta => {
             // mmap the file, because only a small fraction of it is read.
@@ -806,13 +763,9 @@ fn get_metadata_section<'p>(
         }
     };
     let blob = MetadataBlob::new(raw_bytes);
-    if blob.is_compatible() {
-        Ok(blob)
-    } else {
-        Err(MetadataError::LoadFailure(format!(
-            "invalid metadata version found: {}",
-            filename.display()
-        )))
+    match blob.check_compatibility() {
+        Ok(()) => Ok(blob),
+        Err(version) => Err(MetadataError::VersionMismatch(version)),
     }
 }
 
@@ -926,6 +879,8 @@ enum MetadataError<'a> {
     NotPresent(&'a Path),
     /// The file was present and invalid.
     LoadFailure(String),
+    /// The file was present, but compiled with a different rustc version.
+    VersionMismatch(String),
 }
 
 impl fmt::Display for MetadataError<'_> {
@@ -935,6 +890,11 @@ impl fmt::Display for MetadataError<'_> {
                 f.write_str(&format!("no such file: '{}'", filename.display()))
             }
             MetadataError::LoadFailure(msg) => f.write_str(msg),
+            MetadataError::VersionMismatch(found_version) => f.write_str(&format!(
+                "rustc version mismatch. expected {}, found {}",
+                rustc_version(),
+                found_version,
+            )),
         }
     }
 }